# A tutorial for this file is available at www.relataly.com
# Tested with Python 3.9.13, 1.2.0, Pandas 1.3.4, OpenAI 0.27.3, Tweepy 4.13.0, Requests 2.26.0, 

import logging
import pandas as pd
import openai
import tweepy
import csv
import requests

# Set API Keys and Authentification
logging.info('Setting NewsAPI API Key')
NEWSAPI_API_KEY = <your API key> # replace with own API key

logging.info('Setting OpenAI API Key')
openai.api_key = <your API key> # replace with own API key

logging.info('Setting Twitter API Key')
auth=tweepy.OAuthHandler(<your API key>,
                         <your API secret>)
auth.set_access_token(<your access token>,
                      <your access secret>)
twitter_api=tweepy.API(auth)

CSV_NAME = 'news_log.csv'

#### NewsAPI
def fetch_news(number=10):
    # Fetch tech news from NewsAPI
    url = f"https://newsapi.org/v2/top-headlines?country=us&category=technology&apiKey={NEWSAPI_API_KEY}"
    response = requests.get(url).json()
    news_items = response["articles"]
    df = pd.DataFrame(news_items)
    df = df[["title", "description", "url"]].dropna()
    return df.head(number)

#### OpenAI Engine
def openai_request(instructions, task, sample = [], model_engine='gpt-3.5-turbo'):
    prompt = [{"role": "system", "content": instructions }, 
              {"role": "user", "content": task }]
    prompt = sample + prompt
    completion = openai.ChatCompletion.create(model=model_engine, messages=prompt, temperature=0.2, max_tokens=400)
    return completion.choices[0].message.content


#### Define OpenAI Prompt for news Relevance
def select_relevant_news_prompt(news_articles, topics, n):    
    instructions = f'Your task is to examine a list of News and return a list of boolean values that indicate which of the News are in scope of a list of topics. \
    Return a list of True or False values that indicate the relevance of the News.'
    task =  f"{news_articles} /n {topics}?" 
    sample = [
        {"role": "user", "content": f"[new AI model available from Nvidia, We Exploded the AMD Ryzen 7, Release of b2 Game, XGBoost 3.0 improvices Decision Forest Algorithms, New Zelda Game Now Available, Ukraine Uses a New Weapon] /n {topics}?"},
        {"role": "assistant", "content": "[True, False, False, True, False, False]"},
        {"role": "user", "content": f"[Giga Giraff found in Sounth Africa, We Exploded the AMD Ryzen 7, Release of b2 Game, Donald Trump to make a come back, New Zelda Game Now Available, Ukraine Uses a New Weapon] /n {topics}?"}, 
        {"role": "assistant", "content": "[False, False, False, False, False, False]"}]
    return instructions, task, sample


#### Define OpenAI Prompt for news Relevance
def check_previous_posts_prompt(title, old_posts):    
    instructions = f'Your objective is to compare a news title with a list of previous news and determine whether it covers a similar topic that was already covered by a previous title. \
        Rate the overlap on a scale between 1 and 10 with 1 beeing a full overlap and 10 representing an unrelated topic. "'
    task =  f"'{title}.' Previous News: {old_posts}."
    sample = [
        {"role": "user", "content": "'Nvidia launches new AI model.' Previous News: [new AI model available from Nvidia, We Exploded the AMD Ryzen 7 7800X3D, The Lara Croft Collection For Switch Has Been Rated By The ESRB]."},
        {"role": "assistant", "content": "1"},
        {"role": "user", "content": "'Big Explosion of an AMD Ryzen 7.' Previous News: [Improving Mental Wellbeing Through Physical Activity, The Lara Croft Collection For Switch Has Been Rated By The ESRB]."},
        {"role": "assistant", "content": "10"},
        {"role": "user", "content": "'new AI model available from Google.' Previous News : [new AI model available from Nvidia, The Lara Croft Collection For Switch Has Been Rated By The ESRB]."},
        {"role": "assistant", "content": "9"},
        {"role": "user", "content": "'What Really Made Geoffrey Hinton Into an AI Doomer - WIRED.' Previous News : [Why AI's 'godfather' Geoffrey Hinton quit Google, new AI model available from Nvidia, The Lara Croft Collection For Switch Has Been Rated By The ESRB]."},
        {"role": "assistant", "content": "4"}]
    return instructions, task, sample


#### Define OpenAI Prompt for News Tweet
def create_tweet_prompt(title, description, tiny_url):
    instructions = f'You are a twitter user that creates tweets with a maximum length of 280 characters.'
    task = f"Create an informative tweet on twitter based on the following news title and description. \
        The tweet must use a maximum of 280 characters. \
        Include the {tiny_url}. But do not include any other urls.\
        Title: {title}. \
        Description: {description}. \
        Use hashtags to reach a wider audience. \
        Do not include any emojis in the tweet"
    return instructions, task


#### Define OpenAI Prompt for news Relevance
def previous_post_check(title, old_posts):
    instructions, task, sample = check_previous_posts_prompt(title, old_posts)
    response = openai_request(instructions, task, sample)
    return eval(response)


#### Define OpenAI Prompt for News Tweet
def create_fact_tweet_prompt(old_terms):
    instructions = f'You are a twitter user that creates tweets with a length below 280 characters.'
    task = f"Choose a technical term from the field of AI, machine learning or data science. Then create a twitter tweet that describes the term. Just return a python dictionary with the term and the tweet. "
    # if old terms not empty
    if old_terms != []:
        avoid_terms =f'Avoid the following terms, because you have previously tweetet about them: {old_terms}'
        task = task + avoid_terms
    sample = [
        {"role": "user", "content": f"Choose a technical term from the field of AI, machine learning or data science. Then create a twitter tweet that describes the term. Just return a python dictionary with the term and the tweet."},
        {"role": "assistant", "content": "{'GradientDescent': '#GradientDescent is a popular optimization algorithm used to minimize the error of a model by adjusting its parameters. \
         It works by iteratively calculating the gradient of the error with respect to the parameters and updating them accordingly. #ML'}"}]
    return instructions, task, sample

# Load previous information from a csv file
def get_history_from_csv(csv_name):
    try:
        # try loading the csv file
        df = pd.read_csv(csv_name)
    except:
        # create the csv file
        df = pd.DataFrame(columns=['title'])
        df.to_csv(csv_name, index=False)
    return df

def check_tweet_length(tweet):
    return False if len(tweet) > 280 else True


# Create the fact tweet
def create_fact_tweet(chance_for_tweet = 0.5):
    df_old_facts = get_history_from_csv(CSV_FACTS_NAME)

    if random.random() < chance_for_tweet:
        # create a fact tweet
        instructions, tasks, sample = create_fact_tweet_prompt(list(df_old_facts.tail(10)['title']))
        tweet = openai_request(instructions, tasks, sample)
        tweet_text = list(eval(tweet).values())[0]

        # tweet creation
        print(f'Creating fact tweet: {tweet_text}')
        
        # check tweet length and post tweet
        if check_tweet_length(tweet):
            twitter_auth().update_status(tweet_text)
            term = list(eval(tweet).keys())[0]
            # save the fact in the csv file
            with open(f'{CSV_FACTS_NAME}', 'a', newline='') as file:
                            writer = csv.writer(file)
                            writer.writerow([term])
        else: 
            print('error tweet too long')
    else:
        print('No fact tweet created')
    

def create_news_tweet(title, description, url):
    # create tiny url
    tiny_url = create_tiny_url(url)

    # define prompt for tweet creation
    instructions, task = create_tweet_prompt(title, description, tiny_url)
    tweet_text = openai_request(instructions, task)

    print(f'Creating new tweet: {tweet_text}')
    # check tweet length and post tweet
    if check_tweet_length(tweet_text):
            status = twitter_auth().update_status(tweet_text)
            print(f"Tweeted: {title}")
            with open(f'{CSV_NEWS_NAME}', 'a', newline='') as file:
                writer = csv.writer(file)
                writer.writerow([title])
    else: 
        status = 'error tweet too long'
    return status


def create_tiny_url(url):
    response = requests.get(f'http://tinyurl.com/api-create.php?url={url}')
    return response.text

#### Main Bot
def main_bot():
    # Read the old CSV data
    # try opening the csv file and creeate it if it does not exist
    df_old_news = get_history_from_csv(CSV_NEWS_NAME)
    df_old_news = df_old_news.tail(16)
    # Fetch news data
    df = fetch_news(12)
    
    # Check the Relevance of the News and Filter those not relevant
    relevant_topics ="[AI, Machine Learning, Data Science, OpenAI, Artificial Intelligence, Data Mining, Data Analytics]"
    instructions, task, sample = select_relevant_news_prompt(list(df['title']), relevant_topics, len(list(df['title'])))
    relevance = openai_request(instructions, task, sample)
    relevance_list = eval(relevance)

    s = 0
    df = df[relevance_list]
    if len(df) > 0:
        for index, row in df.iterrows():
            if s == 1:
                break
            logging.info('info:' + row['title'])
            title = row['title']
            title = title.replace("'", "")
            description = row['description']
            url = row['url']            
                                             
            if (title not in df_old_news.title.values):
                doublicate_check = previous_post_check(title, list(df_old_news.tail(10)['title']))
                if doublicate_check > 3:
                    # Create a tweet
                    response = create_news_tweet(title, description, url)
            
                else: 
                    print(f"Already tweeted: {title}")
            else: 
                print("No news articles found")
                create_fact_tweet(chance_for_tweet=0.5)

main_bot()

No news articles found
No fact tweet created
No news articles found
Creating fact tweet: Looking for a way to cluster your data? Try #KMeans! It is an unsupervised learning algorithm that groups similar data points together based on their distance to a centroid. #MachineLearning #DataScience
Creating tweet: {'KMeans': 'Looking for a way to cluster your data? Try #KMeans! It is an unsupervised learning algorithm that groups similar data points together based on their distance to a centroid. #MachineLearning #DataScience'}

import pandas as pd
import numpy as np
import json
import requests
import datetime as dt
import logging
import threading
import time
from __future__ import print_function

import tweepy 
import gate_api
from gate_api.exceptions import ApiException, GateApiException
from twitter_secrets import twitter_secrets as ts # place the twitter_secrets file under <User>/anaconda3/Lib

class Prices:
    """Class that uses the gate api to retrieve currency data."""

    def __init__(self, config):
        self._config = config
        self._logger = logging.getLogger(__name__)
        configuration = gate_api.Configuration(host="https://api.gateio.ws/api/v4")
        api_client = gate_api.ApiClient(configuration)
        self._api_instance = gate_api.SpotApi(api_client)
        self._price_history = {}
        self._cont_update_thread = None
        self._stop_cont_update_thread = None
        self._price_history_lock = threading.Lock()

    def get_price_history(self):
        """Returns a dictionary with the price histories for the currencies."""
        return self._price_history, self._price_history_lock

    def get_latest_prices(self):
        """Gets new price data and adds the values to a DataFrame.

        Returns the DataFrame in a dictionary with the currencies as keys."""
        timestamp = dt.datetime.now()
        try:
            api_response = self._api_instance.list_tickers()
        except GateApiException as e:
            logging.warning(
                "Gate api exception, label: %s, message: %s\n" % (e.label, e.message)
            )
            return {}
        except ApiException as e:
            logging.warning("Exception when calling SpotApi->list_tickers: %s\n" % e)
            return {}
        latest_prices = {}
        for response in api_response:
            currency = response.currency_pair
            if "USDT" not in currency or "BEAR" in currency:
                continue
            value_dict = {
                "base_volume": pd.to_numeric(response.base_volume),
                "change_percentage": pd.to_numeric(response.change_percentage),
                "etf_leverage": pd.to_numeric(response.etf_leverage),
                "etf_net_value": pd.to_numeric(response.etf_net_value),
                "etf_pre_net_value": pd.to_numeric(response.etf_pre_net_value),
                "etf_pre_timestamp": response.etf_pre_timestamp,
                "high_24h": pd.to_numeric(response.high_24h),
                "highest_bid": pd.to_numeric(response.highest_bid),
                "high_bid": pd.to_numeric(response.highest_bid),
                "last": pd.to_numeric(response.last),
                "low_24h": pd.to_numeric(response.low_24h),
                "lowest_ask": pd.to_numeric(response.lowest_ask),
                "quote_volume": pd.to_numeric(response.quote_volume),
                "timestamp": timestamp,
            }
            latest_prices[currency] = pd.DataFrame(value_dict, index=[1])
        return latest_prices

    def start_cont_update(self):
        self._stop_cont_update_thread = threading.Event()
        self._stop_cont_update_thread.clear()
        self._cont_update_thread = threading.Thread(
            target=self._cont_update,
            args=(
                self._stop_cont_update_thread,
                self._price_history_lock,
            ),
        )
        self._cont_update_thread.start()
        self._logger.info("Started continuous price logging")

    def _cont_update(self, stop_event, lock):
        """Continuously adds new prices to the price history."""
        while not stop_event.is_set():
            start_time = time.time()
            lock.acquire()
            for currency, df in self.get_latest_prices().items():
                if currency in self._price_history.keys():
                    self._price_history[currency] = self._price_history[
                        currency
                    ].append(df, ignore_index=True)
                else:
                    self._price_history[currency] = df
            lock.release()
            self._logger.debug("Currency_dfs updated")
            self._wait_before_update(start_time)

    def _wait_before_update(self, start_time):
        elapsed_time = time.time() - start_time
        self._logger.debug(f"Elapsed time: {elapsed_time}")
        if elapsed_time > self._config["price_update_delay"]:
            delay = 0
            self._logger.warning(
                #"It took longer to retrieve the price data than the update_delay!"
            )
        else:
            delay = self._config["price_update_delay"] - elapsed_time
        self._logger.debug(f"Waiting {delay}s until next update")
        time.sleep(delay)

def calc_indicators(price_history):
    indicators = {}
    indicators_over_all = calc_indicators_over_all(price_history)
    for currency, df in price_history.items():
        if len(df) <= 2:
            logging.getLogger().debug(
                f"Skipped '{currency} when calculating indicators due to a lack of information"
            )
            continue
        volume = df["base_volume"].iloc[-1]
        last_price = df["last"].iloc[-1]
        moving_avg_price = df["last"].mean()
        moving_average_volume = df["base_volume"].mean()
        moving_average_deviation_percent = np.round(
            div(last_price, moving_avg_price) - 1, 2
        )

        price_before = df["last"].iloc[-2]
        price_delta = last_price - price_before
        price_delta_p = div(price_delta, last_price)
        price_delta_before = price_before - df["last"].iloc[-3]
        price_delta_p_before = div((price_before - df["last"].iloc[-3]), price_before)
        low_24h = df["low_24h"].iloc[-1]
        high_24h = df["high_24h"].iloc[-1]
        low_high_diff_p = div(high_24h - low_24h, low_24h)
        change_percentage = df["change_percentage"].iloc[-1]

        indicator_values = {
            "last_price": last_price,
            "price_before": price_before,
            "volume": volume,
            "moving_avg_price": moving_avg_price,
            "moving_average_volume": moving_average_volume,
            "moving_average_deviation_percent": moving_average_deviation_percent,
            "price_delta_p": price_delta_p,
            "price_delta": price_delta,
            "price_delta_before": price_delta_before,
            "price_delta_p_before": price_delta_p_before,
            "high_24h": high_24h,
            "low_24h": low_24h,
            "low_high_diff_p": low_high_diff_p,
            "change_percentage": change_percentage,
        }
        indicator_values.update(indicators_over_all)
        indicators[currency] = indicator_values
    return indicators


def calc_indicators_over_all(price_history):
    avg_change_p = 0
    for currency, df in price_history.items():
        avg_change_p += df["change_percentage"].iloc[-1]
    nr_of_currencies = len(price_history)
    avg_change_p = div(avg_change_p, nr_of_currencies)
    values = {
        "avg_change_p": avg_change_p,
    }
    return values


def div(dividend, divisor, alt_value=0.0):
    return dividend / divisor if divisor != 0 else alt_value

def check_signal(currency, indicators, cs_config):
    ind = indicators[currency]
    signal = ''
    if (ind['moving_avg_price'] > 0
            and ind['last_price'] > 0.0
            and abs(ind['price_delta']) > 0.0
            and abs(ind['price_delta_p']) >= cs_config["delta_threshold_p"]
            and ind['volume'] > 0
    ):
        # up
        if ind['price_delta'] > 0:
            movement_type = 'up +'
            if abs(ind['price_delta_p_before']) > cs_config["delta_threshold_p"]:
                if ind['price_delta_before'] <= 0:
                    movement_type = 'recovery from ' + str(ind['price_before']) + ' to ' + str(ind['last_price'])
                else:
                    if ind['price_delta_p'] * (1-cs_config["delta_threshold_p"]) > ind['price_delta_p_before']:
                        movement_type = 'upward trend accelerates +'
                    elif ind['price_delta_p'] < ind['price_delta_p_before'] * (1-cs_config["delta_threshold_p"]):
                        movement_type = 'upward trend slows down +'
                    elif ind['price_delta_p'] * (1+cs_config["delta_threshold_p"]) >= ind['price_delta_p_before'] >= ind['price_delta_p'] * (1-cs_config["delta_threshold_p"]):
                        movement_type = 'upward trend continues +'
        # down
        elif ind['price_delta'] < 0:
            movement_type = 'down '
            if abs(ind['price_delta_p_before']) > cs_config["delta_threshold_p"]:
                if ind['price_delta_before'] > 0:
                    movement_type = 'pullback from ' + str(ind['price_before']) + ' to ' + str(ind['last_price'])
                else:
                    if ind['price_delta_p'] * (1-cs_config["delta_threshold_p"]) > ind['price_delta_p_before']:
                        movement_type = 'down trend accelerates '
                    elif ind['price_delta_p'] * (1+cs_config["delta_threshold_p"]) >= ind['price_delta_p_before'] >= ind['price_delta_p'] * (1-cs_config["delta_threshold_p"]):
                        movement_type = 'down trend continues '
                    elif ind['price_delta_p'] < ind['price_delta_p_before'] * (1+cs_config["delta_threshold_p"]):
                        movement_type = 'downward trend slows down '

        signal = get_signal_log(movement_type, currency, ind['price_delta_p'], ind['last_price'],
                                ind['moving_avg_price'], ind['volume'], ind['price_delta'], ind['change_percentage'],
                                ind['high_24h'], ind['low_24h'], ind['low_high_diff_p'])

        check_24h_peak(currency, ind['last_price'], ind['low_24h'], ind['high_24h'])

    return signal
    # trade_signal


def check_24h_peak(currency, last_price, low_24h, high_24h):
    if last_price < low_24h:
        print(currency + ' new 24h low $' + str(last_price))
    elif last_price > high_24h:
        print(currency + ' new 24h high $' + str(last_price))


def get_signal_log(movement_type, currency, price_delta_p, last_price, moving_avg_price, volume, price_delta,
                   daily_up_p, high_24h, low_24h, low_high_diff_p):
    signal = f'{currency} {movement_type} ' \
             f'{np.round(price_delta_p * 100, 5)}% ' \
             f'MA:${np.round(moving_avg_price, 6)} ' \
             f'last_price:${np.round(last_price, 6)} ' \
             f'price delta:{np.round(price_delta, 6)} ' \
             f'volume:${np.round(volume, 1)} ' \
             f'daily_change:{np.round(daily_up_p, 2)}% ' \
             f'high_24h:${high_24h} ' \
             f'low_24h:${low_24h} ' \
             f'low_high_diff_p:{np.round(low_high_diff_p * 100, 2)}%'
    return signal

# Twitter Consumer API keys
CONSUMER_KEY    = "api123"
CONSUMER_SECRET = "api123"

# Twitter Access token & access token secret
ACCESS_TOKEN    = "api123"
ACCESS_SECRET   = "api123"

BEARER_TOKEN = "api123"

class TwitterSecrets:
    """Class that holds Twitter Secrets"""

    def __init__(self):
        self.CONSUMER_KEY    = CONSUMER_KEY
        self.CONSUMER_SECRET = CONSUMER_SECRET
        self.ACCESS_TOKEN    = ACCESS_TOKEN
        self.ACCESS_SECRET   = ACCESS_SECRET
        self.BEARER_TOKEN   = BEARER_TOKEN
        
        # Tests if keys are present
        for key, secret in self.__dict__.items():
            assert secret != "", f"Please provide a valid secret for: {key}"

twitter_secrets = TwitterSecrets()

consumer_key = ts.CONSUMER_KEY
consumer_secret = ts.CONSUMER_SECRET
access_token = ts.ACCESS_TOKEN
access_secret = ts.ACCESS_SECRET

### Print API Auth Data (leave disabled for security reasons)
# print(f'consumer_key: {consumer_key}')
# print(f'consumer_secret: {consumer_secret}')
# print(f'access_token: {access_token}')
# print(f'access_secret: {access_token}')

#authenticating to access the twitter API
auth=tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_secret)
api=tweepy.API(auth)

def send_pricechange_tweet(signal):
    api.update_status(f"{signal} \n {relataly_url}")

RUNS = 50 # the bot will stop after 50 price points
CYCLE_DELAY = 20 # the interval for checking the data and retrieving another price point
EVAL_PRICES_DELAY = 10
CURRENCY_PAIR = "" # the bot will retrieve data for all currency pairs listed on gate.io
PRICES_CONFIG = {"price_update_delay": 20}
TWITTER_ACTIVE = False

CHECK_SIGNAL_CONFIG = {
    "moving_avg_threshold_down_p": 0.10,
    "moving_avg_threshold_up_p": 0.10,
    "delta_threshold_p": 0.07,
    'enable_twitter': TWITTER_ACTIVE,
}

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO, format="\033[02m%(asctime)s %(levelname)s: %(message)s"
    )
    logger = logging.getLogger(__name__)
    prices = Prices(PRICES_CONFIG)
    prices.start_cont_update()
    currency_dfs = {}
    logging.info(f"Crypto bot is starting - please wait")
    logger.info(f"Collecting crypto data from gate.io for {EVAL_PRICES_DELAY}s")
    time.sleep(EVAL_PRICES_DELAY)
    logger.info(f"\n<< Crypto signal bot started :-) >>")
    logger.info(f"<< Checking prices every {CYCLE_DELAY} seconds >>")
    logger.info(f"Now checking for signals - please wait\n")
    for i in range(RUNS):
        price_history, lock = prices.get_price_history()
        lock.acquire()
        indicators = calc_indicators(price_history)
        lock.release()
        for currency in indicators.keys():
            if not indicators[currency]:
                continue
            signal = check_signal(
                currency,
                indicators,
                CHECK_SIGNAL_CONFIG,
            )
            if signal:
                logger.info(signal)
                if CHECK_SIGNAL_CONFIG['enable_twitter']:
                    send_pricechange_tweet(signal)
                    print('send via twitter')
        time.sleep(CYCLE_DELAY)

2022-03-09 11:40:38,939 INFO: Started continuous price logging
2022-03-09 11:40:38,940 INFO: Crypto bot is starting - please wait
2022-03-09 11:40:38,940 INFO: Collecting crypto data from gate.io for 10s
2022-03-09 11:40:48,941 INFO: 
<< Crypto signal bot started :-) >>
2022-03-09 11:40:48,942 INFO: << Checking prices every 20 seconds >>
2022-03-09 11:40:48,942 INFO: Now checking for signals - please wait

2022-03-09 11:52:06,800 INFO: EOSBULL_USDT up + 19.42446% MA:$1.1e-05 last_price:$1.4e-05 price delta:3e-06 volume:$1272326905.1 daily_change:33.65% high_24h:$1.16e-05 low_24h:$9.8e-06low_high_diff_p:18.37%
EOSBULL_USDT new 24h high $1.39e-05
send via twitter

# Twitter Consumer API keys
CONSUMER_KEY    = "api123"
CONSUMER_SECRET = "api123"

# Twitter Access token & access token secret
ACCESS_TOKEN    = "api123"
ACCESS_SECRET   = "api123"

BEARER_TOKEN = "api123"

class TwitterSecrets:
    """Class that holds Twitter Secrets"""

    def __init__(self):
        self.CONSUMER_KEY    = CONSUMER_KEY
        self.CONSUMER_SECRET = CONSUMER_SECRET
        self.ACCESS_TOKEN    = ACCESS_TOKEN
        self.ACCESS_SECRET   = ACCESS_SECRET
        self.BEARER_TOKEN   = BEARER_TOKEN
        
        # Tests if keys are present
        for key, secret in self.__dict__.items():
            assert secret != "", f"Please provide a valid secret for: {key}"

twitter_secrets = TwitterSecrets()

import pandas as pd
import tweepy 

# place the twitter_secrets file under <User>/anaconda3/Lib
from twitter_secrets import twitter_secrets as ts

consumer_key = ts.CONSUMER_KEY
consumer_secret = ts.CONSUMER_SECRET
access_token = ts.ACCESS_TOKEN
access_secret = ts.ACCESS_SECRET

### Print API Auth Data (leave disabled for security reasons)
# print(f'consumer_key: {consumer_key}')
# print(f'consumer_secret: {consumer_secret}')
# print(f'access_token: {access_token}')
# print(f'access_secret: {access_token}')

api_key: api123 
api_secret: api123
access_token: api123
access_secret: api123

## In case you prefer to load the auth data from a local yaml file, use the following code
import yaml
# load the API keys and further data from your local yaml file
# place the api_config_twitter.yml file under <python notebook>/API_Keys/
yaml_file = open('API Keys/api_config_twitter.yml', 'r')  
p = yaml.load(yaml_file, Loader=yaml.FullLoader)

try:
    consumer_key = p['api_key']
    consumer_secret = p['api_secret']
    access_token = p['access_token']
    access_secret = p['access_secret']
except ValueError: 
        print('error')

#authenticating to access the twitter API
auth=tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_secret)
api=tweepy.API(auth)

# Define the tweet text
tweet='this is an automated test tweet using #Python $BTC $ETH'

# Generate text tweet
api.update_status(tweet)

# Define the tweet text
tweet_text='This is another automated test tweet using #Python'
image_path ='Test Images/ETH_price.png'

# Generate text tweet with media (image)
status = api.update_with_media(image_path, tweet_text)

"""Replace the values below with your own Twitter API Tokens"""

# Twitter Bearer Token
BEARER_KEY = "your own BEARER KEY"


class TwitterSecrets:
    """Class that holds Twitter Secrets"""

    def __init__(self):
        self.BEARER_KEY = BEARER_KEY
        
        # Tests if keys are present
        for key, secret in self.__dict__.items():
            assert secret != "", f"Please provide a valid secret for: {key}"

twitter_secrets = TwitterSecrets()

# imports the twitter_secrets python file in which we store the twitter API keys
from twitter_secrets import twitter_secrets as ts

bearer_token = ts.BEARER_TOKEN

import requests 
import json 
import pandas as pd

# imports the twitter_secrets python file in which we store the twitter API keys
from twitter_secrets import twitter_secrets as ts

# a function that provides a bearer token to the API
def create_headers(bearer_token):
    headers = {"Authorization": "Bearer {}".format(bearer_token)}
    return headers

# this function defines the rules on what tweets to pull    
def set_rules(headers, delete, bearer_token, rules):
    payload = {"add": rules}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        headers=headers,
        json=payload,
    )
    if response.status_code != 201:
        raise Exception(
            "Cannot add rules (HTTP {}): {}".format(response.status_code, response.text)
        )
    print(json.dumps(response.json()))
    
# this function requests the current rules in place
def get_rules(headers, bearer_token):
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream/rules", headers=headers
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot get rules (HTTP {}): {}".format(response.status_code, response.text)
        )
    print(json.dumps(response.json()))
    return response.json()

# this function resets all rules
def delete_all_rules(headers, bearer_token, rules):
    if rules is None or "data" not in rules:
        return None

    ids = list(map(lambda rule: rule["id"], rules["data"]))
    payload = {"delete": {"ids": ids}}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        headers=headers,
        json=payload
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot delete rules (HTTP {}): {}".format(
                response.status_code, response.text
            )
        )
    print(json.dumps(response.json()))

# this function starts the stream
def get_stream(headers, set, bearer_token, expansions, fields, save_to_disk, save_path):
    data = []
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream" + expansions + fields, headers=headers, stream=True,
    )
    print(response.status_code)
    if response.status_code != 200:
        raise Exception(
            "Cannot get stream (HTTP {}): {}".format(
                response.status_code, response.text
            )
        )
    i = 0
    for response_line in response.iter_lines():
        i += 1
        if i == max_results:
            break
        else:
            json_response = json.loads(response_line)
            #print(json.dumps(json_response, indent=4, sort_keys=True))
            try:
                save_tweets(json_response)
                if save_to_disk == True:
                    save_media_to_disk(json_response, save_path)
            except (json.JSONDecodeError, KeyError) as err:
                # In case the JSON fails to decode, we skip this tweet
                print(f"{i}/{max_results}: ERROR: encountered a problem with a line of data... \n")
                continue

# this function saves a tweet to the SQLite DB                
def save_tweets(tweet):
    print(json.dumps(tweet, indent=4, sort_keys=True))
    data = tweet['data']
    public_metrics = data['public_metrics']
    tweet_list.append([data['id'], data['author_id'], data['created_at'], data['text'], public_metrics['like_count']])

# the max number of tweets that will be returned
max_results = 20

# save to disk
save_media_to_disk = False
save_path = ""

# You can adjust the rules if needed
search_rules = [
    {"value": "dog has:images", "tag": "dog pictures", "lang": "en"},
    {"value": "cat has:images -grumpy", "tag": "cat pictures", "lang": "en"},
]
tweet_fields = "?tweet.fields=attachments,author_id,created_at,public_metrics"
expansions = ""
tweet_list = []


bearer_token = ts.BEARER_TOKEN
headers = create_headers(bearer_token)
rules = get_rules(headers, bearer_token)
delete = delete_all_rules(headers, bearer_token, rules)
set = set_rules(headers, delete, bearer_token, search_rules)
get_stream(headers, set, bearer_token, expansions, tweet_fields, save_media_to_disk, save_path)

df = pd.DataFrame (tweet_list, columns = ['tweetid', 'author_id' , 'created_at', 'text', 'like_count'])
df

import requests 
import json 
import pandas as pd
import urllib
import os
from os import path
from datetime import datetime as dt

# imports the twitter_secrets python file in which we store the twitter API keys
from twitter_secrets import twitter_secrets as ts

def create_headers(bearer_token):
    headers = {"Authorization": "Bearer {}".format(bearer_token)}
    return headers
        
def set_rules(headers, delete, bearer_token, rules):
    payload = {"add": rules}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        headers=headers,
        json=payload,
    )
    if response.status_code != 201:
        raise Exception(
            "Cannot add rules (HTTP {}): {}".format(response.status_code, response.text)
        )
    print(json.dumps(response.json()))
    
def get_rules(headers, bearer_token):
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream/rules", headers=headers
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot get rules (HTTP {}): {}".format(response.status_code, response.text)
        )
    print(json.dumps(response.json()))
    return response.json()

def delete_all_rules(headers, bearer_token, rules):
    if rules is None or "data" not in rules:
        return None

    ids = list(map(lambda rule: rule["id"], rules["data"]))
    payload = {"delete": {"ids": ids}}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        headers=headers,
        json=payload
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot delete rules (HTTP {}): {}".format(
                response.status_code, response.text
            )
        )
    print(json.dumps(response.json()))

def get_stream(headers, set, bearer_token, expansions, fields, save_to_disk, save_path):
    data = []
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream" + expansions + fields, headers=headers, stream=True,
    )
    print(response.status_code)
    if response.status_code != 200:
        raise Exception(
            "Cannot get stream (HTTP {}): {}".format(
                response.status_code, response.text
            )
        )
    i = 0
    for response_line in response.iter_lines():
        i += 1
        if i == max_results:
            break
        else:
            json_response = json.loads(response_line)
            #print(json.dumps(json_response, indent=4, sort_keys=True))
            try:
                save_tweets(json_response)
                if save_to_disk == True:
                    save_media_to_disk(json_response, save_path)
            except (json.JSONDecodeError, KeyError) as err:
                # In case the JSON fails to decode, we skip this tweet
                print(f"{i}/{max_results}: ERROR: encountered a problem with a line of data... \n")
                continue
                
def save_tweets(tweet):
    #print(json.dumps(tweet, indent=4, sort_keys=True))
    data = tweet['data']
    includes = tweet['includes']
    media = includes['media']
    for line in media:
        tweet_list.append([data['id'], line['url']])  
        
def save_media_to_disk(tweet, save_path):
    data = tweet['data']
    #print(json.dumps(data, indent=4, sort_keys=True))
    includes = tweet['includes']
    media = includes['media']
    for line in media:
        media_url = line['url']
        media_key = line['media_key']
        pic = urllib.request.urlopen(media_url)
        file_path = save_path + "/" + media_key + ".jpg"
        try:
            with open(file_path, 'wb') as localFile:
                localFile.write(pic.read())
            tweet_list.append(media_key, media_url)
        except Exception as e:
            print('exception when saving media url ' + media_url + ' to path: ' + file_path)
            if path.exists(file_path):
                print("path exists")
    
def createDir(save_path):
    try:
        os.makedirs(save_path)
    except OSError:
        print ("Creation of the directory %s failed" % save_path)
        if path.exists(savepath):
            print("file already exists")
    else:
        print ("Successfully created the directory %s " % save_path)

# save to disk
save_to_disk = True
 
if save_to_disk == True: 
    # detect the current working directory and print it
    base_path = os.getcwd()
    print ("The current working directory is %s" % base_path)
    img_dir = '/twitter/downloaded_media/'
    # the write path in which the data will be stored. If it does not yet exist, it will be created
    now = dt.now()
    dt_string = now.strftime("%d%m%Y-%H%M%S")# ddmmYY-HMS
    save_path = base_path + img_dir + dt_string
    createDir(save_path)

# the max number of tweets that will be returned
max_results = 10

# You can adjust the rules if needed
search_rules = [
    {"value": "dog has:images", "tag": "dog pictures", "lang": "en"},
]

media_fields = "&media.fields=duration_ms,height,media_key,preview_image_url,public_metrics,type,url,width"
expansions = "?expansions=attachments.media_keys"
tweet_list = []

bearer_token = ts.BEARER_TOKEN
headers = create_headers(bearer_token)
rules = get_rules(headers, bearer_token)
delete = delete_all_rules(headers, bearer_token, rules)
set = set_rules(headers, delete, bearer_token, search_rules)
get_stream(headers, set, bearer_token, expansions, media_fields, save_to_disk, save_path)

df = pd.DataFrame (tweet_list, columns = ['tweetid', 'preview_image_url'])
df