add twitter feature

twitter_scraper.py

@@ -0,0 +1,47 @@
+import snscrape.modules.twitter as sntwitter
+import datetime as dt
+import pandas as pd
+import sys
+import os
+import re
+import tqdm
+
+
+def scrape_tweets(query, max_tweets=10, output_path="./scraper/output/" ):
+    tweets_list = []
+
+    for i,tweet in tqdm(enumerate(sntwitter.TwitterSearchScraper(query).get_items())):
+        if max_tweets != -1 and i >= int(max_tweets):
+            break
+        tweets_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username, tweet.likeCount, tweet.retweetCount, tweet.replyCount, tweet.quoteCount, tweet.url, tweet.lang])
+
+    df = pd.DataFrame(tweets_list, columns=['Datetime', 'Tweet Id', 'Text', 'Username', 'Likes', 'Retweets', 'Replies', 'Quotes', 'URL', 'Language'])
+    df = df[df["Language"] == "in"]
+    return df
+
+def remove_unnecessary_char(text):
+    text = re.sub("\[USERNAME\]", " ", text)
+    text = re.sub("\[URL\]", " ", text)
+    text = re.sub("\[SENSITIVE-NO\]", " ", text)
+    text = re.sub('  +', ' ', text)
+    return text
+
+def preprocess_tweet(text):
+    text = re.sub('\n',' ',text)
+    text = re.sub('^(\@\w+ ?)+',' ',text)
+    text = re.sub(r'\@\w+',' ',text)
+    text = re.sub('((www\.[^\s]+)|(https?://[^\s]+)|(http?://[^\s]+))',' ',text)
+    text = re.sub('/', ' ', text)
+    text = re.sub('  +', ' ', text)
+    return text
+
+def remove_nonaplhanumeric(text):
+    text = re.sub('[^0-9a-zA-Z]+', ' ', text)
+    return text
+
+def preprocess_text(text):
+    text = preprocess_tweet(text)
+    text = remove_unnecessary_char(text)
+    text = remove_nonaplhanumeric(text)
+    text = text.lower()
+    return text