Twitter-Sentiment-Analysis/readData.py at master · saflisgit/Twitter-Sentiment-Analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import pandas as pd
import re

from nltk.corpus import stopwords
from nltk.stem import PorterStemmer

def read_and_clean_data(filename):

    data = read_from_csv(filename)
    data = tolower(data)
    data = removeMentions(data)
    data = removeNonLetters(data)
    data = remove_stop_words(data)
    data = stem(data)
    data = removeNonLetters(data)

    return data

# ==================================================================================================

def read_from_csv(filename):

    # READING THE DATA FROM CSV FILE

    data = pd.read_csv(filename, encoding='latin-1', usecols=['tweet', 'Sentiment'])

    data = data[['tweet', 'Sentiment']]

    return data

# ==================================================================================================

def print_head_and_tail(data):
    print('===========Head and tail of data============')
    print(data.head())
    print('---------------------------------------------')
    print(data.tail())


# ==================================================================================================


def remove_stop_words(data):
    # STOP WORD REMOVAL

    stop = stopwords.words('english')

    data['tweet'] = data['tweet'].apply(
        lambda x: ' '.join([word for word in x.split() if word not in stop]))

    return data


# ==================================================================================================

def stem(data):
    # STEMMING

    ps = PorterStemmer()

    data['tweet'] = data['tweet'].apply(
        lambda x: ' '.join([ps.stem(word) for word in x.split()]))

    return data

def removeMentions(data):
    data['tweet'] = [re.sub(r'@[A-Za-z0-9_;&,.#]+', '', tweet) for tweet in data['tweet']]
    data['tweet'] = [re.sub(r'@+', '', tweet) for tweet in data['tweet']]
    data['tweet'] = [re.sub('https?://[A-Za-z0-9./]+', '', tweet) for tweet in data['tweet']]
    data['tweet'] = [re.sub('www?[A-Za-z0-9./]+', '', tweet) for tweet in data['tweet']]

    return data

def removeNonLetters(data):
    data['tweet'] = [re.sub("[^a-zA-Z]", " ", tweet) for tweet in data['tweet']]
    data['tweet'] = [re.sub(' +', ' ', tweet) for tweet in data['tweet']]

    return data

def removespaces(data):
    data['tweet'] = [re.sub(" ", "", tweet) for tweet in data['tweet']]

    return data

def tolower(data):
    data['tweet'] = [tweet.lower() for tweet in data['tweet']]

    return data

def read_and_clean_sentence (sentence) :
    sentence = re.sub(r'@[A-Za-z0-9_;&,.#]+', '', sentence)
    sentence = re.sub(r'@+', '', sentence)
    sentence = re.sub('https?://[A-Za-z0-9./]+', '', sentence)
    sentence = re.sub('www?[A-Za-z0-9./]+', '', sentence)
    sentence = re.sub("[^a-zA-Z]", " ", sentence)
    sentence = re.sub(' +', ' ', sentence)
    ps = PorterStemmer()
    stop = stopwords.words('english')
    sentence =' '.join([word for word in sentence.split() if word not in stop])
    sentence =' '.join([ps.stem(word) for word in sentence.split() ])

    return sentence