forked from protonx-tf-03-projects/GRU
-
Notifications
You must be signed in to change notification settings - Fork 0
/
data.py
171 lines (136 loc) · 5.83 KB
/
data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
# Libraries imported.
import re
import os
import io
import tensorflow as tf
import pandas as pd
import nltk
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import json
from constant import *
nltk.download('punkt')
nltk.download('wordnet')
class Dataset:
def __init__(self, data_path, vocab_size, data_classes, vocab_folder):
self.data_path = data_path
self.vocab_size = vocab_size
self.data_classes = data_classes
self.sentences_tokenizer = None
self.label_dict = None
self.vocab_folder = vocab_folder
self.save_tokenizer_path = '{}tokenizer.json'.format(self.vocab_folder)
self.save_label_path = 'label.json'
if os.path.isfile(self.save_tokenizer_path):
with open(self.save_tokenizer_path) as file:
data = json.load(file)
self.sentences_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)
if os.path.isfile(self.save_label_path):
with open(self.save_label_path) as file:
self.label_dict = json.load(file)
def labels_encode(self, labels, data_classes):
'''Encode labels to categorical'''
labels.replace(data_classes, inplace=True)
labels_target = labels.values
labels_target = tf.keras.utils.to_categorical(labels_target)
return labels_target
def removeHTML(self, text):
'''Remove html tags from a string'''
clean = re.compile('<.*?>')
return re.sub(clean, '', text)
def removePunc(self, text):
#Remove punction in a texts
return re.sub(r'[^\w\s]','', text)
def removeURLs(self, text):
#Remove url link in texts
return re.sub(r'^https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE)
def removeEmoji(self, data):
#Each emoji icon has their unique code
#Gather all emoji icon code and remove it in texts
cleanr= re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
return re.sub(cleanr, '',data)
def sentence_cleaning(self, sentence):
'''Cleaning text'''
out_sentence = []
for line in tqdm(sentence):
line = self.removeHTML(line)
line = self.removePunc(line)
line = self.removeURLs(line)
line = self.removeEmoji(line)
text = re.sub("[^a-zA-Z]", " ", line)
word = word_tokenize(text.lower())
lemmatizer = WordNetLemmatizer()
lemm_word = [lemmatizer.lemmatize(i) for i in word]
out_sentence.append(lemm_word)
return (out_sentence)
def data_processing(self, sentences, labels):
'''Preprocessing both text and labels'''
print("|--data_processing ...")
sentences = self.sentence_cleaning(sentences)
labels = self.labels_encode(labels, data_classes=self.data_classes)
return sentences, labels
def build_tokenizer(self, sentences, vocab_size, char_level=False):
print("|--build_tokenizer ...")
tokenizer = tf.keras.preprocessing.text.Tokenizer(
num_words= vocab_size, oov_token=OOV, char_level=char_level)
tokenizer.fit_on_texts(sentences)
return tokenizer
def tokenize(self, tokenizer, sentences, max_length):
print("|--tokenize ...")
sentences = tokenizer.texts_to_sequences(sentences)
sentences = tf.keras.preprocessing.sequence.pad_sequences(sentences, maxlen=max_length,
padding=PADDING, truncating=TRUNC)
return sentences
def load_dataset(self, max_length, data_name, label_name):
print(" ")
print("Load dataset ... ")
datastore = pd.read_csv(self.data_path)
sentences = datastore[data_name]
labels = datastore[label_name]
self.label_dict = dict((item, idx)
for idx, item in enumerate(set(labels)))
# Cleaning
sentences, labels = self.data_processing(sentences, labels)
# Tokenizing
self.sentences_tokenizer = self.build_tokenizer(sentences, self.vocab_size)
tensor = self.tokenize(
self.sentences_tokenizer, sentences, max_length)
print(" ")
print("Save tokenizer ... ")
# Saving tokenizer
if not os.path.exists(self.vocab_folder):
try:
os.makedirs(self.vocab_folder)
except OSError as e:
raise IOError("Failed to create folders")
tokenizer_json = self.sentences_tokenizer.to_json()
with io.open(self.save_tokenizer_path, 'w', encoding='utf-8') as file:
file.write(json.dumps(tokenizer_json, ensure_ascii=False))
# Saving label dict
with open('label.json', 'w') as f:
json.dump(self.label_dict, f)
print("Done! Next to ... ")
print(" ")
return tensor, labels
def build_dataset(self, max_length=128, test_size=0.2, buffer_size=128, batch_size=128, data_name='review', label_name='sentiment'):
sentences, labels = self.load_dataset(
max_length, data_name, label_name)
X_train, X_val, y_train, y_val = train_test_split(
sentences, labels, test_size=test_size, stratify=labels, random_state=42)
# Convert to tensor
train_dataset = tf.data.Dataset.from_tensor_slices((tf.convert_to_tensor(
X_train, dtype=tf.int64), tf.convert_to_tensor(y_train, dtype=tf.int64)))
train_dataset = train_dataset.shuffle(buffer_size).batch(batch_size)
val_dataset = tf.data.Dataset.from_tensor_slices((tf.convert_to_tensor(
X_val, dtype=tf.int64), tf.convert_to_tensor(y_val, dtype=tf.int64)))
val_dataset = val_dataset.shuffle(buffer_size).batch(batch_size)
return train_dataset, val_dataset