-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathexp_clustering_dbscan.py
197 lines (158 loc) · 6.65 KB
/
exp_clustering_dbscan.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
import os
import gc
import pickle
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
import multiprocessing
from sklearn.cluster import DBSCAN
from sklearn.metrics.pairwise import cosine_distances
from sklearn.metrics import homogeneity_completeness_v_measure
from exp_corpus_loader import get_corpus_bow
from exp_corpus_loader import get_corpus_tfidf
from exp_corpus_loader import get_corpus_doc2vec
from exp_corpus_loader import get_corpus_nel
from text.bag_of_words import bow_from_news
from text.tfidf import tfidf_from_news
from text.doc2vec import doc2vec_from_news
from text.nel import nel_from_news
n_jobs = multiprocessing.cpu_count()
results_file = 'exp_clustering_dbscan_results.csv'
def jaccard_distances(nel_list):
def jaccard(list1, list2):
intersection = len(list(set(list1).intersection(list2)))
union = (len(list1) + len(list2)) - intersection
return float(intersection / union)
def one_to_many(idx1, list1):
res = []
for idx2 in range(0, idx1):
res.append(1 - jaccard(list1, nel_list[idx2]))
res = np.concatenate([np.array(res), np.zeros(len(nel_list) - idx1)])
return res
dist_mx = Parallel(n_jobs=n_jobs)(
delayed(one_to_many)(idx1, list1) for idx1, list1 in enumerate(nel_list))
dist_mx = np.array(dist_mx)
return dist_mx + np.transpose(dist_mx)
def chunk_cosine_distances(vectors):
chunk = 500
mx_len = vectors.shape[0]
dist_mx = np.zeros(shape=(mx_len, mx_len))
for idx1 in range(0, mx_len, chunk):
gc.collect()
end1 = idx1 + chunk
if end1 > mx_len:
end1 = mx_len
res_row = None
for idx2 in range(0, mx_len, chunk):
end2 = idx2 + chunk
if end2 > mx_len:
end2 = mx_len
if isinstance(res_row, np.ndarray):
dist = cosine_distances(
vectors[idx1:end1], vectors[idx2:end2])
res_row = np.hstack((res_row, dist))
else:
res_row = cosine_distances(
vectors[idx1:end1], vectors[idx2:end2])
dist_mx[idx1:end1] = res_row
return dist_mx
def load(filename):
with open(filename, "rb") as fp:
return pickle.load(fp)
def save(obj, filename):
with open(filename, "wb") as fp:
pickle.dump(obj, fp, protocol=4)
test_eps = np.arange(0.05, 0.91, 0.05)
test_min_samples = [5, 10, 15, 20, 25]
techniques = [{'name': 'Doc2Vec',
'vectors': doc2vec_from_news,
'corpus': get_corpus_doc2vec,
'dist': cosine_distances,
'filename': 'data/vectors_doc2vec.bin',
'dist_file': 'data/dist_doc2vec.bin'},
{'name': 'NEL',
'vectors': nel_from_news,
'corpus': get_corpus_nel,
'dist': jaccard_distances,
'filename': 'data/vectors_nel.bin',
'dist_file': 'data/dist_nel.bin'},
{'name': 'TF-IDF',
'vectors': tfidf_from_news,
'corpus': get_corpus_tfidf,
'dist': chunk_cosine_distances,
'filename': None,
'dist_file': 'data/dist_tfidf.bin'},
{'name': 'BOW',
'vectors': bow_from_news,
'corpus': get_corpus_bow,
'dist': chunk_cosine_distances,
'filename': None,
'dist_file': 'data/dist_bow.bin'}]
results = pd.DataFrame(data={'technique': [],
'clus': [],
'clus_params': [],
'num_clusters': [],
'homogeneity': [],
'completeness': [],
'v_measure': [],
'outliers': []})
for technique in techniques:
gc.collect()
corpus, labels = technique['corpus']()
dist_fun = technique['dist']
file = technique['filename']
dist_file = technique['dist_file']
vector_fun = technique['vectors']
print("Computing clustering for technique", technique['name'], '...')
if file and os.path.isfile(file):
doc_vectors = load(file)
else:
doc_vectors = vector_fun(corpus,
filename=file)
print("Computing/loading distances for technique", technique['name'], '...')
if dist_file and os.path.isfile(dist_file):
vectors_dist = load(dist_file)
else:
vectors_dist = dist_fun(doc_vectors)
save(vectors_dist, dist_file)
print("Computing/loading distances for technique", technique['name'], '... DONE!')
articles_filter = np.where(labels != "Unclassified")
keep_idxs = np.r_[0:len(labels)][articles_filter]
# new_vectors = []
# for idx, vec in enumerate(doc_vectors):
# if idx in keep_idxs:
# new_vectors.append(vec)
# doc_vectors = new_vectors
new_dist = []
for idx, row in enumerate(vectors_dist):
if idx in keep_idxs:
new_dist.append(row[articles_filter])
vectors_dist = np.array(new_dist)
labels = labels[articles_filter]
for eps in test_eps:
for min_samples in test_min_samples:
dbscan = DBSCAN(eps=eps,
min_samples=min_samples,
metric='precomputed',
n_jobs=2)
pred_labels = dbscan.fit_predict(vectors_dist)
outliers = np.count_nonzero(np.where(pred_labels == -1))
num_clusters = len(set(pred_labels))
(homog,
compl,
v_measure) = homogeneity_completeness_v_measure(labels,
pred_labels)
results = results.append({'technique': technique['name'],
'clus': 'DBSCAN',
'clus_params': 'eps=' + str(eps) + ';min_samples=' + str(min_samples),
'num_clusters': num_clusters,
'homogeneity': homog,
'completeness': compl,
'v_measure': v_measure,
'outliers': outliers},
ignore_index=True)
print(technique['name'], 'DBSCAN', 'eps=' + str(eps) + ';min_samples=' + str(min_samples), num_clusters, homog, compl, v_measure, 'outliers=', outliers)
results.to_csv(results_file, index=False)
vectors_dist = None
doc_vectors = None
corpus = None