-
Notifications
You must be signed in to change notification settings - Fork 0
/
VacancyHelper2.py
190 lines (172 loc) · 8.17 KB
/
VacancyHelper2.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
"""
@author: Aayush Agrawal
@Purpose - Re-usable code in Python 3 for Recommender systems
ML-small-dataset - https://grouplens.org/datasets/movielens/
"""
import pandas as pd
import numpy as np
from scipy import sparse
from lightfm import LightFM
from sklearn.metrics.pairwise import cosine_similarity
def create_interaction_matrix(df,user_col, item_col, rating_col, norm= False, threshold = None):
'''
Function to create an interaction matrix dataframe from transactional type interactions
Required Input -
- df = Pandas DataFrame containing user-item interactions
- user_col = column name containing user's identifier
- item_col = column name containing item's identifier
- rating col = column name containing user feedback on interaction with a given item
- norm (optional) = True if a normalization of ratings is needed
- threshold (required if norm = True) = value above which the rating is favorable
Expected output -
- Pandas dataframe with user-item interactions ready to be fed in a recommendation algorithm
'''
interactions = df.groupby([user_col, item_col])[rating_col] \
.sum().unstack().reset_index(). \
fillna(0).set_index(user_col)
if norm:
interactions = interactions.applymap(lambda x: 1 if x > threshold else 0)
return interactions
def create_user_dict(interactions):
'''
Function to create a user dictionary based on their index and number in interaction dataset
Required Input -
interactions - dataset create by create_interaction_matrix
Expected Output -
user_dict - Dictionary type output containing interaction_index as key and user_id as value
'''
user_id = list(interactions.index)
user_dict = {}
counter = 0
for i in user_id:
user_dict[i] = counter
counter += 1
return user_dict
def create_item_dict(df,id_col,name_col):
'''
Function to create an item dictionary based on their item_id and item name
Required Input -
- df = Pandas dataframe with Item information
- id_col = Column name containing unique identifier for an item
- name_col = Column name containing name of the item
Expected Output -
item_dict = Dictionary type output containing item_id as key and item_name as value
'''
item_dict ={}
for i in range(df.shape[0]):
item_dict[(df.loc[i,id_col])] = df.loc[i,name_col]
return item_dict
def runMF(interactions, n_components=30, loss='warp', k=15, epoch=30,n_jobs = 4):
'''
Function to run matrix-factorization algorithm
Required Input -
- interactions = dataset create by create_interaction_matrix
- n_components = number of embeddings you want to create to define Item and user
- loss = loss function other options are logistic, brp
- epoch = number of epochs to run
- n_jobs = number of cores used for execution
Expected Output -
Model - Trained model
'''
x = sparse.csr_matrix(interactions.values.astype(np.int32))
model = LightFM(no_components= n_components,learning_rate=0.027, loss=loss,k=k)
model.fit(x,epochs=epoch,num_threads = n_jobs)
return model
def sample_recommendation_user(model, interactions, user_id, user_dict,
item_dict,threshold = 0,nrec_items = 10, show = True):
'''
Function to produce user recommendations
Required Input -
- model = Trained matrix factorization model
- interactions = dataset used for training the model
- user_id = user ID for which we need to generate recommendation
- user_dict = Dictionary type input containing interaction_index as key and user_id as value
- item_dict = Dictionary type input containing item_id as key and item_name as value
- threshold = value above which the rating is favorable in new interaction matrix
- nrec_items = Number of output recommendation needed
Expected Output -
- Prints list of items the given user has already bought
- Prints list of N recommended items which user hopefully will be interested in
'''
n_users, n_items = interactions.shape
#print(interactions)
user_x = user_dict[user_id]
#print (user_x)
# The predict will return all the scores from the matrix for the row (this is the user) for each column (item)
scores = pd.Series(model.predict(user_x,np.arange(n_items)))
scores.index = interactions.columns
#print (scores)
scores = list(pd.Series(scores.sort_values(ascending=False).index))
known_items = list(pd.Series(interactions.loc[user_id,:][interactions.loc[user_id,:] > threshold].index).sort_values(ascending=False))
scores = [x for x in scores if x not in known_items]
return_score_list = scores[0:nrec_items]
known_items = list(pd.Series(known_items).apply(lambda x: item_dict[x]))
scores = list(pd.Series(return_score_list).apply(lambda x: item_dict[x]))
if show == True:
print("Known Likes:")
counter = 1
for i in known_items:
print(str(counter) + '- ' + i)
counter+=1
print("\n Recommended Items:")
counter = 1
for i in scores:
print(str(counter) + '- ' + i)
counter+=1
return return_score_list
def sample_recommendation_item(model,interactions,item_id,user_dict,item_dict,number_of_user):
'''
Funnction to produce a list of top N interested users for a given item
Required Input -
- model = Trained matrix factorization model
- interactions = dataset used for training the model
- item_id = item ID for which we need to generate recommended users
- user_dict = Dictionary type input containing interaction_index as key and user_id as value
- item_dict = Dictionary type input containing item_id as key and item_name as value
- number_of_user = Number of users needed as an output
Expected Output -
- user_list = List of recommended users
'''
n_users, n_items = interactions.shape
x = np.array(interactions.columns)
scores = pd.Series(model.predict(np.arange(n_users), np.repeat(x.searchsorted(item_id),n_users)))
user_list = list(interactions.index[scores.sort_values(ascending=False).head(number_of_user).index])
return user_list
def create_item_emdedding_distance_matrix(model,interactions):
'''
Function to create item-item distance embedding matrix
Required Input -
- model = Trained matrix factorization model
- interactions = dataset used for training the model
Expected Output -
- item_emdedding_distance_matrix = Pandas dataframe containing cosine distance matrix b/w items
'''
df_item_norm_sparse = sparse.csr_matrix(model.item_embeddings)
similarities = cosine_similarity(df_item_norm_sparse)
item_emdedding_distance_matrix = pd.DataFrame(similarities)
item_emdedding_distance_matrix.columns = interactions.columns
item_emdedding_distance_matrix.index = interactions.columns
return item_emdedding_distance_matrix
def item_item_recommendation(item_emdedding_distance_matrix, item_id,
item_dict, n_items = 10, show = True):
'''
Function to create item-item recommendation
Required Input -
- item_emdedding_distance_matrix = Pandas dataframe containing cosine distance matrix b/w items
- item_id = item ID for which we need to generate recommended items
- item_dict = Dictionary type input containing item_id as key and item_name as value
- n_items = Number of items needed as an output
Expected Output -
- recommended_items = List of recommended items
'''
recommended_items = list(pd.Series(item_emdedding_distance_matrix.loc[item_id,:]. \
sort_values(ascending = False).head(n_items+1). \
index[1:n_items+1]))
if show == True:
print("Item of interest :{0}".format(item_dict[item_id]))
print("Item similar to the above item:")
counter = 1
for i in recommended_items:
print(str(counter) + '- ' + item_dict[i])
counter+=1
return recommended_items