at_prep.py

from data_prep import *
from sdt import *
import numpy as np
import math
import concurrent.futures
import itertools
import scipy
from scipy.spatial import distance
from scipy.stats.mstats_basic import rankdata
import scipy.sparse as sp
from scipy.optimize import linprog
from scipy import stats
from bert4keras.models import build_transformer_model
from bert4keras.tokenizers import SpTokenizer, Tokenizer
from tqdm import tqdm
import pickle
from multiprocessing import Process


allSegment, allAffect, allOtherFeelings, allPredLabel, allTrueLabel, allTestNum, allSubjectNum = load_data(
)
firstAffect, firstOtherFeelings, firstSafety, firstComfort, firstPredLabel, firstTrueLabel = load_data(
    mode=1)
secondAffect, secondOtherFeelings, secondSafety, secondComfort, secondPredLabel, secondTrueLabel = load_data(
    mode=2)
thirdAffect, thirdOtherFeelings, thirdSafety, thirdComfort, thirdPredLabel, thirdTrueLabel = load_data(
    mode=3)

data2chn = {
    (0, 0): '一点也没有快乐',
    (0, 1): '较轻微快乐',
    (0, 2): '较强烈快乐',
    (0, 3): '非常强烈快乐',
    (1, 0): '一点也没有兴趣',
    (1, 1): '较轻微兴趣',
    (1, 2): '较强烈兴趣',
    (1, 3): '非常强烈兴趣',
    (2, 0): '一点也没有惊奇',
    (2, 1): '较轻微惊奇',
    (2, 2): '较强烈惊奇',
    (2, 3): '非常强烈惊奇',
    (3, 0): '一点也没有恐惧',
    (3, 1): '较轻微恐惧',
    (3, 2): '较强烈恐惧',
    (3, 3): '非常强烈恐惧',
    (4, 0): '一点也没有紧张',
    (4, 1): '较轻微紧张',
    (4, 2): '较强烈紧张',
    (4, 3): '非常强烈紧张',
    (5, 0): '一点也没有满意',
    (5, 1): '较轻微满意',
    (5, 2): '较强烈满意',
    (5, 3): '非常强烈满意'
}

chn2embeddings = {
    '一点也没有快乐': 0,
    '较轻微快乐': 0,
    '较强烈快乐': 0,
    '非常强烈快乐': 0,
    '一点也没有兴趣': 0,
    '较轻微兴趣': 0,
    '较强烈兴趣': 0,
    '非常强烈兴趣': 0,
    '一点也没有惊奇': 0,
    '较轻微惊奇': 0,
    '较强烈惊奇': 0,
    '非常强烈惊奇': 0,
    '一点也没有恐惧': 0,
    '较轻微恐惧': 0,
    '较强烈恐惧': 0,
    '非常强烈恐惧': 0,
    '一点也没有紧张': 0,
    '较轻微紧张': 0,
    '较强烈紧张': 0,
    '非常强烈紧张': 0,
    '一点也没有满意': 0,
    '较轻微满意': 0,
    '较强烈满意': 0,
    '非常强烈满意': 0
}


def convert(data):

    return [data2chn[(int(i), int(j))] for i, j in enumerate(data)]


def n_component(length, wd=False):

    s1 = [2**(i + 1) for i in range(10) if 2**(i + 1) <= length / 2]
    s2 = [
        s1[i] + s1[i + 1] for i in range(len(s1) - 1)
        if s1[i] + s1[i + 1] <= length / 2
    ]

    if wd:
        return sorted(s1 + s2)
    else:
        return sorted([1] + s1 + s2)


def compute_kernel_bias(vecs):

    vecs = np.array(vecs)
    mu = vecs.mean(axis=0, keepdims=True)
    cov = np.cov(vecs.T)

    try:
        u, s, vh = np.linalg.svd(cov)
    except:
        u, s, vh = scipy.linalg.svd(cov, lapack_driver='gesvd')

    W = np.dot(u, np.diag(1 / np.sqrt(s)))

    return W, -mu, cov


def transform(vecs, kernel=None, bias=None):

    if not (kernel is None or bias is None):
        return (vecs + bias).dot(kernel)


def abs_distance(x, y, n):

    return abs(x - y) / n


def mean_distance(x, y, n):

    return 1 - np.mean((x, y)) / n


def min_distance(x, y, n):

    return 1 - min(x, y) / n


def am_distance(x, y, n):

    return abs(x - y) * (1 - np.mean((x, y)) / n)


def max_distance(x, y, n):

    return max(x, y) / n


def pearson_distance(x, y, n=None):

    return distance.correlation(x, y)


def euclidean_distance(x, y, n=None):

    return np.linalg.norm(x - y)


def mahalanobis_distance(x, y, cov, n=None):

    return np.sqrt(np.dot(np.dot(x - y, cov), x - y))


def cosine_distance(x, y, n=None):

    return 1 - np.sum(x * y) / (np.linalg.norm(x) * np.linalg.norm(y))


def manhattan_distance(x, y, n=None):

    return np.sum(np.abs(x - y))


def wasserstein_distance(p, q, D):

    A_eq = []

    for i in range(len(p)):

        A = np.zeros_like(D)
        A[i, :] = 1
        A_eq.append(A.reshape(-1))

    for i in range(len(q)):

        A = np.zeros_like(D)
        A[:, i] = 1
        A_eq.append(A.reshape(-1))

    A_eq = np.array(A_eq)
    b_eq = np.concatenate([p, q])
    D = D.reshape(-1)
    result = linprog(D, A_eq=A_eq[:-1], b_eq=b_eq[:-1])

    return result.fun


def word_mover_distance(x, y):

    p = np.ones(x.shape[0]) / x.shape[0]
    q = np.ones(y.shape[0]) / y.shape[0]
    D = np.sqrt(np.square(x[:, None] - y[None, :]).mean(axis=2))

    return wasserstein_distance(p, q, D)


def word_rotator_distance(x, y):

    x_norm = (x**2).sum(axis=1, keepdims=True)**0.5
    y_norm = (y**2).sum(axis=1, keepdims=True)**0.5
    p = x_norm[:, 0] / x_norm.sum()
    q = y_norm[:, 0] / y_norm.sum()
    D = 1 - np.dot(x / x_norm, (y / y_norm).T)

    return wasserstein_distance(p, q, D)


def form_matrix(e1, e2):

    max_e1 = np.max(e1, axis=0)
    max_e2 = np.max(e2, axis=0)
    mean_e1 = np.mean(e1, axis=0)
    mean_e2 = np.mean(e2, axis=0)
    min_e1 = np.min(e1, axis=0)
    min_e2 = np.min(e2, axis=0)

    return (max_e1, max_e2), (mean_e1, mean_e2), (min_e1, min_e2), \
(np.concatenate((max_e1, mean_e1)), np.concatenate((max_e2, mean_e2))), \
(np.concatenate((max_e1, min_e1)), np.concatenate((max_e2, min_e2))), \
(np.concatenate((mean_e1, min_e1)), np.concatenate((mean_e2, min_e2))), \
(np.concatenate((max_e1, mean_e1, min_e1)), np.concatenate((max_e2, mean_e2, min_e2)))


def explore_vector_av(e1, e2, wd=False, n=None, cov=None):

    if wd:

        score = []

        try:
            score.append(word_mover_distance(e1, e2))
        except:
            
            e1 = np.nan_to_num(e1, np.sum(e1[~np.isnan(e1)].reshape(-1)))
            e2 = np.nan_to_num(e2, np.sum(e2[~np.isnan(e2)].reshape(-1)))
            
            try:
                score.append(word_mover_distance(e1, e2))
            except:
                score.append(word_mover_distance(e1+1e-20, e2+1e-20))

        try:
            score.append(word_rotator_distance(e1, e2))
        except:
            
            e1 = np.nan_to_num(e1, np.sum(e1[~np.isnan(e1)].reshape(-1)))
            e2 = np.nan_to_num(e2, np.sum(e2[~np.isnan(e2)].reshape(-1)))
            
            try:
                score.append(word_rotator_distance(e1, e2))
            except:
                score.append(word_rotator_distance(e1+1e-20, e2+1e-20))

        return score

    else:
        score = []
        if n != None:

            score.append(abs_distance(e1[0], e2[0], n))
            score.append(mean_distance(e1[0], e2[0], n))
            score.append(min_distance(e1[0], e2[0], n))
            score.append(am_distance(e1[0], e2[0], n))
            score.append(max_distance(e1[0], e2[0], n))

        else:
            score.append(pearson_distance(e1, e2))
            score.append(euclidean_distance(e1, e2))
            score.append(mahalanobis_distance(e1, e2, cov))
            score.append(cosine_distance(e1, e2))
            score.append(manhattan_distance(e1, e2))

        return score


def explore_matrix_av(c, wd=False):

    score = []

    m1 = [i[0] for i in c]
    m2 = [i[-1] for i in c]
    n = len(m1)

    if wd:

        kernel, bias, _ = compute_kernel_bias([v for i in m1 + m2 for v in i])
        n_components = n_component(len(m1[0][0]), wd=True)

        for i in tqdm(range(len(m1))):

            s = []
            v1, v2 = m1[i], m2[i]
            s.extend(explore_vector_av(v1, v2, wd=True))
            s.extend(
                explore_vector_av(transform(v1, kernel=kernel, bias=bias),
                                  transform(v2, kernel=kernel, bias=bias),
                                  wd=True))

            for nc in n_components:

                s.extend(
                    explore_vector_av(transform(v1,
                                                kernel=kernel[:, :nc],
                                                bias=bias),
                                      transform(v2,
                                                kernel=kernel[:, :nc],
                                                bias=bias),
                                      wd=True))

            score.append(s)

        return score

    else:
        kernel, bias, cov = compute_kernel_bias(m1 + m2)
        n_components = n_component(len(m1[0]))
        COV, M1, M2, N = [], [], [], []
        COV.append(cov)
        M1.append(m1)
        M2.append(m2)
        N.append(None)

        temp_m1 = [
            np.ravel(transform(v1, kernel=kernel, bias=bias)) for v1 in m1
        ]
        temp_m2 = [
            np.ravel(transform(v2, kernel=kernel, bias=bias)) for v2 in m2
        ]

        COV.append(np.cov(np.array(temp_m1 + temp_m2).T))
        M1.append(temp_m1)
        M2.append(temp_m2)
        N.append(None)

        for nc in n_components:

            temp_m1 = [
                np.ravel(transform(v1, kernel=kernel[:, :nc], bias=bias))
                for v1 in m1
            ]
            temp_m2 = [
                np.ravel(transform(v2, kernel=kernel[:, :nc], bias=bias))
                for v2 in m2
            ]

            if nc == 1:
                COV.append(None)
                N.append(n)
            else:
                COV.append(np.cov(np.array(temp_m1 + temp_m2).T))
                N.append(None)

            M1.append(temp_m1)
            M2.append(temp_m2)

        for i in tqdm(range(len(m1))):

            s = []

            for j in range(len(COV)):

                v1, v2, cov, nn = M1[j][i], M2[j][i], COV[j], N[j]
                s.extend(explore_vector_av(v1, v2, cov=cov, n=nn))

            score.append(s)

        return score


def explore_plm_av(data, embeddings, of=False):

    MD, MS = [], []
    C1D, C2D, C3D, C4D, C5D, C6D, C7D = [], [], [], [], [], [], []
    C1S, C2S, C3S, C4S, C5S, C6S, C7S = [], [], [], [], [], [], []

    for i in tqdm(range(len(data[0]))):
        s = []
        d = data[0][i]

        if of != 'only':

            item1 = convert(d[0])
            item2 = convert(d[1])

            if of and data[-1][i] != '(空)':
                item1.append(data[-1][i])
        else:
            item1 = [data[-1][i]]
            item2 = ['(空)']

        e1_d, e1_s = embeddings(item1)
        e2_d, e2_s = embeddings(item2)

        MD.append((e1_d, e2_d))
        MS.append((e1_s, e2_s))

        c1d, c2d, c3d, c4d, c5d, c6d, c7d = form_matrix(e1_d, e2_d)
        c1s, c2s, c3s, c4s, c5s, c6s, c7s = form_matrix(e1_s, e2_s)

        C1D.append(c1d)
        C2D.append(c2d)
        C3D.append(c3d)
        C4D.append(c4d)
        C5D.append(c5d)
        C6D.append(c6d)
        C7D.append(c7d)
        C1S.append(c1s)
        C2S.append(c2s)
        C3S.append(c3s)
        C4S.append(c4s)
        C5S.append(c5s)
        C6S.append(c6s)
        C7S.append(c7s)

    scoreMD = explore_matrix_av(MD, wd=True)
    scoreMS = explore_matrix_av(MS, wd=True)

    scoreD1 = explore_matrix_av(C1D)
    scoreD2 = explore_matrix_av(C2D)
    scoreD3 = explore_matrix_av(C3D)
    scoreD4 = explore_matrix_av(C4D)
    scoreD5 = explore_matrix_av(C5D)
    scoreD6 = explore_matrix_av(C6D)
    scoreD7 = explore_matrix_av(C7D)

    scoreS1 = explore_matrix_av(C1S)
    scoreS2 = explore_matrix_av(C2S)
    scoreS3 = explore_matrix_av(C3S)
    scoreS4 = explore_matrix_av(C4S)
    scoreS5 = explore_matrix_av(C5S)
    scoreS6 = explore_matrix_av(C6S)
    scoreS7 = explore_matrix_av(C7S)

    return np.concatenate((scoreMD, scoreMS, scoreD1, scoreD2, scoreD3, scoreD4, scoreD5, scoreD6, scoreD7, \
                          scoreS1, scoreS2, scoreS3, scoreS4, scoreS5, scoreS6, scoreS7), axis=1)


def original_emb():

    d2c_1 = {0: [0, 0, 0, 1], 1: [0, 0, 1, 0], 2: [0, 1, 0, 0], 3: [1, 0, 0, 0]}
    d2c_2 = {0: [0, 1], 1: [1, 0]}

    def data2code(data):

        return np.array([d2c_1[i] if len(data) > 1 else d2c_2[i] for i in data])

    def explore_original_av(data):

        score = []
        mv_data = [[[l / 3 if idx < 6 and len(j) > 1 else l for idx, l in enumerate(j)] for j in i] for i in data]
        X_m = np.mean(sum(mv_data, []), axis=0)
        rank_data = rankdata([[sum(i[0]), sum(i[1])] for i in data])
        cov = np.cov(sum(mv_data, []), rowvar=False)

        for i in tqdm(range(len(data))):

            s = []
            rank_x = rank_data[i]
            x = data[i]
            mv_x = mv_data[i]
            s.append(abs_distance(rank_x[0], rank_x[1], len(data) * 2))
            s.append(mean_distance(rank_x[0], rank_x[1], len(data) * 2))
            s.append(min_distance(rank_x[0], rank_x[1], len(data) * 2))
            s.append(am_distance(rank_x[0], rank_x[1], len(data) * 2))
            s.append(max_distance(rank_x[0], rank_x[1], len(data) * 2))
            s.append(pearson_distance(np.array(mv_x[0]), np.array(mv_x[1])))
            s.append(euclidean_distance(np.array(x[0]), np.array(mv_x[1])))
            s.append(mahalanobis_distance(np.array(mv_x[0]), np.array(mv_x[1]), cov))
            s.append(cosine_distance(np.array([xx + 1e-20 for xx in mv_x[0]]), np.array([xx + 1e-20 for xx in mv_x[1]])))
            s.append(
                cosine_distance(
                    np.array([xx - X_m[idx] for idx, xx in enumerate(mv_x[0])]),
                    np.array([xx - X_m[idx] for idx, xx in enumerate(mv_x[1])])))
            s.append(manhattan_distance(np.array(mv_x[0]), np.array(mv_x[1])))
            s.append(word_mover_distance(data2code(x[0]), data2code(x[1])))
            s.append(word_rotator_distance(data2code(x[0]), data2code(x[1])))
            score.append(s)

        return score

    pa = [[[i[0], i[1], i[2], i[-1]] for i in j] for j in allAffect]
    na = [[[i[3], i[4]] for i in j] for j in allAffect]     
    mf = [[[0], [0]] if i == '(空)' else [[0], [1]] for i in firstOtherFeelings+secondOtherFeelings+thirdOtherFeelings]

    print('Processing All Affect')
    allScore = explore_original_av(allAffect)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/aa.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('Processing All Affect and Other Feelings')
    allScore = explore_original_av([[j[0]+mf[idx][0], j[1]+mf[idx][1]] for idx, j in enumerate(allAffect)])
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/aa+of.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('Processing Positive Affect')
    allScore = explore_original_av(pa)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/pa.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('Processing Positive Affect and Other Feelings')
    allScore = explore_original_av([[j[0]+mf[idx][0], j[1]+mf[idx][1]] for idx, j in enumerate(pa)])
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/pa+of.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('Processing Negative Affect')
    allScore = explore_original_av(na)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/na.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('Processing Negative Affect and Other Feelings')
    allScore = explore_original_av([[j[0]+mf[idx][0], j[1]+mf[idx][1]] for idx, j in enumerate(na)])
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/na+of.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)

    print('Processing Other Feelings')
    allScore = explore_original_av(mf)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = 'data/av_data/original/of.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    

def word2vector(model, mode, name):

    if mode == 'char':
        vocab = set(''.join(data2chn.values())+' '.join(allOtherFeelings+['。', '；']))

    elif mode == 'mix':

        import jieba
        jieba.initialize()

        def pre_tokenize(text):
        
            return [
                w.replace(' ', u'\u2582').replace('\n', u'\u2583').replace('，', ',')
                for w in jieba.cut(text, cut_all=False)
            ]

        vocab = set(sum([pre_tokenize(i) for i in data2chn.values()], [])+sum([pre_tokenize(i) for i in allOtherFeelings], [])+['。', '；'])

    word2index = {w: i for i, w in enumerate(vocab)}
    index2word = {i: w for w, i in word2index.items()}

    SEED = 0

    if model == 'word':

        n_symbols = 1292607
        filePath = './PLMs/sgns/word.txt'
        DIM = 300

    elif model == 'word+ngram':

        n_symbols = 1285531
        filePath = './PLMs/sgns/word+ngram.txt'
        DIM = 300

    elif model == 'word+char':

        n_symbols = 1292679
        filePath = './PLMs/sgns/word+char.txt'
        DIM = 300

    elif model == 'word+char+ngram': 

        n_symbols = 1348468
        filePath = './PLMs/sgns/word+char+ngram.txt'
        DIM = 300

    elif model == 'v010-d200':

        n_symbols = 8824330
        filePath = './PLMs/tencent/v010-d200.txt'
        DIM = 200

    elif model == 'v020-d200-small':

        n_symbols = 2000000
        filePath = './PLMs/tencent/v020-d200-small.txt'
        DIM = 200

    elif model == 'v020-d200-large':

        n_symbols = 12287936
        filePath = './PLMs/tencent/v020-d200-large.txt'
        DIM = 200

    elif model == 'v020-d100-small':

        n_symbols = 2000000
        filePath = './PLMs/tencent/v020-d100-small.txt'
        DIM = 100

    elif model == 'v020-d100-large':

        n_symbols = 12287936
        filePath = './PLMs/tencent/v020-d100-large.txt'
        DIM = 100

    elif 'tw_' or 'cw_' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if model == 'tw_word':
            n_symbols = 636013

        elif model == 'tw_ngram_1':
            n_symbols = 636025

        elif model == 'tw_ngram_2':
            n_symbols = 628667

        elif model == 'tw_ngram_3':
            n_symbols = 6969164

        elif 'tw_char' in model:
            n_symbols = 636086

        elif model == 'tw_position_1':
            n_symbols = 636006

        elif model == 'tw_position_2':
            n_symbols = 636086

        elif model == 'cw_word':
            n_symbols = 636000

        elif model == 'cw_ngram_1':
            n_symbols = 6967693

        elif model == 'cw_ngram_2':
            n_symbols = 13589304

        elif model == 'cw_ngram_3':
            n_symbols = 6969093

        elif model == 'cw_char_1':
            n_symbols = 636248

        elif model == 'cw_char_2':
            n_symbols = 792679

        elif model == 'cw_char_3':
            n_symbols = 1117509

        elif model == 'cw_position_1':
            n_symbols = 1271139

        elif model == 'cw_position_2':
            n_symbols = 6294532

    if 'baidu' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 636013

        elif 'word+ngram_' in model:
            n_symbols = 636025

        elif 'word+char_' in model:
            n_symbols = 636086

        elif 'word+char+ngram_' in model:
            n_symbols = 635974

    if 'wiki' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 352217

        elif 'word+ngram_' in model:
            n_symbols = 352217

        elif 'word+char_' in model:
            n_symbols = 352221

        elif 'word+char+ngram_' in model:
            n_symbols = 352272

    if 'renmin' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 355987

        elif 'word+ngram_' in model:
            n_symbols = 355989

        elif 'word+char_' in model:
            n_symbols = 355996

        elif 'word+char+ngram_' in model:
            n_symbols = 356053

    if 'sogou' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 364990

        elif 'word+ngram_' in model:
            n_symbols = 364992

        elif 'word+char_' in model:
            n_symbols = 365076

        elif 'word+char+ngram_' in model:
            n_symbols = 365113

    if 'financial' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 467370

        elif 'word+ngram_' in model:
            n_symbols = 467377

        elif 'word+char_' in model:
            n_symbols = 467389

        elif 'word+char+ngram_' in model:
            n_symbols = 467210

    if 'zhihu' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 259922

        elif 'word+ngram_' in model:
            n_symbols = 259936

        elif 'word+char_' in model:
            n_symbols = 260008

        elif 'word+char+ngram_' in model:
            n_symbols = 259753

    if 'weibo' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 195202

        elif 'word+ngram_' in model:
            n_symbols = 195202

        elif 'word+char_' in model:
            n_symbols = 195202

        elif 'word+char+ngram_' in model:
            n_symbols = 195197

    if 'literature' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 187959

        elif 'word+ngram_' in model:
            n_symbols = 187960

        elif 'word+char_' in model:
            n_symbols = 187985

        elif 'word+char+ngram_' in model:
            n_symbols = 187980

    if 'sikuquanshu' in model:

        DIM = 300
        filePath = './PLMs/sgns/' + model + '.txt'

        if 'word_' in model:
            n_symbols = 19527

        elif 'word+ngram_' in model:
            n_symbols = 19527

    index_dict = {}
    embedding_weights = np.empty((n_symbols, DIM))
    with open(filePath, encoding='utf-8') as fp:

        index = 0
        for l in tqdm(fp):
            l = l.strip().split(' ')
            if len(l) > DIM:
                word = l[0]
                index_dict[word] = index
                embedding_weights[index, :] = np.asarray([float(i) for i in l[1:]], dtype='float32')
                index += 1

    np.random.seed(SEED)
    shape = (len(word2index), DIM)
    scale = math.sqrt(3.0 / DIM)
    w2v_embedding = np.random.uniform(low=-scale, high=scale, size=shape)

    count = 0
    for i in range(0, len(word2index)):
        w = index2word[i]
        g = index_dict.get(w)
        if g is not None:
            w2v_embedding[i, :] = embedding_weights[g, :]
            count += 1

    print('{num_tokens}-{per:.3f}% tokens in vocab found in word2vector and copied to embedding.'.format(
        num_tokens=count, per=count/float(len(word2index))*100))

    def embeddings(text):

        r_document, r_sentence, count = [], [], 0
        for t in text:
            
            r = []

            if mode == 'char':
                for tt in t:
                    r.append((w2v_embedding[word2index[tt]]))
            elif mode == 'mix':
                for tt in pre_tokenize(t):
                    r.append((w2v_embedding[word2index[tt]]))

            r_sentence.extend(r)
            if count < len(text)-1:
                r_sentence.append(w2v_embedding[word2index['；']])
            r_document.append(np.mean(r, axis=0))
            count += 1
        
        return np.array(r_document), np.array(r_sentence)

    run(embeddings, name)


def tencent_emb():

    import jieba
    jieba.initialize()

    def pre_tokenize(text):

        return [
            w.replace(' ', u'\u2582').replace('\n', u'\u2583').replace('，', ',')
            for w in jieba.cut(text, cut_all=False)
        ]

    vocab = set(sum([pre_tokenize(i) for i in data2chn.values()], [])+sum([pre_tokenize(i) for i in allOtherFeelings], [])+['。', '；'])
    
    print(vocab)

    return 1

    word2index = {w: i for i, w in enumerate(vocab)}
    index2word = {i: w for w, i in word2index.items()}

    TENCENT_DIM = 200
    SEED = 0
    tencent_n_symbols = 8824330
    tencent_index_dict = {}
    tencent_embedding_weights = np.empty((tencent_n_symbols, TENCENT_DIM))
    filePath = './PLMs/Tencent_AILab_ChineseEmbedding/Tencent_AILab_ChineseEmbedding.txt'

    with open(filePath, encoding='utf-8') as fp:
        index = 0
        for l in tqdm(fp):
            l = l.strip().split(' ')
            if len(l) > TENCENT_DIM:
                word = l[0]
                tencent_index_dict[word] = index
                tencent_embedding_weights[index, :] = np.asarray([float(i) for i in l[1:]], dtype='float32')
                index += 1

    np.random.seed(SEED)
    shape = (len(word2index), TENCENT_DIM)
    scale = math.sqrt(3.0 / TENCENT_DIM)
    tencent_embedding = np.random.uniform(low=-scale, high=scale, size=shape)

    count = 0
    for i in tqdm(range(0, len(word2index))):
        w = index2word[i]
        g = tencent_index_dict.get(w)
        if g is not None:
            tencent_embedding[i, :] = tencent_embedding_weights[g, :]
            count += 1

    print('{num_tokens}-{per:.3f}% tokens in vocab found in TENCENT and copied to embedding.'.format(
        num_tokens=count, per=count/float(len(word2index))*100))

    def embeddings(text):
        
        r_document, r_sentence, count = [], [], 0
        for t in text:
            r = []
            for tt in pre_tokenize(t):
                r.append((tencent_embedding[word2index[tt]]))
            r_sentence.extend(r)
            if count < len(text)-1:
                r_sentence.append(tencent_embedding[word2index['；']])
            r_document.append(np.mean(r, axis=0))
            count += 1
        return np.array(r_document), np.array(r_sentence)

    run(embeddings, '002')


def plm_pt_emb(name, number):

    if 'clue/' in name:

        if 'xlnet' in name:
            
            from transformers import XLNetTokenizer, XLNetModel
            tokenizer = XLNetTokenizer.from_pretrained(name)
            plm = XLNetModel.from_pretrained(name)
        
        elif 'albert' in name:
            
            from transformers import BertTokenizer, AlbertModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = AlbertModel.from_pretrained(name)
        
        else:
            from transformers import BertTokenizer, BertModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = BertModel.from_pretrained(name)

    elif 'hfl/' in name:

        if 'mrc' in name:

            from transformers import BertTokenizer, BertForQuestionAnswering
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = BertForQuestionAnswering.from_pretrained(name)
        
        elif 'cino' in name:

            from transformers import XLMRobertaTokenizer, XLMRobertaModel
            tokenizer = XLMRobertaTokenizer.from_pretrained(name)
            plm = XLMRobertaModel.from_pretrained(name)

        elif 'electra' or 'xlnet' in name:

            from transformers import AutoTokenizer, AutoModel
            tokenizer = AutoTokenizer.from_pretrained(name)
            plm = AutoModel.from_pretrained(name)
        
        else:
            from transformers import BertTokenizer, BertModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = BertModel.from_pretrained(name)

    elif 'dpr-ctx' in name:

        from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
        tokenizer = DPRContextEncoderTokenizer.from_pretrained(name)
        plm = DPRContextEncoder.from_pretrained(name)

    elif 'dpr-question' in name:

        from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer
        tokenizer = DPRQuestionEncoderTokenizer.from_pretrained(name)
        plm = DPRQuestionEncoder.from_pretrained(name)

    elif 'voidful/albert' in name:

        from transformers import AutoTokenizer, AlbertForMaskedLM
        tokenizer = AutoTokenizer.from_pretrained(name)
        plm = AlbertForMaskedLM.from_pretrained(name)

    elif 'uer/' in name:

        if name in ['uer/bart-large-chinese-cluecorpussmall', 'uer/bart-base-chinese-cluecorpussmall', 'uer/bart-chinese-6-960-cluecorpussmall', 'uer/bart-chinese-4-768-cluecorpussmall']:

            from transformers import BertTokenizer, BartForConditionalGeneration
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = BartForConditionalGeneration.from_pretrained(name)

        elif name in ['uer/pegasus-large-chinese-cluecorpussmall', 'uer/pegasus-base-chinese-cluecorpussmall']:

            from transformers import BertTokenizer, PegasusForConditionalGeneration
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = PegasusForConditionalGeneration.from_pretrained(name)

        elif 'albert' in name:

            from transformers import BertTokenizer, AlbertModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = AlbertModel.from_pretrained(name)

        elif name in ['uer/roberta-base-finetuned-chinanews-chinese', 'uer/roberta-base-finetuned-ifeng-chinese', 'uer/roberta-base-finetuned-dianping-chinese', 'uer/roberta-base-finetuned-jd-binary-chinese', 'uer/roberta-base-finetuned-jd-full-chinese', ]:

            from transformers import AutoModelForSequenceClassification, AutoTokenizer
            tokenizer = AutoTokenizer.from_pretrained(name)
            plm = AutoModelForSequenceClassification.from_pretrained(name)

        elif name in ['uer/roberta-base-finetuned-cluener2020-chinese', ]:

            from transformers import AutoModelForTokenClassification, AutoTokenizer
            tokenizer = AutoTokenizer.from_pretrained(name)
            plm = AutoModelForTokenClassification.from_pretrained(name)

        elif name in ['uer/roberta-base-chinese-extractive-qa', ]:

            from transformers import AutoModelForQuestionAnswering, AutoTokenizer
            tokenizer = AutoTokenizer.from_pretrained(name)
            plm = AutoModelForQuestionAnswering.from_pretrained(name)

        elif name in ['uer/t5-v1_1-base-chinese-cluecorpussmall', 'uer/t5-v1_1-small-chinese-cluecorpussmall', 'uer/t5-base-chinese-cluecorpussmall', 'uer/t5-small-chinese-cluecorpussmall', ]:

            from transformers import MT5ForConditionalGeneration, BertTokenizer
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = MT5ForConditionalGeneration.from_pretrained(name)

        elif name in ['uer/gpt2-chinese-lyric', 'uer/gpt2-chinese-couplet', 'uer/gpt2-chinese-poem', 'uer/gpt2-chinese-ancient', 'uer/gpt2-chinese-cluecorpussmall', 'uer/gpt2-distil-chinese-cluecorpussmall', '']:

            from transformers import BertTokenizer, GPT2LMHeadModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = GPT2LMHeadModel.from_pretrained(name)

        elif name in ['uer/roberta-base-word-chinese-cluecorpussmall', 'uer/roberta-medium-word-chinese-cluecorpussmall', 'uer/roberta-small-word-chinese-cluecorpussmall', 'uer/roberta-mini-word-chinese-cluecorpussmall', 'uer/roberta-tiny-word-chinese-cluecorpussmall']:

            from transformers import AlbertTokenizer, BertModel
            tokenizer = AlbertTokenizer.from_pretrained(name)
            plm = BertModel.from_pretrained(name)

        else:

            from transformers import BertTokenizer, BertModel
            tokenizer = BertTokenizer.from_pretrained(name)
            plm = BertModel.from_pretrained(name)

    elif 'xlm-roberta' in name:

        from transformers import AutoTokenizer, AutoModelForMaskedLM
        tokenizer = AutoTokenizer.from_pretrained(name)
        plm = AutoModelForMaskedLM.from_pretrained(name)

    elif 'google/canine' in name:

        from transformers import CanineTokenizer, CanineModel
        tokenizer = CanineTokenizer.from_pretrained(name)
        plm = CanineModel.from_pretrained(name)

    else:

        from transformers import AutoTokenizer, AutoModel
        tokenizer = AutoTokenizer.from_pretrained(name)
        plm = AutoModel.from_pretrained(name)

    def encoder1(text):

        encoded_input = tokenizer(text, return_tensors='pt')
        output = plm(**encoded_input, output_hidden_states=True)

        return np.mean(np.concatenate((np.mean(output.hidden_states[1].detach().numpy(), axis=1), 
               np.mean(output.hidden_states[-1].detach().numpy(), axis=1))), axis=0)

    def encoder2(text):

        encoded_input = tokenizer(text, return_tensors='pt')
        output = plm(**encoded_input, output_hidden_states=True)
        
        return np.mean(np.concatenate((output.hidden_states[1].detach().numpy(), output.hidden_states[-1].detach().numpy())), axis=0)

    def embeddings_sentence(text, encoder='2'):

        if encoder == '2':
            return encoder2(text)
        elif encoder == '1':
            return encoder1(text)
    
    for i in chn2embeddings:
        chn2embeddings[i] = embeddings_sentence(i, encoder='1')

    def embeddings(text):

        r_document = [chn2embeddings[t] if t in chn2embeddings else embeddings_sentence(t, encoder='1') for t in text]
        r_sentence = embeddings_sentence('；'.join(text))
        return np.array(r_document), r_sentence
    
    t = embeddings(['一点也没有满意',])
    print(t[0].shape, t[1].shape)

    run(embeddings, number)


def plm_tf_emb(name, number, model='bert'):

    import jieba
    import json
    import numpy as np
    from bert4keras.models import build_transformer_model
    from bert4keras.tokenizers import SpTokenizer, Tokenizer
    from bert4keras.backend import keras, K
    from bert4keras.tokenizers import Tokenizer
    from bert4keras.models import build_transformer_model
    from keras.models import Model

    jieba.initialize()

    if name == 'ZhuiyiTechnology/GAU':
        
        from bert4keras.gau_model import GAU_alpha
        model = GAU_alpha

    if name in ['ZhuiyiTechnology/mT5-small', 'ZhuiyiTechnology/mT5-base']:
        
        config_path = './PLMs/' + name + '/bert_config.json'
        checkpoint_path = './PLMs/' + name + '/model.ckpt-1000000'
        spm_path = './PLMs/' + name + '/sentencepiece_cn.model'
        keep_tokens_path = './PLMs/' + name + '/sentencepiece_cn_keep_tokens.json'
        tokenizer = SpTokenizer(spm_path,
                                token_start=None,
                                token_end='&lt;/s&gt;')
        keep_tokens = json.load(open(keep_tokens_path))

    elif name == 'GPT2/1.5B':
        
        config_path = './PLMs/GPT2/1.5B/config.json'
        checkpoint_path = './PLMs/GPT2/1.5B/model.ckpt-100000'
        dict_path = './PLMs/GPT2/1.5B/vocab.txt'

        tokenizer = Tokenizer(dict_path,
                              token_start=None,
                              token_end=None,
                              do_lower_case=True)

    elif name == 'GPT2/2.6B':

        config_path = './PLMs/GPT2/2.6B/config.json'
        checkpoint_path = './PLMs/GPT2/2.6B/model.ckpt'
        spm_path = './PLMs/GPT2/2.6B/chinese_vocab.model'

        def pre_tokenize(text):
            return [
                w.replace(' ', u'\u2582').replace('\n', u'\u2583')
                for w in jieba.cut(text, cut_all=False)
            ]

        tokenizer = SpTokenizer(spm_path,
                                token_start=None,
                                token_end=None,
                                pre_tokenize=pre_tokenize,
                                token_translate={u'\u2583': '<cls>'})

    else:
        config_path = './PLMs/' + name + '/bert_config.json'
        checkpoint_path = './PLMs/' + name + '/bert_model.ckpt'
        dict_path = './PLMs/' + name + '/vocab.txt'

        tokenizer = Tokenizer(dict_path, do_lower_case=True)

    if name in [
            'ZhuiyiTechnology/WoBERT', 'ZhuiyiTechnology/WoNEZHA',
            'ZhuiyiTechnology/WoBERT+', 'ZhuiyiTechnology/RoFormer', 'ZhuiyiTechnology/RoFormer-small'
            'ZhuiyiTechnology/T5-small', 'ZhuiyiTechnology/T5-base'
    ]:
        tokenizer = Tokenizer(dict_path,
                              do_lower_case=True,
                              pre_tokenize=lambda s: jieba.cut(s, HMM=False))

    class GlobalAveragePooling1D(keras.layers.GlobalAveragePooling1D):

        def call(self, inputs, mask=None):

            if mask is not None:
                
                mask = K.cast(mask, K.floatx())[:, :, None]
                return K.sum(inputs * mask, axis=1) / K.sum(mask, axis=1)
            else:
                
                return K.mean(inputs, axis=1)

    if name in [
            'ZhuiyiTechnology/T5-small', 'ZhuiyiTechnology/T5-base',
            'ZhuiyiTechnology/mT5-small', 'ZhuiyiTechnology/mT5-base'
    ]:
        if name in ['ZhuiyiTechnology/mT5-small', 'ZhuiyiTechnology/mT5-base']:
            plm = build_transformer_model(
                config_path=config_path,
                checkpoint_path=checkpoint_path,
                keep_tokens=keep_tokens,
                model=model,
                return_keras_model=False,
                name='T5',
            )
        else:
            plm = build_transformer_model(
                config_path=config_path,
                checkpoint_path=checkpoint_path,
                model=model,
                return_keras_model=False,
                name='T5',
            )

        plm = plm.encoder

        encoder_layers, count = [], 0
        while True:
            try:
                output = plm.get_layer(
                    'Encoder-Transformer-%d-FeedForward-Norm' % count).output
                encoder_layers.append(output)
                count += 1
            except:
                break

    elif name == 'GPT2/1.5B':
        plm = build_transformer_model(
            config_path,
            checkpoint_path,
            model=model
        )

        encoder_layers, count = [], 0
        while True:
            try:
                output = plm.get_layer('Transformer-%d-FeedForward-Norm-1' %
                                       count).output
                encoder_layers.append(output)
                count += 1
            except:
                break

    else:
        plm = build_transformer_model(config_path,
                                      checkpoint_path,
                                      model=model)

        encoder_layers, count = [], 0
        while True:
            try:

                if name == 'ZhuiyiTechnology/GAU':
                    output = plm.get_layer('Transformer-%d-GatedAttentionUnit-Norm' %
                                        count).output
                else:
                    output = plm.get_layer('Transformer-%d-FeedForward-Norm' %
                                        count).output
                
                encoder_layers.append(output)
                count += 1
            except:
                break

    n_last, outputs1, outputs2 = 2, [], []
    for i in range(n_last):

        outputs1.append(GlobalAveragePooling1D()(encoder_layers[-i]))
        outputs2.append(encoder_layers[-i])

    output1 = keras.layers.Average()(outputs1)
    output2 = keras.layers.Average()(outputs2)

    encoder1 = Model(plm.inputs, output1)
    encoder2 = Model(plm.inputs, output2)

    def embeddings_sentence(text, encoder=encoder2):

        token_ids, segment_ids = tokenizer.encode(text)
        if encoder == encoder2:
            if name in [
                    'ZhuiyiTechnology/T5-small',
                    'ZhuiyiTechnology/T5-base',
                    'ZhuiyiTechnology/mT5-small',
                    'ZhuiyiTechnology/mT5-base', 'GPT2/1.5B', 'GPT2/2.6B'
            ]:
                return encoder2.predict([np.array([token_ids])], verbose=0)[0]
            else:
                return encoder2.predict(
                    [np.array([token_ids]),
                     np.array([segment_ids])], verbose=0)[0]
        else:
            if name in [
                    'ZhuiyiTechnology/T5-small',
                    'ZhuiyiTechnology/T5-base',
                    'ZhuiyiTechnology/mT5-small',
                    'ZhuiyiTechnology/mT5-base', 'GPT2/1.5B', 'GPT2/2.6B'
            ]:
                return encoder1.predict([np.array([token_ids])], verbose=0)[0]
            else:
                return encoder1.predict(
                    [np.array([token_ids]),
                     np.array([segment_ids])], verbose=0)[0]

    for i in chn2embeddings:
        chn2embeddings[i] = embeddings_sentence(i, encoder=encoder1)

    def embeddings(text):

        r_document = [
            chn2embeddings[t] if t in chn2embeddings else embeddings_sentence(
                t, encoder=encoder1) for t in text
        ]
        r_sentence = embeddings_sentence('；'.join(text))
        return np.array(r_document), r_sentence

    t = embeddings(['一点也没有满意',])
    print(t[0].shape, t[1].shape)

    run(embeddings, number)
    

def run(embeddings, number):

    print('PLM ' + number + ' Processing All Affect')
    allScore = explore_plm_av((allAffect, allOtherFeelings),
                              embeddings,
                              of=False)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_aa/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('PLM ' + number + ' Processing All Affect and Other Feelings')
    allScore = explore_plm_av((allAffect, allOtherFeelings),
                              embeddings,
                              of=True)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_aa+of/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('PLM ' + number + ' Processing Positive Affect')
    allScore = explore_plm_av(([[[i[0], i[1], i[2], i[-1]] for i in j]
                                for j in allAffect], allOtherFeelings),
                              embeddings,
                              of=False)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_pa/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('PLM ' + number + ' Processing Positive Affect and Other Feelings')
    allScore = explore_plm_av(([[[i[0], i[1], i[2], i[-1]] for i in j]
                                for j in allAffect], allOtherFeelings),
                              embeddings,
                              of=True)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_pa+of/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('PLM ' + number + ' Processing Negative Affect')
    allScore = explore_plm_av(
        ([[[i[3], i[4]] for i in j] for j in allAffect], allOtherFeelings),
        embeddings,
        of=False)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_na/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)
    
    print('PLM ' + number + ' Processing Negative Affect and Other Feelings')
    allScore = explore_plm_av(
        ([[[i[3], i[4]] for i in j] for j in allAffect], allOtherFeelings),
        embeddings,
        of=True)
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_na+of/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)

    print('PLM ' + number + ' Processing Other Feelings')
    allScore = explore_plm_av((allAffect, allOtherFeelings),
                              embeddings,
                              of='only')
    firstScore = [allScore[i] for i, j in enumerate(allSegment) if j == 1]
    secondScore = [allScore[i] for i, j in enumerate(allSegment) if j == 2]
    thirdScore = [allScore[i] for i, j in enumerate(allSegment) if j == 3]
    w_filePath = './data/av_data/plm_of/' + number + '.pkl'
    with open(w_filePath, 'wb') as fp:
        pickle.dump((allScore, firstScore, secondScore, thirdScore), fp, -1)