plus-minus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import fnmatch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import gzip\n",
    "import pickle\n",
    "import csv\n",
    "import scipy.sparse\n",
    "Xauth = None\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the update to work despite the broken scipy documentation\n",
    "try:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a.update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix.update\n",
    "except:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a._update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix._update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('useful_venue_list.pkl.gz','rb') as fp:\n",
    "    all_venues = pickle.load(fp)\n",
    "with gzip.open('useful_authors_list.pkl.gz','rb') as fp:\n",
    "    all_authors = pickle.load(fp)\n",
    "with gzip.open('useful_papers.pkl.gz','rb') as fp:\n",
    "    all_papers = pickle.load(fp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_year = all_papers[0][6]\n",
    "max_year = all_papers[-1][6]\n",
    "span_years = max_year - min_year + 1\n",
    "print(min_year,max_year,span_years)\n",
    "conf_idx = {v:i for i,v in enumerate(all_venues)}\n",
    "name_idx = {v:i for i,v in enumerate(all_authors)}\n",
    "n_confs = len(all_venues)\n",
    "n_auths = len(all_authors)\n",
    "n_papers = len(all_papers)\n",
    "print(n_confs,n_auths,n_papers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf =  np.load('clf_gold.pkl.npy')\n",
    "years_per_conf = clf.shape[0]//n_confs\n",
    "YEAR_BLOCKS = span_years//years_per_conf\n",
    "clf[2323]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse\n",
    "import gc\n",
    "if Xauth is None:\n",
    "    valid_ns = set()\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        valid_ns.add(n)\n",
    "    \n",
    "    per_author_val = {}\n",
    "    for n in valid_ns:\n",
    "        author_scores = 1/(np.arange(n)+1)\n",
    "        per_author_val[n] = author_scores/author_scores.sum()\n",
    "        \n",
    "    count_vecs = {}\n",
    "    paper_vecs = []\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "\n",
    "        author_scores = per_author_val[n]\n",
    "        paper_vecs.append([(name_idx[a],j,v) for a,v in zip(authors,author_scores)])\n",
    "        \n",
    "    Xauth = scipy.sparse.dok_matrix((n_auths,years_per_conf*n_confs))\n",
    "    xdict = {}\n",
    "  \n",
    "    for paper_vec in paper_vecs:\n",
    "        for i,j,v in paper_vec:\n",
    "            xdict[(i,j)] = v + xdict.get((i,j),0)\n",
    "\n",
    "    Xauth.my_update(xdict)\n",
    "            \n",
    "    Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "    paper_vec = []\n",
    "    xdict = {}\n",
    "    gc.collect()\n",
    "    auth_years = np.ones((n_auths,2)) * np.array([3000,1000]) \n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        for a in authors:\n",
    "            i = name_idx[a]\n",
    "            auth_years[i,0] = min(auth_years[i,0],year)\n",
    "            auth_years[i,1] = max(auth_years[i,1],year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmu_uni = pd.read_csv('other_ranks/cmu_faculty.csv')\n",
    "cmu_uni = cmu_uni.fillna('Other')\n",
    "cmu_uni = cmu_uni[cmu_uni.dept == 'RI']\n",
    "#uni_names = ['Andrea Tagliasacchi','Paul G. Kry']#['Xuemin Shen','H. Vincent Poor','Kang G. Shin','Mohamed-Slim Alouini','Lajos Hanzo']#list(cmu_uni.name)\n",
    "#uni_names = list(faculty_affil[faculty_affil.affiliation == 'Johns Hopkins University'].name)\n",
    "uni_names = list(cmu_uni.name)\n",
    "print(len(uni_names))\n",
    "conf_counts = {}\n",
    "conf_counts_value = {}\n",
    "\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    if year < 2000:\n",
    "        continue\n",
    "    n = len(authors)\n",
    "    for a in authors:\n",
    "        if a in uni_names:\n",
    "            conf_counts[venue] = 1/n + conf_counts.get(venue,0)\n",
    "            conf_counts_value[venue] = clf[years_per_conf*(conf_idx[venue]) + (year-min_year)//YEAR_BLOCKS]/n + conf_counts_value.get(venue,0)\n",
    "conf_counts_value = {k: v/conf_counts[k] for k,v in conf_counts_value.items()}\n",
    "ri_fav_confs = [(conf_counts[_[1]]*conf_counts_value[_[1]],_[1],conf_counts[_[1]],conf_counts_value[_[1]]) for _ in sorted([(v,k) for k,v in conf_counts.items() if v > 0],reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confs_to_filter = [_[1] for _ in sorted(ri_fav_confs,reverse=True) if _[-2] >= 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "only_after = 1970\n",
    "start_year_idx = max(0,only_after-min_year)//YEAR_BLOCKS\n",
    "\n",
    "year_filter = np.zeros_like(clf).reshape((-1,years_per_conf))\n",
    "\n",
    "if False: # filter confs\n",
    "    for conf in confs_to_filter:\n",
    "        year_filter[conf_idx[conf],start_year_idx:] = 1\n",
    "else:\n",
    "    year_filter[:,start_year_idx:] = 1\n",
    "total_scores = Xauth.dot(clf * year_filter.reshape((-1)))\n",
    "\n",
    "\n",
    "year_filter = np.zeros_like(clf).reshape((-1,years_per_conf))\n",
    "\n",
    "for conf in confs_to_filter:\n",
    "    year_filter[conf_idx[conf],start_year_idx:] = 1\n",
    "ri_scores = Xauth.dot(clf * year_filter.reshape((-1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ri_scores.shape,Xauth.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(total_scores)[::-1]\n",
    "for k in range(10):\n",
    "    idx = best_idx[k]\n",
    "    print('{:30s}\\t{:.2f}'.format(all_authors[idx],total_scores[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New APM?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse\n",
    "import gc\n",
    "Xauth = None\n",
    "unique_auth = set()\n",
    "\n",
    "\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    auth_set = tuple(sorted(authors))\n",
    "    unique_auth.add(auth_set)\n",
    "\n",
    "unique_auth = sorted(list(unique_auth))\n",
    "unique_auth_idx = {v:k for k,v in enumerate(unique_auth)}\n",
    "\n",
    "n_unique_auths = len(unique_auth)\n",
    "Xauth = scipy.sparse.dok_matrix((n_unique_auths,n_auths))\n",
    "y = np.zeros(n_unique_auths)\n",
    "xdict = {}\n",
    "\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    auth_set = tuple(sorted(authors))\n",
    "    k = unique_auth_idx[auth_set]\n",
    "    n = len(authors)\n",
    "    for a,v in zip(authors,per_author_val[n]):\n",
    "        j = name_idx[a]\n",
    "        # 1/n, 1\n",
    "        xdict[(k,j)] = 1/n\n",
    "    k_idx = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "    y[k] += clf[k_idx]\n",
    "Xauth.my_update(xdict)\n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "DEBUG_THING = False\n",
    "FI =  bool(int(os.environ.get('FIT_INTERCEPT',1))) #7\n",
    "\n",
    "new_set = [3e-3,1e-1]\n",
    "new_set = [1e-7, 326]\n",
    "new_set = [5e-3, 0.46]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "clf2 = SGDRegressor('huber',alpha=new_set[0],epsilon=new_set[1],penalty='l2',tol=1e-6,max_iter=100,average=True,verbose=1,fit_intercept=FI)\n",
    "clf2.fit(Xauth,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if DEBUG_THING:\n",
    "    X = Xauth\n",
    "    rand_idx = np.arange(X.shape[0])\n",
    "    HOLDOUT=int(round(0.85*rand_idx.shape[0]))\n",
    "\n",
    "    np.random.shuffle(rand_idx)\n",
    "\n",
    "\n",
    "    X_train = X[rand_idx[:HOLDOUT]]\n",
    "    X_test = X[rand_idx[HOLDOUT:]]\n",
    "    y_train = y[rand_idx[:HOLDOUT]]\n",
    "    y_test = y[rand_idx[HOLDOUT:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if DEBUG_THING:\n",
    "    def eval_reg(regW,X_train,X_test,y_train,y_test):\n",
    "        regW2 = np.exp(regW)[0]\n",
    "        eps = np.exp(regW)[1]\n",
    "        clf2 = SGDRegressor('huber',alpha=regW2,penalty='l2',\n",
    "                            tol=1e-9,max_iter=15,average=True,verbose=0,\n",
    "                            fit_intercept=True,epsilon=eps)\n",
    "        #clf2.fit(Xauth,yt)\n",
    "        clf2.fit(X_train,y_train)\n",
    "\n",
    "        diff = clf2.predict(X_test) - y_test\n",
    "        coef = (clf2.coef_)\n",
    "        term1 = abs(diff).mean()\n",
    "        term2 = (diff**2).mean()\n",
    "        err_vec = [term1,term2]\n",
    "        print(err_vec)\n",
    "        return np.prod(err_vec)**(1/len(err_vec))\n",
    "    x0 = np.log(new_set)\n",
    "    print(eval_reg(x0,X_train,X_test,y_train,y_test))\n",
    "    \n",
    "# [1.6194369583392894, 17.338543782917405, 66516]\n",
    "# 123.149956707115"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cma\n",
    "\n",
    "from multiprocessing import Pool\n",
    "x0 = np.log(new_set)\n",
    "if DEBUG_THING:\n",
    "    with Pool(8) as p:\n",
    "        es = cma.CMAEvolutionStrategy(x0,0.3,{'popsize_factor':2.5})\n",
    "        while not es.stop():\n",
    "            solutions = es.ask()\n",
    "\n",
    "            if False:\n",
    "                np.random.shuffle(rand_idx)\n",
    "                X_train = X[rand_idx[:HOLDOUT]]\n",
    "                X_test = X[rand_idx[HOLDOUT:]]\n",
    "                y_train = y[rand_idx[:HOLDOUT]]\n",
    "                y_test = y[rand_idx[HOLDOUT:]]\n",
    "\n",
    "            res = p.starmap(eval_reg, [(_,X_train,X_test,y_train,y_test) for _ in solutions])\n",
    "            #res = [eval_reg(x) for x in solutions]\n",
    "            es.tell(solutions, res)\n",
    "            es.logger.add() \n",
    "            es.disp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if DEBUG_THING:\n",
    "    print(np.exp(es.best.x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "napm_weights = clf2.coef_\n",
    "napm_weights = (napm_weights-napm_weights.mean())/napm_weights.std()\n",
    "_ = plt.hist(napm_weights,100,log=True)\n",
    "print((clf2.intercept_-napm_weights.mean())/napm_weights.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "curious_names = ['Xiaolong Wang 0004','Judy Hoffman','Paris Siminelakis','Roie Levin','Leonid Keselman',\n",
    "                 'Nicholas Rhinehart','Vincent Sitzmann','Siddharth Ancha','Xingyu Lin',\n",
    "                 'Humphrey Hu','Aditya Dhawale','Nick Gisolfi','Andrey Kurenkov','Micah Corah',\n",
    "                 'David F. Fouhey','Chelsea Finn','Akshara Rai','Ankit Bhatia','Xuning Yang',\n",
    "                 'Lerrel Pinto','Alexander Spitzer','Roberto Shu','Allison Del Giorno','Nadine Chang',\n",
    "                 'Justin Johnson 0001','Kumar Shaurya Shankar','Ellen A. Cappo','Hunter Goforth',\n",
    "                 'Amir Roshan Zamir','Jonathan T. Barron','Dorsa Sadigh','Derek Hoiem','Vaggos Chatziafratis',\n",
    "                 'Brian Okorn','David Held','Adam W. Harley','Hsiao-Yu Fish Tung','Tess Lee Hellebrekers','Mark Sheinin']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(napm_weights[name_idx[name]],name) for name in all_authors],reverse=True)\n",
    "uni_scores[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(napm_weights[name_idx[name]],name) for name in curious_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the APM matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = None\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = scipy.sparse.dok_matrix((n_papers,n_auths))\n",
    "y = np.zeros(n_papers)\n",
    "xdict = {}\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    for a in authors:\n",
    "        j = name_idx[a]\n",
    "        xdict[(i,j)] = 1/n\n",
    "    ji = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "    y[i] = clf[ji]\n",
    "\n",
    "Xauth.my_update(xdict)\n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "paper_vec = []\n",
    "xdict = {}\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = plt.hist(y,50)\n",
    "plt.figure()\n",
    "_ = plt.hist((y-y.mean())/y.std(),50)\n",
    "y.std(),y.mean()\n",
    "plt.figure()\n",
    "yt = np.copy(y)\n",
    "where = np.where(y > 0)\n",
    "yt2 = np.log(yt[where])\n",
    "yt[where] = (yt2-yt2.mean())/yt2.std()\n",
    "yt = np.clip(yt,-3.5,3.5)\n",
    "_ = plt.hist(yt,50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "FI =  bool(int(os.environ.get('FIT_INTERCEPT',0))) #7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "clf2 = SGDRegressor('huber',alpha=1e-3,penalty='l2',tol=1e-6,max_iter=1000,average=True,verbose=1,fit_intercept=FI)\n",
    "#clf2.fit(Xauth,yt)\n",
    "clf2.fit(Xauth,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apm_weights = clf2.coef_\n",
    "apm_weights = (apm_weights-apm_weights.mean())/apm_weights.std()\n",
    "_ = plt.hist(apm_weights,100,log=True)\n",
    "print((clf2.intercept_-apm_weights.mean())/apm_weights.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cumulative, pairwise apm\n",
    "Some questions I have\n",
    "* Should you get 1 and 0.5 as the coefficients?\n",
    "* Should you get per_paper or total credit?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "pairwise_authors = {}\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    authors = sorted(authors)\n",
    "    for ap in itertools.combinations(authors, 2):\n",
    "        if not ap in pairwise_authors:\n",
    "            pairwise_authors[ap] = len(pairwise_authors)\n",
    "n_pairwise_authors = len(pairwise_authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = scipy.sparse.dok_matrix((n_auths+n_pairwise_authors,n_auths))\n",
    "y = np.zeros(n_auths+n_pairwise_authors)\n",
    "xdict = {}\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    ji = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "    per_person_credit = clf[ji]/n\n",
    "    for a in authors:\n",
    "        j = name_idx[a]\n",
    "        xdict[(j,j)] = 1\n",
    "        y[j] += per_person_credit\n",
    "    \n",
    "    authors_s = sorted(authors)\n",
    "    for ap in itertools.combinations(authors_s, 2):\n",
    "        j2 = pairwise_authors[ap] + n_auths\n",
    "        a1 = name_idx[ap[0]]\n",
    "        a2 = name_idx[ap[1]]\n",
    "        xdict[(j2,a1)] = 0.5\n",
    "        xdict[(j2,a2)] = 0.5\n",
    "        y[j2] += per_person_credit\n",
    "\n",
    "\n",
    "Xauth.my_update(xdict)\n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "paper_vec = []\n",
    "xdict = {}\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "clf3 = SGDRegressor('huber',alpha=1e-3,penalty='l2',tol=1e-6,max_iter=1000,average=True,verbose=1,fit_intercept=FI)\n",
    "clf3.fit(Xauth,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pw_apm_weights = clf3.coef_\n",
    "pw_apm_weights = (pw_apm_weights-pw_apm_weights.mean())/pw_apm_weights.std()\n",
    "_ = plt.hist(pw_apm_weights,100,log=True)\n",
    "print((clf3.intercept_-pw_apm_weights.mean())/pw_apm_weights.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = scipy.sparse.dok_matrix((n_auths+n_pairwise_authors,n_auths))\n",
    "y = np.zeros(n_auths+n_pairwise_authors)\n",
    "xdict = {}\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    ji = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "    per_person_credit = clf[ji]/n\n",
    "    for a in authors:\n",
    "        j = name_idx[a]\n",
    "        xdict[(j,j)] = 1 + xdict.get((j,j),0)\n",
    "        y[j] += per_person_credit\n",
    "    \n",
    "    authors_s = sorted(authors)\n",
    "    for ap in itertools.combinations(authors_s, 2):\n",
    "        j2 = pairwise_authors[ap] + n_auths\n",
    "        a1 = name_idx[ap[0]]\n",
    "        a2 = name_idx[ap[1]]\n",
    "        xdict[(j2,a1)] = 0.5 + xdict.get((j2,a1),0)\n",
    "        xdict[(j2,a2)] = 0.5 + xdict.get((j2,a2),0)\n",
    "        y[j2] += per_person_credit\n",
    "\n",
    "\n",
    "Xauth.my_update(xdict)\n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "paper_vec = []\n",
    "xdict = {}\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "clf4 = SGDRegressor('huber',alpha=1e-3,penalty='l2',tol=1e-6,max_iter=1000,average=True,verbose=1,fit_intercept=FI)\n",
    "clf4.fit(Xauth,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pweff_apm_weights = clf4.coef_\n",
    "pweff_apm_weights = (pweff_apm_weights-pweff_apm_weights.mean())/pweff_apm_weights.std()\n",
    "_ = plt.hist(pweff_apm_weights,100,log=True)\n",
    "print((clf4.intercept_-pweff_apm_weights.mean())/pweff_apm_weights.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    Xauth = scipy.sparse.dok_matrix((n_auths+n_pairwise_authors,n_auths + 1))\n",
    "    y = np.zeros(n_auths+n_pairwise_authors)\n",
    "    xdict = {}\n",
    "\n",
    "    for i,paper in enumerate(all_papers):\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        ji = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "        per_person_credit = clf[ji]\n",
    "        for a in authors:\n",
    "            j = name_idx[a]\n",
    "            xdict[(j,j)] = 1 + xdict.get((j,j),0)\n",
    "            # an unknown author\n",
    "            xdict[(j,n_auths)] = (n-1) + xdict.get((j,n_auths),0)\n",
    "\n",
    "            y[j] += per_person_credit\n",
    "\n",
    "\n",
    "        authors_s = sorted(authors)\n",
    "        for ap in itertools.combinations(authors_s, 2):\n",
    "            j2 = pairwise_authors[ap] + n_auths + 1\n",
    "            a1 = name_idx[ap[0]]\n",
    "            a2 = name_idx[ap[1]]\n",
    "            xdict[(j2,a1)] = 1.0 + xdict.get((j2,a1),0)\n",
    "            xdict[(j2,a2)] = 1.0 + xdict.get((j2,a2),0)\n",
    "            y[j2] += per_person_credit\n",
    "            xdict[(j2,n_auths)] = (n-2) + xdict.get((j2,n_auths),0)\n",
    "\n",
    "\n",
    "\n",
    "    Xauth.my_update(xdict)\n",
    "\n",
    "    Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "    paper_vec = []\n",
    "    xdict = {}\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    from sklearn.linear_model import SGDRegressor\n",
    "    clf5 = SGDRegressor('huber',alpha=1e-3,penalty='l2',tol=1e-6,max_iter=25,average=True,verbose=1,fit_intercept=FI)\n",
    "    clf5.fit(Xauth,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    pwunk_apm_weights = clf5.coef_[:n_auths]\n",
    "    pwunk_apm_weights = (pwunk_apm_weights-pwunk_apm_weights.mean())/pwunk_apm_weights.std()\n",
    "    _ = plt.hist(pwunk_apm_weights,100,log=True)\n",
    "    print((clf5.coef_[n_auths]-pwunk_apm_weights.mean())/pwunk_apm_weights.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get author year counts\n",
    "auth_pro_years = defaultdict(set)\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    for a in authors:\n",
    "        auth_pro_years[a].add(year)\n",
    "            \n",
    "    authors_s = sorted(authors)\n",
    "    for ap in itertools.combinations(authors_s, 2):\n",
    "        auth_pro_years[ap].add(year)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth_pro_years_count = {k: len(v) for k,v in auth_pro_years.items()}\n",
    "auth_pro_years = None\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "year_span2 = np.array([auth_pro_years_count[n] for n in all_authors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = scipy.sparse.dok_matrix((n_auths+n_pairwise_authors,n_auths))\n",
    "y = np.zeros(n_auths+n_pairwise_authors)\n",
    "xdict = {}\n",
    "\n",
    "for i,paper in enumerate(all_papers):\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    ji = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "    credit = clf[ji]\n",
    "    author_scores = per_author_val[n]\n",
    "\n",
    "    for a,c in zip(authors,author_scores):\n",
    "        j = name_idx[a]\n",
    "        xdict[(j,j)] = c + xdict.get((j,j),0)\n",
    "\n",
    "        y[j] += c*credit\n",
    "\n",
    "    authors_s = sorted([(a,c) for a,c in zip(authors,author_scores)])\n",
    "    for ap in itertools.combinations(authors_s, 2):\n",
    "        a1,c1 = ap[0]\n",
    "        a2,c2 = ap[1] \n",
    "        j2 = pairwise_authors[(a1,a2)] + n_auths\n",
    "        a1 = name_idx[a1]\n",
    "        a2 = name_idx[a2]\n",
    "        xdict[(j2,a1)] = c1 + xdict.get((j2,a1),0)\n",
    "        xdict[(j2,a2)] = c2 + xdict.get((j2,a2),0)\n",
    "        y[j2] += (c1+c2)*credit\n",
    "\n",
    "\n",
    "Xauth.my_update(xdict) \n",
    "\n",
    "Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "paper_vec = []\n",
    "xdict = {}\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yn = np.zeros_like(y)\n",
    "for k,v in name_idx.items():\n",
    "    yn[v] = auth_pro_years_count[k]\n",
    "for k,v in pairwise_authors.items():\n",
    "    yn[n_auths+v] = auth_pro_years_count[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "clf5 = SGDRegressor('huber',alpha=1e-3,penalty='l2',tol=1e-6,max_iter=25,average=True,verbose=1,fit_intercept=FI)\n",
    "clf5.fit(Xauth,y/yn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pwunk_apm_weights = clf5.coef_\n",
    "pwunk_apm_weights = (pwunk_apm_weights-pwunk_apm_weights.mean())/pwunk_apm_weights.std()\n",
    "_ = plt.hist(pwunk_apm_weights,100,log=True)\n",
    "print((clf5.intercept_-pwunk_apm_weights.mean())/pwunk_apm_weights.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#year_span2 = (auth_years[:,1] - auth_years[:,0]) + 1\n",
    "year_span2 = np.maximum(1,year_span2)\n",
    "\n",
    "eff_scores = total_scores/year_span2\n",
    "df_clfs2 = pd.DataFrame(np.vstack([total_scores,eff_scores,apm_weights,pw_apm_weights,pweff_apm_weights,pwunk_apm_weights]).T,columns=['Total','Per Year','APM','PW APM','PW Eff APM','PW APM Yearly'])\n",
    "#print(df_clfs2.corr('spearman').to_latex())\n",
    "corr = df_clfs2.corr('spearman')\n",
    "print(corr.mean(0))\n",
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(napm_weights)[::-1]\n",
    "for k in range(180):\n",
    "    idx = best_idx[k]\n",
    "    name = all_authors[idx]\n",
    "    if name + ' 0001' in name_idx:\n",
    "        print('.',end='')\n",
    "        #continue\n",
    "    print('{:30s}\\t{:.2f}\\t{:.4f}'.format(all_authors[idx],total_scores[idx],napm_weights[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(apm_weights)[::-1]\n",
    "for k in range(180):\n",
    "    idx = best_idx[k]\n",
    "    name = all_authors[idx]\n",
    "    if name + ' 0001' in name_idx:\n",
    "        print('.',end='')\n",
    "        #continue\n",
    "    print('{:30s}\\t{:.2f}\\t{:.4f}'.format(all_authors[idx],total_scores[idx],apm_weights[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(pw_apm_weights)[::-1]\n",
    "for k in range(180):\n",
    "    idx = best_idx[k]\n",
    "    name = all_authors[idx]\n",
    "    if name + ' 0001' in name_idx:\n",
    "        print('.',end='')\n",
    "        #continue\n",
    "    print('{:30s}\\t{:.2f}\\t{:.2f}\\t{:.2f}'.format(name,total_scores[idx],pw_apm_weights[idx],apm_weights[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(pweff_apm_weights)[::-1]\n",
    "for k in range(180):\n",
    "    idx = best_idx[k]\n",
    "    name = all_authors[idx]\n",
    "    if name + ' 0001' in name_idx:\n",
    "        print('.',end='')\n",
    "        #continue\n",
    "    print('{:30s}\\t{:.2f}\\t{:.2f}\\t{:.2f}'.format(name,total_scores[idx],pweff_apm_weights[idx],apm_weights[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_idx = np.argsort(pwunk_apm_weights)[::-1]\n",
    "for k in range(180):\n",
    "    idx = best_idx[k]\n",
    "    name = all_authors[idx]\n",
    "    if name + ' 0001' in name_idx:\n",
    "        print('.',end='')\n",
    "        #continue\n",
    "    print('{:30s}\\t{:.2f}\\t{:.2f}\\t{:.2f}'.format(name,total_scores[idx],pwunk_apm_weights[idx],apm_weights[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(pwunk_apm_weights[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(pweff_apm_weights[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(pw_apm_weights[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(napm_weights[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(apm_weights[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_scores = sorted([(total_scores[name_idx[name]],name) for name in uni_names if name in name_idx],reverse=True)\n",
    "uni_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('napm'+str(FI),napm_weights)\n",
    "np.save('pwapm'+str(FI),pw_apm_weights)\n",
    "np.save('pweffapm'+str(FI),pweff_apm_weights)\n",
    "np.save('pwunkapm'+str(FI),pwunk_apm_weights)\n",
    "np.save('apm'+str(FI),apm_weights)\n",
    "np.save('total',total_scores)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}