cleaned_venues_to_weights.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# configure weight generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import fnmatch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import gzip\n",
    "import pickle\n",
    "import csv\n",
    "import scipy.sparse\n",
    "Xauth = None\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the update to work despite the broken scipy documentation\n",
    "try:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a.update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix.update\n",
    "except:\n",
    "    a = scipy.sparse.dok_matrix((10,10))\n",
    "    a._update({(0,0):1.0})\n",
    "    scipy.sparse.dok_matrix.my_update = scipy.sparse.dok_matrix._update\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_to_get_idx = int(os.environ.get('REGRESSION_TASK_IDX',0))\n",
    "NORM_VOLUME = float(os.environ.get('REGRESSION_SIZE_NORM',2))\n",
    "NORM_CONF_NUM = bool(int(os.environ.get('REGRESSION_NORM_CONF_NUM',1)))\n",
    "\n",
    "be_verbose = not ('REGRESSION_TASK_IDX' in os.environ)\n",
    "\n",
    "\n",
    "WEIGHT_TO_GET = ['faculty','nsfmarginal','nsftotal','salary'][weight_to_get_idx]\n",
    "USE_LOG = False\n",
    "if weight_to_get_idx == 1:\n",
    "    USE_LOG = True\n",
    "TOP_K = 75\n",
    "SGD_ITER = 300\n",
    "YEAR_BLOCKS = 2 # 1 uses a by-year model\n",
    "BY_YEAR_SIGMA = {0: 2, 1: 2, 2: 4, 3:4}[weight_to_get_idx] # how many years to splat the by-year model\n",
    "weight_file_template = 'weights_{}_above6_{}_{}_{}_{}_{}.pkl'\n",
    "L2REG = 3e-3\n",
    "LRPRINT = -int(np.log10(L2REG)*10)\n",
    "NORM_YEARS = False\n",
    "if WEIGHT_TO_GET != 'faculty':\n",
    "    TOP_K = 0\n",
    "\n",
    "# get a new filename\n",
    "for i in range(50):\n",
    "    tmp = weight_file_template.format(WEIGHT_TO_GET,'log' if USE_LOG else 'linear',YEAR_BLOCKS,TOP_K,LRPRINT,i)\n",
    "    if not os.path.exists(tmp):\n",
    "        break\n",
    "# overwrite in the case of command line executation\n",
    "if 'REGRESSION_TASK_IDX' in os.environ:\n",
    "    i = 0\n",
    "weight_file = weight_file_template.format(WEIGHT_TO_GET,'log' if USE_LOG else 'linear',YEAR_BLOCKS,TOP_K,LRPRINT,i)\n",
    "weight_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open('useful_venue_list.pkl.gz','rb') as fp:\n",
    "    all_venues = pickle.load(fp)\n",
    "with gzip.open('useful_authors_list.pkl.gz','rb') as fp:\n",
    "    all_authors = pickle.load(fp)\n",
    "with gzip.open('useful_papers.pkl.gz','rb') as fp:\n",
    "    all_papers = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_idx = {v:i for i,v in enumerate(all_venues)}\n",
    "name_idx = {v:i for i,v in enumerate(all_authors)}\n",
    "n_confs = len(all_venues)\n",
    "n_auths = len(all_authors)\n",
    "r1_confs = pickle.load(open('old_version/r1_confs.pkl','rb'))\n",
    "r1_confs_dict = {_:1 for _ in r1_confs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faculty_affil = pd.read_csv('faculty-affiliations.csv')\n",
    "ranks = pd.read_csv('other_ranks/ranks.csv')\n",
    "def csv2dict_str_str(fname):\n",
    "    with open(fname, mode='r') as infile:\n",
    "        rdr = csv.reader(infile)\n",
    "        d = {rows[0].strip(): rows[1].strip() for rows in rdr}\n",
    "    return d\n",
    "aliasdict = csv2dict_str_str('dblp-aliases-expanded.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'nsf' in WEIGHT_TO_GET :\n",
    "    df_nsf = pd.read_pickle('nsf2.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,a in enumerate(all_authors):\n",
    "    #ns = a.split(' ')\n",
    "    #n_s = ns[0] + ' ' + ns[-1]\n",
    "\n",
    "    #if not (ns[0] + ns[-1]).isalpha():\n",
    "    #    continue\n",
    "    #if n_s not in name_idx:\n",
    "    #    name_idx[n_s] = name_idx[a]\n",
    "    # this version is better but maybe worse?\n",
    "    split_name = a.split(' ')\n",
    "    if not split_name[-1].isalpha() and len(split_name) > 2:\n",
    "        first_last = split_name[0] +' ' + split_name[-2]\n",
    "    else: \n",
    "        first_last = split_name[0] +' ' + split_name[-1]\n",
    "    if first_last not in name_idx:\n",
    "        name_idx[first_last] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numbered_names = set([_ for _ in list(all_authors) if _.split(' ')[-1].isdigit()]) #and int(_.split(' ')[-1][-1]) > 1])\n",
    "ambi_numbers = [' '.join(_.split(' ')[:-1]) for _ in numbered_names]\n",
    "for name in numbered_names:\n",
    "    ns = name.split(' ')\n",
    "    ambi_numbers.append(ns[0] + ' ' + ns[-2])\n",
    "#ambi_numbers = numbered_names\n",
    "\n",
    "clobber_names = {}\n",
    "for name in all_authors:\n",
    "    clobber_names[name] = 1 + clobber_names.get(name,0)\n",
    "for name in all_authors:\n",
    "    ns = name.split(' ')\n",
    "    if ns[-1].isdigit():\n",
    "        n2 = ' '.join(ns[:-1])\n",
    "        clobber_names[n2] = 1 + clobber_names.get(n2,0)\n",
    "        if len(ns) > 3:\n",
    "            n2 = ns[0] + ' ' + ns[-2]\n",
    "            clobber_names[n2] = 1 + clobber_names.get(n2,0)\n",
    "    else:\n",
    "        if len(ns) > 2:\n",
    "            n2 = ns[0] + ' ' + ns[-1]\n",
    "            clobber_names[n2] = 1 + clobber_names.get(n2,0)\n",
    "\n",
    "clobbered_names = [k for k,v in clobber_names.items() if v > 1]\n",
    "ambi_numbers = set(ambi_numbers+clobbered_names)\n",
    "len(clobbered_names),len(ambi_numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ambiguous_matches = list(ambi_numbers)\n",
    "ambiguous_matches += [ 'John Anderson','George Lakoff','Barbara White','William Bechtel', 'Hooman Darabi',  'Ermanno Bencivenga','George Lakoff','Robert Martin','David Kaplan','Seth Yalcin','Giacomo Bonanno','Adam Sennet','John Strain','Ohyun Kwon','Thomas Schwartz','Wendy Fong','Steven Lehman','Jeffrey Hoyt','Michael Katz','Gregory J. O. Beran','Kenneth Leung','Won Kim','Michael Ball','William Evans','Zheng Sun','John Kim','Tao Wang','William Smith','Richard Wallace','David Cohen','Wei Liu','Wei Wei','Bo Liu','Yuan Xie','Bo Li','Song-Chun Zhu','Vladimir Chernov','Richard Wilson','Gang Li','Jun Liu','Li Tian','Li Li','Ran Wei','Lin Zhang','Feng Wang','Wei Wang','Xi Zhang','Lei Wang','Yu Zhang','Jun Li','Gang Liu','Dong Wang','Gang Zeng','Yue Zhang','Jing Huang','Wei Chen','Yue Wang','Jing Liu','Han Li','Xi Chen','Yang Yang','Xiangmin Xu','Rong Wang','Xin Liu','Lei He','Feng Guo','Yan Zhang','Wei Xiong','Zheng Zhang','Fei Xu','Rui Wang','Yuan Zhang','Yu Zhou','Jing Yang','Li Zhang','Ke Zhang','Jiang Du','Kwang-Ting Cheng','Kai Liu','Hong Liu','Xia Li','Zhu Wang','Xiang Zhang','Liming Zhang','Qi Zhang','Feng Liu','Xu Chen','Bingyuan Liu','Xiaohua Huang','Yu Qiao','Qing Zhao','Chao Wang','Bin Chen','Pei Chen','Xu Yang','Yue Liu','Aditi Chandra','Yang Liu','Dong Li','Chen Li','Zhenbiao Yang','Yi Zhang','Tao Jiang','Jian Zhang']\n",
    "ambiguous_matches += ['Curtis Roads','David Wessel','Kaija Saariaho','Stephen Morris','George Taylor','William Miller','Behzad Razavi','Yu Hu','Ning Li','Gert Cauwenberghs','Ying Wei','Payam Heydari','Ping Liang','Babak Daneshrad','Zhiying Wang','Alexander Vardy','Wenjun Zhang','Rajarshi Mukherjee','Sangho Shin','Majid Sarrafzadeh','Upamanyu Madhow','Hao Chen','Edward Lee','Yi Chen','Yong Kim','Michael Cheng','Xiaolong Li','Kenneth Rose','Ke Xu','Chen-Nee Chuah','Wentai Liu','Qiang Zhou','Jiong Li','Hadi Esmaeilzadeh']\n",
    "ambiguous_matches += ['William White','Robert Rosenthal','Daniel Schneider','Min Zhao','Mark Wilson']\n",
    "#ambiguous_matches += ['Ming Liu','Shantanu Sinha','Jishen Zhao','Yanhong Li','Lifeng Lai','Young-Han Kim','Somayeh Sojoudi','Shuguang Cui','Mahnoosh Alizadeh','Kannan Ramchandran','Tara Javidi','Anant Sahai','Steven Weber','Shaolei Ren','Jing Xu','Lara Dolecek','Kamalika Chaudhuri','Massimo Franceschetti','Ramtin Pedarsani','Bin Yu','Francesco Bullo','David Tse','Hao Li','Benjamin Recht','Marco Levorato','Duncan Callaway','Lior Pachter','Paulo Tabuada','Timothy Brown','Joseph Wang','Fabio Pasqualetti','Sharon Aviran','Kang Zhang','Jing Wang','Michael Bell','Anil Aswani','George Varghese','Francesco Borrelli','Phong Nguyen','Athina Markopoulou','Hamid Jafarkhani','Po-Ning Chen','Wotao Yin','Khoa Nguyen','Shu Lin','Kameshwar Poolla','Yueyue Fan','Dipak Ghosal','Abhay Parekh']\n",
    "ambiguous_matches +=['Zhe Chen','Yizhou Sun','Yanhong Liu','Shachar Lovett','Rina Dechter','Borivoje Nikolic','Krste Asanovic','Elad Alon','Ying Hu','Ye Li','Xin Chen',]\n",
    "ambiguous_matches +=['Steven Lee','Wen Jiang','Yu Huang','John Campbell','Wei Zhou','Gang Chen','Shivendu Shivendu','Vijay Khatri','Yi Sun','Yong Chen','Bin Liu','Mark Anderson','Erik Rolland','Jing Zhao','Li Fan','Yi Xie','Li Cai','Jia Shen','Lili Yang','Tong Wang','John Campbell','Wei Zhou','Gang Chen','Lixia Zhang','Sujit Dey','Jiasi Chen','Jin Zhang','Lin Liu','Mary Hegarty',]\n",
    "ambiguous_matches += ['Xin Zhou','Min Li','Huiying Li','Dong Yu','Yong Huang','Jing Shi','Zhiwei Zhang','Xiao Hu']\n",
    "ambiguous_matches += ['Joseph Barton','Jason Woo','Evelyn S. Tecoma','Bei Wang']\n",
    "ambiguous_matches += ['Bin Yu','Yi Tang','Jing Wang','Wei Ren']\n",
    "ambiguous_matches += ['Ming Gu','Wei Xu','Bo Yu','Tao Yang','Lin Lin''Zhi Ding','Tao Ye','Kai Zhu','Steven Weber']\n",
    "ambiguous_matches += ['Muhammad Arif','Ke Li','Ming Liu','Jiawei Chen','Hao Li','Yang Xu','Xin Guo','Hao Cheng','Ye Zhang']\n",
    "ambiguous_matches += ['David Pearce','Ilya Dumer','Richard Bamler','Pablo Tamayo','Pamela Samuelson','Michael Pratt','Bruce Blumberg','Patrick Farrell','Albert Lai','Phong Nguyen','Michael Levine','Marco Conti','Oliver Arnold','Deepak Gupta','Chun-Nan Hsu','Jun Wu','Hui Sun','Michael Franklin','Richard Allen','Ida Sim']\n",
    "ambiguous_matches += ['Michael Rios','Joel Watson','Lara Buchak','Michael Weiner','Venkatesan Sundaresan','Hoori Ajami','Neil Jones','Yi-Lin Yang','Adrian Preda','Jonathan Wurtele','Kumar Sharma','Phioanh Nghiemphu','Mark Asta','Lei Song','Boris Maciejovsky','Nader Pourmand','Fang Wei','Jie Zheng','Shane White','Stephen Small','Xiaobin Yang','Steven Evans','Jiming Jiang']\n",
    "ambiguous_matches += ['Robin Hill','Marcus Opp','Yimin Zou','Lera Boroditsky','Morana Alac','Diba Mirza','Volkan Rodoplu','Michael Ryan','Grace Chang']\n",
    "\n",
    "#globecom + icc\n",
    "ambiguous_matches += ['Izhak Rubin','Zhi Ding','Shuguang Cui','Albert Chan','Lei Cao','Robert Cohen','Ming Xiao','Hyong Kim','Massimo Tornatore','Bill Lin','Daniel Lee',\"Daniel O'Neill\",'Lin Tian','Go Hasegawa','Meng Chen','Haitao Zheng','Jing Xu','Biao He','Bo Wei','Eric Wong','Lin Lin','Xu Wang','Tao Chen','Hong Zhou']\n",
    "\n",
    "# still not easy\n",
    "ambiguous_matches += ['Wu Li','Thomas Strohmer','Alfred Kobsa','Itay Neeman','Jonathan Furner','Jinyi Qi','Peng Ding','Alexandru Nicolau','Itay Neeman','Kang Zhang','Ilan Adler',\"Barry O'Neill\",'Gideon Weiss']\n",
    "\n",
    "# the pagerank gods\n",
    "ambiguous_matches += ['Wei Zhang','Wei Li','Lei Wang','Jing Li','Yang Liu','Yu Zhang','Lei Zhang','Jun Wang','Li Zhang','Jing Wang','Xin Wang','Hai Jin','Hui Li','Jian Wang','Yan Li','Jing Zhang','Wen Gao','Li Li','Wei Chen','Wei Liu','Yang Li','Yan Zhang','Yang Yang','Jun Zhang','Yong Wang','Xin Li','Yan Wang']\n",
    "\n",
    "ambiguous_matches = set(ambiguous_matches)\n",
    "if WEIGHT_TO_GET == 'salary':\n",
    "\n",
    "    uc_profs = faculty_affil[faculty_affil.affiliation.str.contains('University of California')]\n",
    "    # salary data\n",
    "    dt = {'Employee Name': str,\n",
    "    'Job Title': str,\n",
    "    'Base Pay': float,\n",
    "    'Overtime Pay': float,\n",
    "    'Other Pay': float,\n",
    "    'Benefits': float,\n",
    "    'Total Pay': float,\n",
    "    'Total Pay & Benefits': float,\n",
    "    'Year': float,\n",
    "    'Notes': str,\n",
    "    'Agency': str,\n",
    "    'Status': str}\n",
    "    na_values = [ 'Aggregate','#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN', '-NaN', '-nan', '1.#IND', '1.#QNAN', 'N/A', 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null']\n",
    "    dfs = [pd.read_csv('./download/university-of-california-{}.csv'.format(i),dtype=dt,na_values=na_values) for i in range(2015,2022)]\n",
    "    print('fac_size',[_.shape for _ in dfs],sum([_.shape[0] for _ in dfs]))\n",
    "    dfs = [_[_['Job Title'].str.contains('PROF')] for _ in dfs]\n",
    "    #dfs = [_[_['Job Title'].str.contains('B/E/E')] for _ in dfs]\n",
    "    dfs = [_[['Employee Name','Total Pay & Benefits']] for _ in dfs]\n",
    "    dfs = [_.reset_index(drop=True) for _ in dfs]\n",
    "    [_.shape for _ in dfs],sum([_.shape[0] for _ in dfs])\n",
    "    from collections import defaultdict\n",
    "    ca_pay = defaultdict(int)\n",
    "    for df in dfs:\n",
    "        df = df.fillna(0)\n",
    "        for row in df.itertuples():\n",
    "            ca_pay[row[1]] = max(ca_pay[row[1]],row[2])\n",
    "            \n",
    "    \n",
    "    keys = list(ca_pay.keys())\n",
    "    ca_pay_prof = {}\n",
    "    for name in keys:\n",
    "        name_s = name.split(' ')\n",
    "        if name in ambiguous_matches:\n",
    "            continue\n",
    "        if name in name_idx:\n",
    "            n = name\n",
    "        elif name_s[0] + name_s[-1] in name_idx:\n",
    "            n = name_s[0] + name_s[-1] \n",
    "        else:\n",
    "            continue\n",
    "        #if n in faculty_affil.name.str.lower():\n",
    "        ca_pay_prof[name] = (name_idx[n] ,ca_pay[name])\n",
    "\n",
    "        #faculty_affil\n",
    "    #print(len(ca_pay_prof))\n",
    "    #print(ca_pay_prof['Kristofer Pister'],ca_pay_prof['Pieter Abbeel'],ca_pay_prof['Sergey Levine'],ca_pay_prof['Jitendra Malik'],len(ca_pay_prof))\n",
    "    ca_prof_n = len(ca_pay_prof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[df.shape for df in dfs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if WEIGHT_TO_GET == 'salary':\n",
    "    uc_prof_dict = {k:1 for k in uc_profs.name}\n",
    "    i = 0\n",
    "    #ca_pay_prof = {k:v for k,v in ca_pay_prof.items() if k in uc_prof_dict}\n",
    "    #\n",
    "    ca_prof_n = len(ca_pay_prof)\n",
    "    print('faculty found:',ca_prof_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    # salary data\n",
    "    dt = {'Employee Name': str,\n",
    "    'Job Title': str,\n",
    "    'Base Pay': float,\n",
    "    'Overtime Pay': float,\n",
    "    'Other Pay': float,\n",
    "    'Benefits': float,\n",
    "    'Total Pay': float,\n",
    "    'Total Pay & Benefits': float,\n",
    "    'Year': float,\n",
    "    'Notes': str,\n",
    "    'Agency': str,\n",
    "    'Status': str}\n",
    "    na_values = [ 'Aggregate','#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN', '-NaN', '-nan', '1.#IND', '1.#QNAN', 'N/A', 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null']\n",
    "    dfs = [pd.read_csv('downloads/university-of-california-{}.csv'.format(i),dtype=dt,na_values=na_values) for i in range(2015,2020)]\n",
    "    [_.shape for _ in dfs],sum([_.shape[0] for _ in dfs])\n",
    "    dfs = [_[_['Job Title'].str.contains('PROF')] for _ in dfs]\n",
    "    dfs = [_[_['Job Title'].str.contains('B/E/E')] for _ in dfs]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfs[0][dfs[0]['Total Pay & Benefits'] > 1e6]\n",
    "#dfs[0][dfs[0]['Employee Name'] == 'Stefano Soatto']\n",
    "#ASSOC PROF-AY-B/E/E #PROF-AY-B/E/E #PROF-AY-B/E/E PROF-AY-B/E/E \t73921.0\n",
    "#PROF-AY  B/E/E\n",
    "# dfs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# generate years and authorship matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_year = all_papers[0][6]\n",
    "max_year = all_papers[-1][6]\n",
    "span_years = max_year - min_year + 1\n",
    "\n",
    "\n",
    "if YEAR_BLOCKS!=0:\n",
    "    offset_years = [i//YEAR_BLOCKS for i in range(span_years)]\n",
    "    year_ind = max(offset_years)+1\n",
    "    year_span_printable = {}\n",
    "    for i in range(year_ind):\n",
    "        start_year = offset_years.index(i) + min_year\n",
    "        end_year = len(offset_years) - 1 - offset_years[::-1].index(i) + min_year\n",
    "        year_span_printable[i] = str(start_year)[-2:] +'t' + str(end_year)[-2:]\n",
    "    list(year_span_printable.values())\n",
    "years_per_conf = year_ind if BY_YEAR_SIGMA != 0 else span_years\n",
    "\n",
    "if BY_YEAR_SIGMA != 0:\n",
    "    import scipy.stats\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.style.use('fivethirtyeight')\n",
    "    plt.style.use('default')\n",
    "\n",
    "    weights = []\n",
    "    for i in range(years_per_conf):\n",
    "        a = np.array([scipy.stats.norm.pdf( (j-i)/BY_YEAR_SIGMA) for j in range(years_per_conf)])\n",
    "        a[a < 0.05] = 0\n",
    "        weights.append(a/np.linalg.norm(a))\n",
    "    plt.figure(figsize=(6,3))\n",
    "    plt.subplot(1,2,1)\n",
    "    _ = plt.plot(YEAR_BLOCKS*np.arange(years_per_conf)+min_year,weights[(2000-min_year)//YEAR_BLOCKS],lw=4)\n",
    "    plt.xlim(1970,2020)\n",
    "\n",
    "    plt.xticks(np.arange(1970,2021,10),[str(_) for _ in np.arange(1970,2021,10)])\n",
    "    plt.grid(True)\n",
    "    plt.tight_layout()\n",
    "\n",
    "    plt.subplot(1,2,2)\n",
    "    _ = plt.plot(YEAR_BLOCKS*np.arange(years_per_conf)+min_year,weights[(2018-min_year)//YEAR_BLOCKS],lw=4)\n",
    "    plt.xlim(1970,2020)\n",
    "    plt.xticks(np.arange(1970,2021,10),[str(_) for _ in np.arange(1970,2021,10)])\n",
    "    plt.grid(True)\n",
    "    plt.tight_layout()\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('tgauss.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    with open('blacklist.pkl','rb') as fp:\n",
    "        BLACKLIST = pickle.load(fp)\n",
    "    BLACKLIST = set(BLACKLIST)\n",
    "    len(BLACKLIST)\n",
    "    import random\n",
    "    CONFS_TO_SPLIT = set(['ICRA','CDC','CVPR','NIPS','AAAI','CHI','ICML','IJCAI','CIKM'])\n",
    "    SPLIT_SIZES = [0.5,0.25,0.13,0.12]\n",
    "    SPLIT_CONST = []\n",
    "    i =0 \n",
    "    for s in SPLIT_SIZES:\n",
    "        i+=s\n",
    "        SPLIT_CONST.append(i)\n",
    "    LOOKUP = {}\n",
    "    for p in zip(BLACKLIST,itertools.product(CONFS_TO_SPLIT,SPLIT_SIZES)):\n",
    "        k = p[1][0] + '_' + str(p[1][1])\n",
    "        #print(p,k)\n",
    "\n",
    "        LOOKUP[k] = p\n",
    "    NORM_VOLUME = np.sqrt(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_of_papers = np.zeros(years_per_conf*n_confs)\n",
    "paper_tmp=[]\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    yr = (year-min_year)//YEAR_BLOCKS\n",
    "    \n",
    "    #if venue in BLACKLIST:\n",
    "    #    continue\n",
    "\n",
    "    #if venue in CONFS_TO_SPLIT:\n",
    "    #    r = random.random()\n",
    "    #    for i,t in enumerate(SPLIT_CONST):\n",
    "    #        if r < t:\n",
    "    #            break\n",
    "    #    key =   venue + '_' + str(SPLIT_SIZES[i])\n",
    "    #   venue = LOOKUP[key][0]\n",
    "    \n",
    "    j = years_per_conf*conf_idx[venue] + yr\n",
    "    count_of_papers[j] += 1\n",
    "    if year == 2018 and venue == 'IEEE Access':\n",
    "        paper_tmp.append(paper)\n",
    "# safe divide\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = plt.hist(np.power(count_of_papers[np.where(count_of_papers > 0)],1/1.618),300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for maxi in np.argsort(count_of_papers)[::-1][:5]:\n",
    "#maxi= np.argmax(count_of_papers)\n",
    "    #print(count_of_papers[maxi],all_venues[maxi//years_per_conf],(maxi%years_per_conf)*YEAR_BLOCKS + 1970,len(paper_tmp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_reshape = count_of_papers.reshape((-1,years_per_conf))\n",
    "number_of_confs_per_year = np.minimum(tmp_reshape,1).sum(0)\n",
    "plt.plot(np.arange(min_year,max_year,YEAR_BLOCKS),number_of_confs_per_year)\n",
    "plt.grid(True)\n",
    "plt.figure()\n",
    "plt.plot(np.arange(min_year,max_year,YEAR_BLOCKS),tmp_reshape.sum(0))\n",
    "plt.figure()\n",
    "plt.plot(np.arange(min_year,max_year,YEAR_BLOCKS),tmp_reshape.sum(0)/number_of_confs_per_year)\n",
    "\n",
    "if NORM_CONF_NUM ==  False:\n",
    "    number_of_confs_per_year = np.ones_like(number_of_confs_per_year)\n",
    "confs_norm_vector = 1.0/number_of_confs_per_year\n",
    "confs_norm_vector /= confs_norm_vector.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "papers_exist= count_of_papers.copy()\n",
    "papers_exist[np.where(papers_exist > 0)] = 1\n",
    "\n",
    "count_of_papers = np.maximum(1,count_of_papers)\n",
    "if NORM_VOLUME == 0:\n",
    "    count_of_papers = np.ones(years_per_conf*n_confs)\n",
    "elif NORM_VOLUME < 0:\n",
    "    count_of_papers = np.log(count_of_papers+1)\n",
    "else:\n",
    "    count_of_papers = count_of_papers ** (1/NORM_VOLUME)\n",
    "count_of_papers /= count_of_papers.mean()\n",
    "\n",
    "papers_exist.sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if BY_YEAR_SIGMA != 0:\n",
    "    import itertools\n",
    "    #pairs_of_years = itertools.product(range(span_years),range(span_years))\n",
    "\n",
    "    wdict = {}\n",
    "    for i,j,k in itertools.product(range(n_confs),range(years_per_conf),range(years_per_conf)):\n",
    "        wdict[i*years_per_conf+j,i*years_per_conf+k] = weights[j][k]\n",
    "    wsa = scipy.sparse.dok_matrix((years_per_conf*n_confs,years_per_conf*n_confs))\n",
    "    wsa.my_update(wdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#wsa.sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dd():\n",
    "    return defaultdict(list)\n",
    "if 'nsf' in WEIGHT_TO_GET or 'salary' in WEIGHT_TO_GET:\n",
    "    from unidecode import unidecode\n",
    "    # what papers everyone (in first/last name sense) published in every year\n",
    "    author_papers = defaultdict(dd)\n",
    "\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        \n",
    "        for a in authors:\n",
    "            a = unidecode(a)\n",
    "            split_name = a.split(' ')\n",
    "            #first_last = split_name[0] +' ' + split_name[-1]\n",
    "            #author_papers[first_last][year].append((venue,n))\n",
    "            if not split_name[-1].isalpha() and len(split_name) > 2:\n",
    "                first_last = split_name[0] +' ' + split_name[-2]\n",
    "            else: \n",
    "                first_last = split_name[0] +' ' + split_name[-1]\n",
    "            if first_last in ambiguous_matches:\n",
    "                continue\n",
    "            author_papers[first_last.lower()][year].append((venue,n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ddn():\n",
    "    return defaultdict(int)\n",
    "if 'nsf' in WEIGHT_TO_GET:\n",
    "    nsf_n_idx = list(df_nsf.columns).index('name') +1\n",
    "    nsf_y_idx = list(df_nsf.columns).index('year')+ 1\n",
    "    nsf_a_idx = list(df_nsf.columns).index('infaward') +1 \n",
    "\n",
    "    nsf_paper_n = 0\n",
    "    # total amount of NSF funding recieved by a person up to a given year\n",
    "    author_amounts = defaultdict(ddn)\n",
    "    for i,row in enumerate(df_nsf.itertuples()):\n",
    "        authors, year, amount = row[nsf_n_idx],row[nsf_y_idx],row[nsf_a_idx]\n",
    "        n = len(authors)\n",
    "        # some infinite amounts exist! bad!\n",
    "        if not np.isfinite(amount):\n",
    "            continue\n",
    "        # what is this even?\n",
    "        if amount < 1000: \n",
    "            continue\n",
    "        amount = amount# min(amount,1e7)\n",
    "        for a in authors:\n",
    "            a = aliasdict.get(a,a)\n",
    "            split_name = a.split(' ')\n",
    "            first_last = split_name[0] +' ' + split_name[-1]\n",
    "            for yr in range(int(year),max_year+1):\n",
    "                author_amounts[first_last.lower()][yr] += amount/n\n",
    "        nsf_paper_n += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth_years = np.ones((n_auths,2)) * np.array([3000,1000]) \n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    for a in authors:\n",
    "        i = name_idx[a]\n",
    "        auth_years[i,0] = min(auth_years[i,0],year)\n",
    "        auth_years[i,1] = max(auth_years[i,1],year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if WEIGHT_TO_GET == 'faculty':\n",
    "    count_vecs = {}\n",
    "    paper_vecs = []\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "\n",
    "        #if venue in BLACKLIST:\n",
    "        #    continue\n",
    "        #    \n",
    "        #if venue in CONFS_TO_SPLIT:\n",
    "        #    r = random.random()\n",
    "        #    for i,t in enumerate(SPLIT_CONST):\n",
    "        #        if r < t:\n",
    "        #            break\n",
    "        #    key =   venue + '_' + str(SPLIT_SIZES[i])\n",
    "        #    venue = LOOKUP[key][0]\n",
    "        \n",
    "        n = len(authors)\n",
    "        j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "        \n",
    "        \n",
    "        if n not in count_vecs:\n",
    "            author_scores = 1/(np.arange(n)+1) # I guess it's the thing to do is \n",
    "            #author_scores[-1] = author_scores[0]\n",
    "            author_score_sum = author_scores.sum()\n",
    "            #author_scores /= author_score_sum\n",
    "            count_vecs[n] = author_scores / author_score_sum\n",
    "        author_scores = count_vecs[n]\n",
    "        paper_vecs.append([(name_idx[a],j,v) for a,v in zip(authors,author_scores)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if WEIGHT_TO_GET == 'faculty':\n",
    "    import scipy.sparse\n",
    "\n",
    "    Xauth = scipy.sparse.dok_matrix((n_auths,years_per_conf*n_confs))\n",
    "    xdict = {}\n",
    "    if False:\n",
    "        for paper in all_papers:\n",
    "            tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "            n = len(authors)\n",
    "            j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "            for ai in range(n):#zip(count_vecs[n],authors):\n",
    "                i = name_idx[authors[ai]]\n",
    "                xdict[(i,j)] = (1/n) + xdict.get((i,j),0)\n",
    "                #xdict[(i,j)] = count_vecs[n][ai] + xdict.get((i,j),0)\n",
    "    else:\n",
    "        for paper_vec in paper_vecs:\n",
    "            for i,j,v in paper_vec:\n",
    "                xdict[(i,j)] = confs_norm_vector[j%years_per_conf] * v/count_of_papers[j] + xdict.get((i,j),0) #\n",
    "\n",
    "    Xauth.my_update(xdict)\n",
    "\n",
    "    Xauth = scipy.sparse.csr_matrix(Xauth)\n",
    "    \n",
    "    \n",
    "    Xreg = scipy.sparse.csr_matrix.copy(Xauth)\n",
    "    #print(Xauth.shape,Xreg.shape)\n",
    "elif 'nsf' in WEIGHT_TO_GET:\n",
    "    # create design mattrix\n",
    "    nsf_paper_n = df_nsf.shape[0]\n",
    "    Xreg = scipy.sparse.dok_matrix((df_nsf.shape[0],years_per_conf*n_confs))\n",
    "    xdict = {}\n",
    "    y = np.zeros(nsf_paper_n,dtype=np.float32)\n",
    "    for i,row in enumerate(df_nsf.itertuples()):\n",
    "        authors, year, amount = row[nsf_n_idx],row[nsf_y_idx],row[nsf_a_idx]\n",
    "        # some infinite amounts exist! bad!\n",
    "        if not np.isfinite(amount):\n",
    "            continue\n",
    "        # what is this even?\n",
    "        if amount < 1000: \n",
    "            continue\n",
    "        for a in authors:\n",
    "            a = aliasdict.get(a,a)\n",
    "            split_name = a.split(' ')\n",
    "            first_last = split_name[0] +' ' + split_name[-1]\n",
    "            for year_a,conf_list in author_papers[first_last.lower()].items():\n",
    "                if year_a <= year:\n",
    "                    for paper in conf_list:\n",
    "                        j = years_per_conf*conf_idx[paper[0]] + (year_a-min_year)//YEAR_BLOCKS\n",
    "                        xdict[(i,j)] = confs_norm_vector[j%years_per_conf] *(1/paper[1])/count_of_papers[j]\n",
    "\n",
    "    Xreg.my_update(xdict)\n",
    "    \n",
    "    #print(Xreg.sum())\n",
    "elif 'salary' == WEIGHT_TO_GET:\n",
    "    Xreg = scipy.sparse.dok_matrix((ca_prof_n,years_per_conf*n_confs))\n",
    "    xdict = {}\n",
    "    y_unique_confs = {}\n",
    "    y = np.zeros(ca_prof_n,dtype=np.float32)\n",
    "    y_paper = np.zeros(ca_prof_n,dtype=np.float32)\n",
    "    for idx,d in enumerate(ca_pay_prof.items()):\n",
    "        k,v = d\n",
    "        a = all_authors[v[0]]\n",
    "        y[idx] = v[1]\n",
    "        sum_paper = 0 \n",
    "        for year_a,conf_list in author_papers[a.lower()].items():\n",
    "            for paper in conf_list:\n",
    "                #if paper[0] not in r1_confs_dict:\n",
    "                #    continue\n",
    "                j = years_per_conf*conf_idx[paper[0]] + (year_a-min_year)//YEAR_BLOCKS\n",
    "                xdict[(idx,j)] = confs_norm_vector[j%years_per_conf]*(1/paper[1])/count_of_papers[j]\n",
    "                sum_paper += 1.0/paper[1]\n",
    "                new_set = y_unique_confs.get(idx,set())\n",
    "                new_set.add(paper[0])\n",
    "                y_unique_confs[idx] = new_set\n",
    "        y_paper[idx] = sum_paper\n",
    "    Xreg.my_update(xdict)\n",
    "    y_orig = np.copy(y)\n",
    "    print(Xreg.sum())\n",
    "    y_unique_confs_vec = np.zeros(ca_prof_n,dtype=np.float32)\n",
    "    for i in range(ca_prof_n):\n",
    "        y_unique_confs_vec[i] = len(y_unique_confs.get(i,set()))\n",
    "print('Design matrix has shape',Xreg.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scaling_confs = np.nan_to_num(conf_sums.reshape((-1,years_per_conf)).sum(1)/new_conf_sums.reshape((-1,years_per_conf)).sum(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if BY_YEAR_SIGMA != 0:\n",
    "    # convert matrix\n",
    "    Xreg = scipy.sparse.csr_matrix(Xreg)\n",
    "    wsa = scipy.sparse.csr_matrix(wsa)\n",
    "\n",
    "    # get sums\n",
    "    conf_sums = np.array(Xreg.sum(0))\n",
    "    # get splat\n",
    "    Xreg = Xreg @ wsa\n",
    "    if True: # try to handle non-existing years correctly\n",
    "        # clear 0s\n",
    "        clear_emptys = scipy.sparse.diags(papers_exist)\n",
    "        Xreg = Xreg @ clear_emptys\n",
    "        # get normalize\n",
    "        new_conf_sums = np.array(Xreg.sum(0))\n",
    "        scaling_confs = np.nan_to_num(conf_sums.reshape((-1,years_per_conf)).sum(1)/new_conf_sums.reshape((-1,years_per_conf)).sum(1))\n",
    "        \n",
    "        norm_matrix = scipy.sparse.diags(np.repeat(scaling_confs,years_per_conf))\n",
    "        # normalize\n",
    "        Xreg = Xreg @ norm_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[_ for _ in all_venues if 'From Database to Cyb' in _]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confs_with_weird = ['GLOBECOM','ICC']\n",
    "if weight_to_get_idx == 3: # TEST:to find bad alias names\n",
    "    ca_prof_names = [v[0] for k,v in enumerate(ca_pay_prof.items())]\n",
    "    vector = np.zeros(Xreg.shape[0])\n",
    "    for conf_to_test in confs_with_weird:\n",
    "        #conf_to_test = 'ISPA/IUCC'\n",
    "        tmp_idx = years_per_conf*conf_idx[conf_to_test]\n",
    "        tmp_idx2 = years_per_conf*(conf_idx[conf_to_test]+1)\n",
    "        vector2 = np.array(Xreg[:,tmp_idx:tmp_idx2].todense())\n",
    "        vector += vector2.sum(1) \n",
    "        #for idx in np.argsort(vector)[::-1]:\n",
    "        #    print(ca_prof_names[idx],vector[idx])\n",
    "    #for idx in np.argsort(vector)[::-1]:\n",
    "        #if vector[idx] > 0:\n",
    "            #print(ca_prof_names[idx],vector[idx])\n",
    "            #print(\"'\"+ca_prof_names[idx]+\"'\",end=',')\n",
    "    #vector2[ca_prof_names.index('Jitendra Malik')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if WEIGHT_TO_GET == 'faculty':\n",
    "    y = np.zeros(n_auths)\n",
    "    for i in range(TOP_K): \n",
    "        uni_name = ranks.iloc[i]['uni']\n",
    "        uni_faculty = faculty_affil[faculty_affil.affiliation == uni_name]\n",
    "        uni_names = np.array(uni_faculty.name)\n",
    "        for name in set([aliasdict.get(n, n) for n in uni_names]):\n",
    "            if name in name_idx:\n",
    "                y[name_idx[name]] = 1\n",
    "elif WEIGHT_TO_GET == 'nsfmarginal':\n",
    "    year_amounts = np.zeros(span_years,dtype=np.float32)\n",
    "    y = np.zeros(nsf_paper_n,dtype=np.float32)\n",
    "\n",
    "    for i,row in enumerate(df_nsf.itertuples()):\n",
    "        authors, year, amount = row[nsf_n_idx],row[nsf_y_idx],row[nsf_a_idx]\n",
    "        \n",
    "        authors2 = []\n",
    "        for a in authors:\n",
    "            a = aliasdict.get(a,a)\n",
    "            split_name = a.split(' ')\n",
    "            first_last = split_name[0] +' ' + split_name[-1]\n",
    "            authors2.append(first_last)\n",
    "        authors = authors2\n",
    "        # some infinite amounts exist! bad!\n",
    "        if not np.isfinite(amount):\n",
    "            continue\n",
    "        if amount <= 20000: #what is that even for?\n",
    "            continue\n",
    "        # maybe the old years are misleading!?\n",
    "        #if year < 2002:\n",
    "        #    continue\n",
    "        # small grants are misleading? 150000\n",
    "        #if amount < 1e7:\n",
    "        #    continue\n",
    "        # giant grants are msileading?\n",
    "        #if amount >= 4e5:\n",
    "        #    amount = 4e5 + np.log((amount-4e5)+1)*4e3\n",
    "        if amount >= 1e7:\n",
    "            amount = 1e7 + np.log((amount-1e7)+1)*1e5\n",
    "        #print(len(authors),sum([(a in author_papers) for a in authors]))\n",
    "        #print(a)\n",
    "        #print(len(authors),sum([(a in author_papers) for a in authors]))\n",
    "        #print(a)\n",
    "        total_authors = len(authors)\n",
    "        needed_authors = 0.5 * total_authors # half of all authors\n",
    "        found_authors = sum([(a.lower() in author_papers) for a in authors])\n",
    "        if needed_authors > 0 and needed_authors <= found_authors:\n",
    "            y[i] = amount* (found_authors/total_authors)\n",
    "            #year_amounts[year-min_year] += amount\n",
    "elif WEIGHT_TO_GET == 'nsftotal':\n",
    "    for i,row in enumerate(df_nsf.itertuples()):\n",
    "        authors, year, amount = row[nsf_n_idx],row[nsf_y_idx],row[nsf_a_idx]\n",
    "        authors2 = []\n",
    "        for a in authors:\n",
    "            a = aliasdict.get(a,a)\n",
    "            split_name = a.split(' ')\n",
    "            first_last = split_name[0] +' ' + split_name[-1]\n",
    "            authors2.append(first_last)\n",
    "        authors = authors2\n",
    "        \n",
    "        # some infinite amounts exist! bad!\n",
    "        if not np.isfinite(amount):\n",
    "            continue\n",
    "\n",
    "        if amount < 10000: #50000\n",
    "            continue\n",
    "        total_authors = len(authors)\n",
    "        needed_authors = 0.5 * total_authors # half of all authors\n",
    "        found_authors = sum([(a.lower() in author_papers) for a in authors])\n",
    "        if needed_authors > 0 and needed_authors <= found_authors:\n",
    "            y[i] = sum([author_amounts[first_last.lower()][year] for first_last in authors])\n",
    "            #year_amounts[year-min_year] += sum([author_amounts[first_last.lower()][year] for first_last in authors])\n",
    "\n",
    "skipped_data = scipy.sparse.diags((y != 0).astype(float))\n",
    "y_orig = np.copy(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'nsf' in WEIGHT_TO_GET:\n",
    "    if USE_LOG: # do log\n",
    "        y = np.copy(np.log(1+y_orig))\n",
    "        #y[y == np.log(1)] = y[y != np.log(1)].mean()\n",
    "    else:\n",
    "        y = np.copy(y_orig)\n",
    "        #y[y == 0] = y[y != 0].mean()\n",
    "    from matplotlib.pyplot import figure,hist\n",
    "    hist((y-y.mean())/y.std(),100)\n",
    "    figure()\n",
    "    _ = hist(y,100)\n",
    "    #print(skipped_data.sum())\n",
    "if 'salary' in WEIGHT_TO_GET:\n",
    "    y = np.copy(y_orig)\n",
    "    skipped_data_vec =  np.ones_like(y)  *(y < 800000) * (y > 120000) * (y_paper >= 3.0) * (y_unique_confs_vec >= 3) #* (y_paper < 500) #* (y_orig > 50000)\n",
    "    #print(skipped_data_vec.sum())\n",
    "    skipped_data_vec = skipped_data_vec.astype(np.float32)\n",
    "    skipped_data = scipy.sparse.diags(skipped_data_vec)\n",
    "    y[skipped_data_vec == 0] = y[skipped_data_vec != 0].mean()\n",
    "    \n",
    "    if USE_LOG: # do log\n",
    "        y = np.copy(np.log(1+y))\n",
    "    else:\n",
    "        y = np.copy(y)\n",
    "    from matplotlib.pyplot import figure,hist\n",
    "    hist((y-y.mean())/y.std(),50)\n",
    "    figure()\n",
    "    \n",
    "    _ = hist(y[abs(y-y.mean()) > 3000],50)\n",
    "    #print(skipped_data_vec.sum(),)\n",
    "    print('faculty used ',skipped_data_vec.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if WEIGHT_TO_GET == 'faculty':\n",
    "    from sklearn.linear_model import SGDClassifier\n",
    "    if False: # old, allows only positive weights, uses elasticnet, etc.\n",
    "        Xreg = scipy.sparse.csr_matrix(Xreg)\n",
    "        #clf = SGDClassifier('modified_huber',average=False,verbose=1,warm_start=True,tol=1e-5,max_iter=1,alpha=1e-4,penalty='elasticnet',l1_ratio=0.9,epsilon=0.75)\n",
    "\n",
    "        clf.fit(Xreg,y)\n",
    "        for i in range(SGD_ITER):\n",
    "            minv = clf.coef_[clf.coef_ > 0].min()\n",
    "            maxv = clf.coef_[clf.coef_ > 0].max()\n",
    "            #clf.coef_ = np.maximum(minv,clf.coef_)\n",
    "            clf = clf.partial_fit(Xreg,y)\n",
    "        #minv = clf.coef_[clf.coef_ > 0].min()\n",
    "        #clf.coef_ = np.maximum(minv,clf.coef_)\n",
    "    else: #simple, maybe worse but simple\n",
    "        #clf = SGDClassifier('modified_huber',average=False,verbose=1,tol=1e-7,max_iter=SGD_ITER,alpha=1e-3)\n",
    "        clf = SGDClassifier('modified_huber',average=False,verbose=be_verbose,tol=1e-9,max_iter=SGD_ITER,alpha=L2REG,epsilon=0.01)\n",
    "        #y[y ==0] = -1\n",
    "        clf.fit(Xreg,y)\n",
    "if 'nsf' in WEIGHT_TO_GET:\n",
    "    from sklearn.linear_model import SGDRegressor\n",
    "\n",
    "    Xreg = scipy.sparse.csr_matrix(Xreg)\n",
    "    clf = SGDRegressor('huber',tol=1e-9,max_iter=3*SGD_ITER,penalty='l2',verbose=be_verbose,alpha=L2REG,epsilon=0.01)\n",
    "    #clf = SGDRegressor('huber',tol=1e-9,max_iter=100,verbose=1,penalty='l1',alpha=1e-7)\n",
    "\n",
    "    clf.fit(skipped_data@Xreg ,y)#(y-y.mean())/y.std()\n",
    "if 'salary' in WEIGHT_TO_GET:\n",
    "    from sklearn.linear_model import SGDRegressor\n",
    "\n",
    "    Xreg = scipy.sparse.csr_matrix(Xreg) \n",
    "    #clf = SGDRegressor('huber',tol=1e-9,max_iter=SGD_ITER*10,penalty='l2',verbose=False,alpha=L2REG,epsilon=0.01,average=True)\n",
    "    #clf = SGDRegressor('huber',tol=1e-9,max_iter=100,verbose=1,penalty='l1',alpha=1e-7)\n",
    "    clf = SGDRegressor('huber',tol=1e-9,max_iter=SGD_ITER*10,penalty='l2',verbose=be_verbose,alpha=L2REG)\n",
    "\n",
    "    clf.fit(skipped_data @Xreg ,y)#(y-y.mean())/y.std())\n",
    "result_clf = np.squeeze(clf.coef_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SS = result_clf.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TMP = result_clf.reshape((-1,years_per_conf))\n",
    "#RESULTS = defaultdict(list)\n",
    "#for k,v in LOOKUP.items():\n",
    "#    true_conf = v[1][0]\n",
    "#    size = v[1][1]\n",
    "#    RESULTS[true_conf].append((size,TMP[conf_idx[v[0]]].mean()/SS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p_vals = []\n",
    "#for k,v in RESULTS.items():\n",
    "#    A = np.array(v)\n",
    "#    res = scipy.stats.pearsonr(A[:,0],A[:,1])\n",
    "#    p_vals.append(res[1])\n",
    "#p_vals = np.array(p_vals)\n",
    "#p_vals.mean(),np.median(p_vals),p_vals.min(),p_vals.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RESULTS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False and WEIGHT_TO_GET == 'faculty':\n",
    "    import csv\n",
    "    with open('pairwise_tiers.csv','rt') as csvfile:\n",
    "        filereader = csv.reader(csvfile)\n",
    "        conf_pairs = [[t.strip('\\ufeff') for t in _ if len(t)!=0] for _ in filereader]\n",
    "    classifier_cost = 0\n",
    "    conf_ord = np.argsort(result_clf)\n",
    "    conf_rank_dict = {}\n",
    "    num_elem = n_confs*years_per_conf\n",
    "    for i in range(num_elem):\n",
    "        idx = conf_ord[i]\n",
    "        conf_name = all_venues[idx//years_per_conf]\n",
    "        conf_score = result_clf[idx]\n",
    "        #if conf_score == 0:\n",
    "        #    conf_rank_dict[conf_name] = num_elem\n",
    "        #else:\n",
    "        conf_rank_dict[conf_name] = i\n",
    "    pair_len = len(conf_pairs)//2\n",
    "    for i in range(pair_len):\n",
    "        better = conf_pairs[2*i]\n",
    "        worse = conf_pairs[2*i+1]\n",
    "        #print(better,worse)\n",
    "        for b in better:\n",
    "            for w in worse:\n",
    "                classifier_cost += (conf_rank_dict[w] < conf_rank_dict[b])\n",
    "                if conf_rank_dict[w] < conf_rank_dict[b]:\n",
    "                    print(w,conf_rank_dict[w],'\\t',b,conf_rank_dict[b])\n",
    "\n",
    "    all_choices = clf.decision_function(Xauth)\n",
    "    frac_correct = (all_choices[y.astype(np.bool)] > 0).sum()\n",
    "    print(classifier_cost,frac_correct/y.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result_clf = np.copy(orig_clf)\n",
    "import matplotlib.pyplot as plt\n",
    "# normalize data by year\n",
    "if NORM_YEARS:\n",
    "    orig_clf = np.copy(result_clf)\n",
    "\n",
    "    result_clf = result_clf.reshape((-1,years_per_conf))\n",
    "\n",
    "    #plt.plot(result_clf.sum(0)/result_clf.sum(0).sum(),label='sum')\n",
    "    plt.plot(result_clf.std(0)/result_clf.std(0).sum(),label='std')\n",
    "    #print(abs(result_clf.mean(0)).mean(),abs(result_clf.std(0)).mean())\n",
    "    plt.legend()\n",
    "    #result_clf = (result_clf)/result_clf.std(0)\n",
    "    result_clf = (result_clf-result_clf.mean(0))/result_clf.std(0)\n",
    "    result_clf = result_clf.reshape((-1))\n",
    "    #result_clf = np.minimum(30,np.maximum(result_clf,-30))\n",
    "\n",
    "else:\n",
    "    orig_clf = np.copy(result_clf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(weight_file,'wb') as fp:\n",
    "    pickle.dump(orig_clf,fp)\n",
    "print('saved {}'.format(weight_file))\n",
    "if 'REGRESSION_TASK_IDX' in os.environ:\n",
    "    # THIS IS FINE. JUST AN EARLY EXIT\n",
    "    sys.exit(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_choice = ['SIGGRAPH','NIPS','3DV','HRI','Comput. Graph. Forum','Shape Modeling International',\n",
    "               'Symposium on Geometry Processing',' Computer Aided Geometric Design','ICLR',\n",
    "               'AAAI','I. J. Robotics Res.','CVPR','International Journal of Computer Vision',\n",
    "               'Robotics: Science and Systems','ICRA','WACV','ICML','AISTATS','CoRR','SIGGRAPH Asia',\n",
    "               'ECCV','ICCV','ISER','Humanoids','3DV','IROS','CoRL','Canadian Conference on AI',\n",
    "               'ACCV ','Graphics Interface','CRV','BMVC']\n",
    "ri_confs = np.zeros(n_confs*years_per_conf)\n",
    "conf_ord = np.argsort(result_clf)\n",
    "#print(clf.intercept_)\n",
    "ms = result_clf.mean()\n",
    "ss = result_clf.std()\n",
    "for i in range(n_confs*years_per_conf):\n",
    "    idx = conf_ord[-(i+1)]\n",
    "    conf_name = all_venues[idx//years_per_conf]\n",
    "    conf_score = result_clf[idx]\n",
    "    if conf_name in conf_choice:\n",
    "        ri_confs[idx] = 1\n",
    "    if conf_name in conf_choice and (idx%years_per_conf)==(year_ind-2):\n",
    "        print_name =conf_name + '_' + year_span_printable[idx%years_per_conf]\n",
    "        print('{:40s}\\t{:.1f}'.format(print_name[:35],(conf_score-ms)/ss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_k = 250\n",
    "for i in range(top_k):\n",
    "    idx = conf_ord[-(i+1)]\n",
    "    conf_name = all_venues[idx//years_per_conf]\n",
    "    conf_score = result_clf[idx]\n",
    "    print_name =conf_name + '_' + year_span_printable[idx%years_per_conf]\n",
    "    print('{:60s}\\t{:.1f}'.format(print_name[:55],(conf_score-ms)/ss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(20000):\n",
    "    idx = conf_ord[-(i+1)]\n",
    "    conf_name = all_venues[idx//years_per_conf]\n",
    "    conf_score = result_clf[idx]\n",
    "    if conf_name in conf_choice:\n",
    "        ri_confs[idx] = 1\n",
    "    if (idx%years_per_conf)==(year_ind-2):\n",
    "        print_name =conf_name + '_' + year_span_printable[idx%years_per_conf]\n",
    "        print('{:100s}\\t{:.1f}'.format(print_name,(conf_score-ms)/ss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "plt.figure()\n",
    "conf_choice2 = ['SIGGRAPH','AAAI','NIPS','CVPR','ICRA','ICML','ICCV','ECCV',\n",
    "               'International Journal of Computer Vision','Robotics: Science and Systems']\n",
    "conf_choices = [conf_choice2, \n",
    "                ['STOC','FOCS','SODA','EC','WINE'],\n",
    "                ['UAI','AAAI','IJCAI','ICML','NIPS'],\n",
    "                ['ICCV','ECCV','CVPR','International Journal of Computer Vision','3DV','WACV','IEEE Trans. Pattern Anal. Mach. Intell.'],\n",
    "                ['ICRA','Robotics: Science and Systems','IROS','CoRL','HRI','ISER','FSR'],\n",
    "                ['SIGGRAPH','SIGGRAPH Asia','ACM Trans. Graph.','Graphics Interface']\n",
    "               ]\n",
    "#conf_choices = [['Robotics: Science and Systems','IROS','ICRA','CoRL','WAFR','HRI','ISER']]\n",
    "for conf_choice2 in conf_choices:\n",
    "    plt.figure()\n",
    "    #conf_choice2 = \n",
    "    conf_choice3 = []\n",
    "    vs = result_clf.std()\n",
    "    for conf in conf_choice2:\n",
    "        idx = conf_idx[conf]\n",
    "        #s = max(result_clf[years_per_conf*idx:years_per_conf*(idx+1)])\n",
    "        s = result_clf[years_per_conf*(idx+1)-1]\n",
    "\n",
    "        conf_choice3.append((s,conf))\n",
    "    plt.figure(figsize=(12,8))\n",
    "    for s,conf in sorted(conf_choice3,reverse=True):\n",
    "        idx = conf_idx[conf]\n",
    "        weights = [result_clf[years_per_conf*idx + yr]/vs for yr in offset_years]\n",
    "        _ = plt.plot(np.arange(min_year,max_year+1),weights,label=conf,lw=5)\n",
    "    plt.grid(True)\n",
    "    plt.xlabel('year')\n",
    "    plt.ylabel('value')\n",
    "    #plt.ylim(-5,20)\n",
    "    plt.legend()\n",
    "    #plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " 'seaborn-white','seaborn','ggplot', 'seaborn-colorblind', 'seaborn-muted','seaborn-whitegrid'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for style in []:#plt.style.available:\n",
    "    plt.style.use(style)\n",
    "    plt.figure(figsize=(12,8))\n",
    "    for s,conf in sorted(conf_choice3,reverse=True):\n",
    "        idx = conf_idx[conf]\n",
    "        weights = [result_clf[years_per_conf*idx + yr]/vs for yr in offset_years]\n",
    "        _ = plt.plot(np.arange(min_year,max_year+1),weights,label=conf,lw=5)\n",
    "    plt.grid(True)\n",
    "    plt.xlabel('year')\n",
    "    plt.ylabel('value')\n",
    "    plt.title(style)\n",
    "    plt.legend()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xauth = None\n",
    "if Xauth is None:\n",
    "    count_vecs = {}\n",
    "    paper_vecs = []\n",
    "    for paper in all_papers:\n",
    "        tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "        n = len(authors)\n",
    "        j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "\n",
    "        if n not in count_vecs:\n",
    "            author_scores = np.ones(n) #1/(np.arange(n)+1) \n",
    "            #author_scores[-1] = author_scores[0]\n",
    "            author_score_sum = author_scores.sum()\n",
    "            #author_scores /= author_score_sum\n",
    "            count_vecs[n] = author_scores #/ author_score_sum\n",
    "        else:\n",
    "            author_scores = count_vecs[n]\n",
    "            paper_vecs.append([(name_idx[a],j,v) for a,v in zip(authors,author_scores)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Xauth = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if Xauth is None or Xauth.shape[1] != years_per_conf*n_confs:\n",
    "    import scipy.sparse\n",
    "    Xauth = scipy.sparse.dok_matrix((n_auths,years_per_conf*n_confs))\n",
    "    xdict = {}\n",
    "    if False:\n",
    "        for paper in all_papers:\n",
    "            tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "            n = len(authors)\n",
    "            j = years_per_conf*conf_idx[venue] + (year-min_year)//YEAR_BLOCKS\n",
    "            for ai in range(n):#zip(count_vecs[n],authors):\n",
    "                i = name_idx[authors[ai]]\n",
    "                #xdict[(i,j)] = 1/n + xdict.get((i,j),0)\n",
    "                xdict[(i,j)] = count_vecs[n][ai] + xdict.get((i,j),0)\n",
    "\n",
    "    else:\n",
    "        for paper_vec in paper_vecs:\n",
    "            for i,j,v in paper_vec:\n",
    "                xdict[(i,j)] = v + xdict.get((i,j),0)\n",
    "\n",
    "    Xauth.my_update(xdict)\n",
    "            \n",
    "    Xauth = scipy.sparse.csr_matrix(Xauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result_clf = np.copy(orig_clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "total_scores = Xauth.dot(result_clf)\n",
    "years_working = (1+auth_years[:,1]-auth_years[:,0])\n",
    "value_scores = (total_scores)/years_working\n",
    "ri_filter_mat = scipy.sparse.diags(ri_confs)\n",
    "ri_total_scores = Xauth.dot(ri_filter_mat).dot(result_clf)\n",
    "ri_value_scores = ri_total_scores/years_working\n",
    "pub_num = Xauth.sum(1)\n",
    "rs = ri_total_scores.std()\n",
    "rm = ri_total_scores.mean()\n",
    "\n",
    "ts = total_scores.std()\n",
    "tm = total_scores.mean()\n",
    "\n",
    "vs = value_scores.std()\n",
    "vm = value_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_cand = ['Pulkit Agrawal',\n",
    " 'Joydeep Biswas',\n",
    " 'Katherine L. Bouman',\n",
    " 'David Braun',\n",
    " 'Jia Deng',\n",
    " 'Naomi T. Fitter',\n",
    " 'David F. Fouhey',\n",
    " 'Saurabh Gupta',\n",
    " 'Judy Hoffman',\n",
    " 'Hanbyul Joo',\n",
    " 'Honglak Lee',\n",
    " 'Changliu Liu',\n",
    " 'Petter Nilsson',\n",
    " \"Matthew O'Toole\",\n",
    " 'Alessandro Roncone',\n",
    " 'Alanson P. Sample',\n",
    " 'Manolis Savva',\n",
    " 'Adriana Schulz',\n",
    " 'Amy Tabb',\n",
    " 'Fatma Zeynep Temel',\n",
    " 'Long Wang',\n",
    " 'Cathy Wu',\n",
    " 'Ling-Qi Yan']\n",
    "print('{:20s}\\t{:4s}\\t{:4s}\\t{:4s}\\t{}\\t{}'.format('name','total','rate','ri','years','pubs'))\n",
    "for ns, name in sorted([(total_scores[name_idx[ni]],ni) for ni in prev_cand],reverse=True):\n",
    "    ni = name_idx[name]\n",
    "    print('{:20s}\\t{:.2f}\\t{:.2f}\\t{:.2f}\\t{:.0f}\\t{:.1f}'.format(name,\n",
    "                                                                  (total_scores[ni]-tm)/ts,\n",
    "                                                                  (value_scores[ni]-vm)/vs,\n",
    "                                                                  (ri_total_scores[ni]-rm)/rs,\n",
    "                                                                  years_working[ni],pub_num[ni,0]))\n",
    "print('')\n",
    "curious_names = ['Xiaolong Wang 0004','Judy Hoffman','Paris Siminelakis','Roie Levin','Leonid Keselman',\n",
    "                 'Nicholas Rhinehart','Vincent Sitzmann','Siddharth Ancha','Xingyu Lin',\n",
    "                 'Humphrey Hu','Avideh Zakhor',\n",
    "                 'David F. Fouhey','Chelsea Finn','Nathan Michael',\n",
    "                 'Lerrel Pinto','Wen Sun 0002','Samuel Clarke','Ge Lv',\n",
    "                 'Justin Johnson',\n",
    "                 'Amir Roshan Zamir','Dominik Peters','Jonathan T. Barron','Dorsa Sadigh','Derek Hoiem','Vaggos Chatziafratis',\n",
    "                 'Brian Okorn','David Held']\n",
    "print('{:20s}\\t{:4s}\\t{:4s}\\t{:4s}\\t{}\\t{}'.format('name','total','rate','ri','years','pubs'))\n",
    "for _,name in sorted([(total_scores[name_idx[_]],_) for _ in curious_names],reverse=True):\n",
    "    ni = name_idx[name]\n",
    "    print('{:20s}\\t{:.2f}\\t{:.2f}\\t{:.2f}\\t{:.0f}\\t{:.1f}'.format(name,\n",
    "                                                                  (total_scores[ni]-tm)/ts,\n",
    "                                                                  (value_scores[ni]-vm)/vs,\n",
    "                                                                  (ri_total_scores[ni]-rm)/rs,\n",
    "                                                                  years_working[ni],pub_num[ni,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_faculty = faculty_affil[faculty_affil.affiliation == 'Carnegie Mellon University'] #Carnegie Mellon University\n",
    "uni_names = np.array(uni_faculty.name)\n",
    "uni_names = list(uni_names)\n",
    "cmu_scores = []\n",
    "for name in set([aliasdict.get(n, n) for n in uni_names]):\n",
    "    if name in name_idx:\n",
    "        score = total_scores[name_idx[name]]\n",
    "        cmu_scores.append(((score-tm)/ts,name))\n",
    "for s,p in sorted(cmu_scores,reverse=True):\n",
    "    print('{:30s}\\t\\t{:.3f}'.format(p,s))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter,defaultdict\n",
    "def di():\n",
    "    return defaultdict(float)\n",
    " \n",
    "author_by_year = defaultdict(di)\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    n = len(authors)\n",
    "    for a in authors:\n",
    "        author_by_year[a][year] += result_clf[years_per_conf*conf_idx[venue] + offset_years[year-min_year]]/n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "plt.figure(figsize=(8,8))\n",
    "example_names = ['Takeo Kanade','Martial Hebert','Christopher G. Atkeson','Howie Choset','Deva Ramanan','Jessica K. Hodgins'] #,'Pieter Abbeel'\n",
    "for example_name in example_names:\n",
    "    example_value = np.zeros(max_year+1-min_year)\n",
    "    years = author_by_year[example_name]\n",
    "    yrs = [_ for _ in years.keys() if _  > 0]\n",
    "    start_year = min(yrs)\n",
    "    end_year = max(yrs)\n",
    "    span = end_year - start_year\n",
    "    start_year,end_year,span\n",
    "    for y,v in years.items():\n",
    "        example_value[y-min_year] += v\n",
    "            \n",
    "    plt.plot(np.arange(min_year,max_year+1)[:-1],gaussian_filter1d(example_value[:-1], sigma=3),label=example_name,lw=3)\n",
    "    #plt.plot(gaussian_filter1d(example_value[:span], sigma=2),label=example_name )\n",
    "\n",
    "#plt.plot((val_by_year/v_count),label='average author')\n",
    "plt.ylabel('annual value (3yr avg)')\n",
    "#plt.xlabel('year since first publication')\n",
    "plt.xlabel('working year')\n",
    "\n",
    "plt.legend()\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_corr = pd.read_csv('other_ranks/correlation_cleaned.csv')\n",
    "df_corr = df_corr.drop(columns=[_ for _ in df_corr.columns if 'Unnamed' in _])\n",
    "df_corr = df_corr.drop(columns=['pms','n_papers'])\n",
    "df_corr = df_corr.rename(columns={'totals': 'venue_score', 'csrp': 'csr_pubs','csrpn': 'csr_adj','gcite': 'influence'})\n",
    "df_corr = df_corr[['name','papers', 'citations', 'h-index',\n",
    "       'i10','csr_pubs', 'csr_adj','venue_score','influence']]\n",
    "df_corr = df_corr.dropna('index')\n",
    "df_corr.index = df_corr.name\n",
    "\n",
    "ts = total_scores.std()\n",
    "tm = total_scores.mean()\n",
    "for name in df_corr.name:\n",
    "    if name in name_idx:\n",
    "        idx = name_idx[name]\n",
    "        df_corr.loc[name,'venue_score'] = (total_scores[idx]-tm)/ts\n",
    "print(df_corr.corr('spearman').loc['influence','venue_score'],df_corr.corr('kendall').loc['influence','venue_score'],df_corr.corr('spearman').loc['h-index','venue_score'])\n",
    "#if clfn == clfs_test.shape[-1]:\n",
    "df_corr.corr('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(weight_file,'wb') as fp:\n",
    "    pickle.dump(orig_clf,fp)\n",
    "print('saved {}'.format(weight_file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "one_auth_confs = {}\n",
    "clf_std = result_clf.std()\n",
    "clf_mean = result_clf.mean()\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    if 'Christopher G. Atkeson' in authors:\n",
    "        one_auth_confs[venue] = 1 + one_auth_confs.get(venue,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for v,k in sorted([(v,k) for k,v in one_auth_confs.items()],reverse=True):\n",
    "#    print('{:40s}\\t{}\\t{:.1f}'.format(k,v,(result_clf[conf_idx[k]*years_per_conf+4]-clf_mean)/clf_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_counts = {}\n",
    "for paper in all_papers:\n",
    "    tag,title, authors, venue, pages, startPage,year,volume,number,url,publtype,eb_toofew,eb_skip = paper\n",
    "    page_counts[pages] = 1 + page_counts.get(pages,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sorted([(v,k) for k,v in page_counts.items()],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = pd.read_csv('other_ranks/faculty_affil_scholar.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fuzzythresh=0.9 #0.9\n",
    "df_merged.loc[df_merged.fuzzyscore > fuzzythresh,'dblpname'] = df_merged[df_merged.fuzzyscore > fuzzythresh].fuzzyname\n",
    "df_merged.loc[df_merged.fuzzyscore > fuzzythresh,'dblpexists'] = 1\n",
    "df_merged = df_merged[df_merged.dblpexists == 1]\n",
    "df_merged['venue_score'] = np.ones_like(df_merged.dblpexists)\n",
    "ts = total_scores.std()\n",
    "tm = total_scores.mean()\n",
    "df_merged = df_merged.set_index(df_merged.dblpname)\n",
    "seen_map = {}\n",
    "for name in df_merged.index:\n",
    "    if name in name_idx:\n",
    "        idx = name_idx[name]\n",
    "        df_merged.loc[name,'venue_score'] = total_scores[idx]\n",
    "        seen_map[name] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = df_merged.drop(columns=['Unnamed: 0','First Name','Last Name','Sholar link','Rank (Full, Associate, Assistant, Other)','Full Name','University_y','University_x','Unnamed: 11',\"ID\",'fuzzyname','dblpexists','fuzzyscore','UniversityID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_datas = []\n",
    "for row in faculty_affil.itertuples():\n",
    "    if row[1] in name_idx and row[1] not in seen_map:\n",
    "        seen_map[row[1]] = 1\n",
    "        new_data = {}\n",
    "        new_data['dblpname'] = row[1]\n",
    "        #new_data['index'] = row[1]\n",
    "        new_data['school'] = row[2]\n",
    "        new_data['venue_score'] = total_scores[idx]\n",
    "        new_datas.append(new_data)\n",
    "        #df_merged = df_merged.append([row[1],np.nan,np.nan,np.nan,row[2],row[1],(total_scores[idx]-tm)/ts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_merged.shape)\n",
    "df_csr_to_add = pd.DataFrame(new_datas)\n",
    "#df_csr_to_add = df_csr_to_add.set_index('dblpname')\n",
    "df_csr_to_add = df_csr_to_add.set_index('dblpname')\n",
    "#df_merged = pd.concat([df_merged,df_csr_to_add])\n",
    "print(df_merged.shape)\n",
    "#print(faculty_affil.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t10 = df_merged[df_merged['t10-index'].notna()]\n",
    "#df_t10.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "school_vals = df_t10.groupby('school').aggregate('sum').sort_values('venue_score',0,False)\n",
    "t10_schools = school_vals.sort_values('t10-index',0,False)\n",
    "school_vals.corr('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " \n",
    "df_t10.corr('spearman')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_hindex = df_merged[df_merged['h-index'].notna()]\n",
    "#df_hindex.corr('spearman')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "school_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bp2 = pd.read_csv('other_ranks/uni_rank_bp.csv')\n",
    "times = pd.read_csv('other_ranks/uni_rank_times.csv')\n",
    "\n",
    "srf2 = pd.read_csv('other_ranks/uni_rank_mergedscholar.csv')\n",
    "st2 = pd.read_csv('other_ranks/uni_rank_st.csv')\n",
    "qt2 = pd.read_csv('other_ranks/uni_rank_qt.csv')\n",
    "sr2 = pd.read_csv('other_ranks/uni_rank_sr.csv')\n",
    "pr2 = pd.read_csv('other_ranks/uni_rank_pr.csv')\n",
    "cm2 = pd.read_csv('other_ranks/uni_rank_cs.csv')\n",
    "usn2 = pd.read_csv('other_ranks/uni_rank_usn.csv')\n",
    "df_csr = pd.read_csv('other_ranks/ranks.csv')\n",
    "\n",
    "pr2.USN2010 = pr2.USN2010.map(lambda x: int(x) if x.isnumeric() else np.nan)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    [(row[2],row[1]) for row in st2.itertuples()],\n",
    "    [(row[2],row[1]) for row in times.itertuples()],\n",
    "    [(row[2],row[1]) for row in qt2.itertuples()],\n",
    "    [(row[6],row[1]) for row in pr2.itertuples()],\n",
    "    [(row[2],row[1]) for row in cm2.itertuples()],\n",
    "    [(row[2],row[1]) for row in sr2.itertuples()],\n",
    "    [(row[-1],row[2]) for row in srf2.itertuples()],\n",
    "\n",
    "    [(row[0],idx+1) for idx,row in enumerate(t10_schools.itertuples())],\n",
    "    [(row[2],row[1]) for row in df_csr.itertuples()],\n",
    "    [(row[0],idx+1) for idx,row in enumerate(school_vals.itertuples())],\n",
    "    [(row[2],row[1]) for row in bp2.itertuples()],\n",
    "    [(row[6],row[4]) for row in pr2.sort_values('NRC95',ascending=True).itertuples() ],\n",
    "    [(row[6],row[3]) for row in pr2.sort_values('USN2010',ascending=True).itertuples() if np.isfinite(row[3]) ],\n",
    "    [(row[2],row[1]) for row in usn2.itertuples()]\n",
    "]\n",
    "dataset_names = ['Shanghai','Times','QS','Prestige','CSMetrics',\n",
    "                 'ScholarRank','ScholarRankFull','t10Sum','CSRankings','Mine','BestPaper','NRC95',\"USN10\",'USN18']\n",
    "n_datasets = len(datasets)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_matrix = np.zeros((n_datasets,n_datasets))\n",
    "count_matrix = np.zeros((n_datasets,n_datasets))\n",
    "name_datasets = [ [v[0] for v in d] for d in datasets ]\n",
    "all_names = sorted(list(set(sum(name_datasets,[]))))\n",
    "all_vec = [sum([name in d for d in name_datasets])>=(len(datasets)-4) for name in all_names]\n",
    "subset_names = [name for name,vec in zip(all_names,all_vec) if vec]\n",
    "subset_names,len(subset_names)\n",
    "import scipy.stats as stats\n",
    "for i in range(n_datasets):\n",
    "    inames = [u[0] for u in datasets[i]]\n",
    "    for j in range(i,n_datasets):\n",
    "        jnames = [u[0] for u in datasets[j]]\n",
    "\n",
    "        #exist_1 = [((ni in subset_names) and (ni in jnames)) for ni in inames]\n",
    "        #exist_2 = [((nj in subset_names) and (nj in inames))for nj in jnames]\n",
    "        exist_1 = [((True) and (ni in jnames)) for ni in inames]\n",
    "        exist_2 = [((True) and (nj in inames))for nj in jnames]\n",
    "        \n",
    "        d1 = np.array(datasets[i])[exist_1]\n",
    "        d2 = np.array(datasets[j])[exist_2]\n",
    "        v1 = d1[:,1].astype(np.float32)\n",
    "        v2 = np.array([d2[np.where(d2[:,0] == name)[0][0],1] for name in d1[:,0]]).astype(np.float32)\n",
    "        c = stats.spearmanr(v1,v2)[0]\n",
    "        corr_matrix[i][j] = c\n",
    "        corr_matrix[j][i] = c\n",
    "        count_matrix[i][j] = len(v1)\n",
    "        count_matrix[j][i] = len(v2)\n",
    "        #print(v1,v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=2)\n",
    "print(corr_matrix)\n",
    "\n",
    "print('mean best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names,corr_matrix.mean(1))],reverse=True):\n",
    "    print('{:30s}\\t{:.3f}'.format(n,s))\n",
    "print('\\n usnews best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names,corr_matrix[-1])],reverse=True):\n",
    "    print('{:30s}\\t{:.3f}'.format(n,s))\n",
    "print('\\n names')\n",
    "\n",
    "for n in dataset_names:\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_year_mean = result_clf.reshape((-1,years_per_conf)).mean(0)\n",
    "by_year_std = result_clf.reshape((-1,years_per_conf)).std(0)\n",
    "print(by_year_std)\n",
    "plt.figure(figsize=(24,6))\n",
    "plt.subplot(1,2,1)\n",
    "confs_of_interest = ['SIGGRAPH','AAAI','NIPS','CVPR','ICRA','ICML','ICCV','ECCV', 'I. J. Robotics Res.',\n",
    "                'WACV','CHI','ACC','HRI',  'AAMAS','IJCAI',\n",
    "               'ISER','Robotics: Science and Systems','IROS','CoRL','ICLR','3DV']\n",
    "#confs_of_interest = ['CVPR','ICRA',\"ICCV\",'CoRL','Robotics: Science and Systems','ECCV','WACV','IROS']\n",
    "#confs_of_interest = ['SIGIR','JCDL','CIKM','KDD','WWW','SIGMOD Conference','VLDB']\n",
    "confs_of_interest = ['SIGIR','JCDL','CIKM','KDD','WWW','SIGMOD Conference','VLDB']\n",
    "#confs_of_interest = ['AAAI',\"NIPS\",'ICML','IJCAI','UAI','AISTATS','COLT']\n",
    "#confs_of_interest = ['ICRA',\"IROS\",'ISER','CoRL','Robotics: Science and Systems',\"WAFR\"]\n",
    "#confs_of_interest = ['SODA','STOC','FOCS','WINE','EC','COLT','Theory of Computing']\n",
    "#confs_of_interest =    ['CHI','ACM Trans. Comput.-Hum. Interact.','CSCW','UbiComp','UIST','ICWSM']\n",
    "\n",
    "for conf in confs_of_interest:\n",
    "    idx = conf_idx[conf]\n",
    "    weights = [(result_clf[years_per_conf*idx + yr]-by_year_mean[yr])/by_year_std[yr] for yr in offset_years]\n",
    "    plt.plot(np.arange(min_year,max_year+1),weights,label=conf,lw=4)\n",
    "plt.grid(True)\n",
    "plt.legend()\n",
    "plt.title('adjusted')\n",
    "plt.subplot(1,2,2)\n",
    "plt.title('unadjusted')\n",
    "for conf in confs_of_interest:\n",
    "    idx = conf_idx[conf]\n",
    "    weights = [result_clf[years_per_conf*idx + yr] for yr in offset_years]\n",
    "    plt.plot(np.arange(min_year,max_year+1),weights,label=conf,lw=4)\n",
    "plt.legend()\n",
    "plt.grid(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_corr.corr('spearman').loc['influence','venue_score'])\n",
    "print(df_corr.corr('spearman').loc['h-index','venue_score'])\n",
    "print(corr_matrix[-1][8])\n",
    "print(df_t10.corr('spearman').loc['venue_score','t10-index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "\n",
    "cmu_facutly = pd.read_csv('other_ranks/cmu_faculty.csv')\n",
    "subdept = {}\n",
    "subdept_count = {}\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "yearly_subdept = {}\n",
    "\n",
    "seen = {}\n",
    "\n",
    "cmu_facutly.dept = cmu_facutly.dept.fillna('CSD')\n",
    "\n",
    "for sd in cmu_facutly.dept.unique():\n",
    "    for row in cmu_facutly[cmu_facutly.dept == sd].itertuples():\n",
    "        name = aliasdict.get(row[1],row[1])\n",
    "        if name in name_idx and name not in seen:\n",
    "            seen[name] = 1\n",
    "            subdept[row[2]] = total_scores[name_idx[name]] + subdept.get(row[2],0)\n",
    "            subdept_count[row[2]] = 1 + subdept_count.get(row[2],0)\n",
    "\n",
    "seen = {}\n",
    "\n",
    "subdept = {k:v for k,v in subdept.items()}\n",
    "\n",
    "for value, dept in sorted([(v,k) for k,v in subdept.items()],reverse=True):\n",
    "    print(dept,value)\n",
    "    example_value = np.zeros(max_year+1-min_year)\n",
    "    count_value = np.zeros(max_year+1-min_year)\n",
    "\n",
    "    for row in cmu_facutly[cmu_facutly.dept == dept].itertuples():\n",
    "        name = aliasdict.get(row[1],row[1])\n",
    "        if name in name_idx and name not in seen:\n",
    "            seen[name] = 1\n",
    "            years = author_by_year[name]\n",
    "            yrs = [_ for _ in years.keys() if _  > 0]\n",
    "            if len(yrs) > 0:\n",
    "                start_year = min(yrs)\n",
    "                end_year = max(yrs)\n",
    "                span = end_year - start_year\n",
    "                start_year,end_year,span\n",
    "                for y,v in years.items():\n",
    "                    example_value[y-min_year] += v\n",
    "                    count_value[y-min_year] += 1\n",
    "\n",
    "    plt.plot(np.arange(min_year,max_year+1)[:-2],gaussian_filter1d((example_value)[:-2], sigma=2),label='{} ({:.1f})'.format(dept,subdept[dept]/1000),lw=3)\n",
    "    #plt.plot(gaussian_filter1d(example_value[:span], sigma=2),label=example_name )\n",
    "\n",
    "#plt.plot((val_by_year/v_count),label='average author')\n",
    "plt.ylabel('annual value (2yr sigma smoothing)')\n",
    "#plt.xlabel('year since first publication')\n",
    "plt.xlabel('working year')\n",
    "\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "print(sum(subdept.values()))\n",
    "print(total_scores[name_idx['Takeo Kanade']])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subdept_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('other_ranks/msar.json') as fp:\n",
    "    msar = json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    df_msar = pd.read_csv('other_ranks/traditional_conf_scores.csv')\n",
    "except:\n",
    "    from fuzzywuzzy import process, fuzz\n",
    "    df_msar = pd.DataFrame(msar)\n",
    "    dblp_conf_name = []\n",
    "    matchable_names = [fuzz._process_and_sort(n,False) for n in all_venues]\n",
    "    for row in df_msar.itertuples():\n",
    "        #print(row[2],row[-1])\n",
    "        try:\n",
    "            if row[-1] in conf_idx:\n",
    "                dblp_conf_name.append(row[-1])\n",
    "            elif row[2] in conf_idx:\n",
    "                dblp_conf_name.append(row[2])\n",
    "            elif len(row[2].split('/')) > 1:\n",
    "                found = False\n",
    "                for subname in row[2].split('/'):\n",
    "                    if found == False and subname in conf_idx:\n",
    "                        dblp_conf_name.append(subname)\n",
    "                        found = True\n",
    "                if found == False:\n",
    "                    raise\n",
    "            elif len(row[2].split('(')) > 1:\n",
    "                substr = row[2].split('(')\n",
    "                found = False\n",
    "                for subname in [substr[0],substr[1][:-1]]:\n",
    "                    if found == False and subname in conf_idx:\n",
    "                        dblp_conf_name.append(subname)\n",
    "                        found = True\n",
    "                if found == False:\n",
    "                    raise\n",
    "            else:\n",
    "                raise\n",
    "        except:\n",
    "            bestshort,bestlong = None,None\n",
    "            if row[2] != None:\n",
    "                matcher = fuzz.SequenceMatcher(None, fuzz._process_and_sort(row[2],False))\n",
    "                n3s = []\n",
    "                for n2 in matchable_names:\n",
    "                    matcher.set_seq2(n2)\n",
    "                    n3s.append(matcher.ratio())\n",
    "                v=np.argmax(n3s)\n",
    "                bestshort = (all_venues[v],n3s[v])\n",
    "                #print(bestshort[1:],end='\\t')\n",
    "            if row[-1] != None:\n",
    "                matcher = fuzz.SequenceMatcher(None, fuzz._process_and_sort(row[-1],False))\n",
    "                n3s = []\n",
    "                for n2 in matchable_names:\n",
    "                    matcher.set_seq2(n2)\n",
    "                    n3s.append(matcher.ratio())\n",
    "                v=np.argmax(n3s)\n",
    "                bestlong = (all_venues[v],n3s[v])\n",
    "                #print(bestlong[1:],end='\\t')\n",
    "            if bestlong and bestlong[-1] > 0.96:\n",
    "                dblp_conf_name.append(bestlong[0])\n",
    "                #print(bestlong,row)\n",
    "            elif bestshort and bestshort[-1] > 0.96:\n",
    "                dblp_conf_name.append(bestshort[0])\n",
    "                #print(bestshort,row)\n",
    "            else:\n",
    "                #print(bestlong,bestshort,row[2],row[-1])\n",
    "                dblp_conf_name.append('NotAConf')\n",
    "    df_msar['dblp_name'] = dblp_conf_name\n",
    "    df_msar.to_csv('other_ranks/traditional_conf_scores.csv')\n",
    "    #ILPS/ISLP/NACLP/SLP\n",
    "    #DISC(WDAG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_msar.shape,df_msar[df_msar.dblp_name == 'NotAConf'].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_msar_filt = df_msar[df_msar.dblp_name != 'NotAConf'].copy()#.sort_values('H',ascending=False)\n",
    "scores = []\n",
    "for row in df_msar_filt.itertuples():\n",
    "    conf = row[-1]\n",
    "    if conf in conf_idx:\n",
    "        idx = conf_idx[conf]\n",
    "        weights = [result_clf[years_per_conf*idx + yr] for yr in offset_years[1984-min_year:2014-min_year]]\n",
    "        #scores.append(result_clf[years_per_conf*idx + offset_years[2014-min_year]])\n",
    "\n",
    "        scores.append(np.max(np.array([w for w in weights])))\n",
    "    else: # some naming issue\n",
    "        scores.append(-1)\n",
    "        print(conf)\n",
    "\n",
    "    #print(scores[-1],weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_msar_filt['venue_scores'] = scores\n",
    "df_msar_filt = df_msar_filt[~df_msar_filt.dblp_name.duplicated()].copy()\n",
    "df_msar_filt = df_msar_filt[~df_msar_filt.venue_scores.duplicated()].copy()\n",
    "#df_msar_filt['h-approx'] = 0.54*np.sqrt(df_msar_filt.citations)\n",
    "\n",
    "df_msar_filt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh = 10\n",
    "print(df_msar_filt[df_msar_filt.H > thresh].shape)\n",
    "df_msar_filt[df_msar_filt.H > thresh].corr('spearman')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_msar_filt[df_msar_filt.category == 'Computer Vision'].sort_values('venue_scores',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(df_msar_filt.venue_scores == -0.029198231466385834).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_msar.category.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}