Class Exercise 9.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f51ffe28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\dave_\\\\Desktop\\\\DHDA\\\\SEMESTER1\\\\DHDS110 -Data Science and Data Analytics\\\\Week 9 - Pyton'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "681beb06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\dave_\\\\Desktop\\\\DHDA\\\\SEMESTER1\\\\DHDS110 -Data Science and Data Analytics\\\\Week 9 - Python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11dee2f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "import seaborn as sb\n",
    "\n",
    "import sas7bdat\n",
    "from sas7bdat import *\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1d6022b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xlrd==1.2.0 in c:\\users\\dave_\\anaconda3\\lib\\site-packages (1.2.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install xlrd==1.2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "16abb7d3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#1) From assignment 1, import the provided xlsx files (the same ones you imported into MSAccess) into Jupyter: \n",
    "#Comment2, FallTable2, InterviewTable2, MedicalHistory2, SurgicalHistory2\n",
    "\n",
    "Comment2 = pd.read_excel(\"Comment2.xlsx\")\n",
    "Interview2 = pd.read_excel(\"InterviewTable2.xlsx\")\n",
    "Fall2 = pd.read_excel(\"FallTable2.xlsx\")\n",
    "MedHx2 = pd.read_excel(\"MedicalHistory2.xlsx\")\n",
    "SurgHx2 = pd.read_excel(\"SurgicalHistory2.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "703209a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SDHospitalAd</th>\n",
       "      <th>SDAdreasonHD</th>\n",
       "      <th>SDAdreasonBC</th>\n",
       "      <th>SDAdreasonCU</th>\n",
       "      <th>SDAdreasonOC</th>\n",
       "      <th>SDAdreasonOCSpec</th>\n",
       "      <th>SDAdreasonOA</th>\n",
       "      <th>SDAdreasonOASpec</th>\n",
       "      <th>SDHysterectomy</th>\n",
       "      <th>SDHysterectomyAge</th>\n",
       "      <th>SubjectID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM10353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>50.0</td>\n",
       "      <td>AM10355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM10357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>infection on antibiotic pump</td>\n",
       "      <td>Yes</td>\n",
       "      <td>40.0</td>\n",
       "      <td>AM10359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>42.0</td>\n",
       "      <td>AM10361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM20731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM20732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM20733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM20841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AM50351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>313 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    SDHospitalAd  SDAdreasonHD  SDAdreasonBC  SDAdreasonCU  SDAdreasonOC  \\\n",
       "0             No             0             0             0             0   \n",
       "1             No             0             0             0             0   \n",
       "2             No             0             0             0             0   \n",
       "3            Yes             0             0             0             0   \n",
       "4             No             0             0             0             0   \n",
       "..           ...           ...           ...           ...           ...   \n",
       "308           No             0             0             0             0   \n",
       "309           No             0             0             0             0   \n",
       "310           No             0             0             0             0   \n",
       "311           No             0             0             0             0   \n",
       "312           No             0             0             0             0   \n",
       "\n",
       "    SDAdreasonOCSpec  SDAdreasonOA              SDAdreasonOASpec  \\\n",
       "0                NaN             0                           NaN   \n",
       "1                NaN             0                           NaN   \n",
       "2                NaN             0                           NaN   \n",
       "3                NaN             1  infection on antibiotic pump   \n",
       "4                NaN             0                           NaN   \n",
       "..               ...           ...                           ...   \n",
       "308              NaN             0                           NaN   \n",
       "309              NaN             0                           NaN   \n",
       "310              NaN             0                           NaN   \n",
       "311              NaN             0                           NaN   \n",
       "312              NaN             0                           NaN   \n",
       "\n",
       "    SDHysterectomy  SDHysterectomyAge SubjectID  \n",
       "0               No                NaN   AM10353  \n",
       "1              Yes               50.0   AM10355  \n",
       "2               No                NaN   AM10357  \n",
       "3              Yes               40.0   AM10359  \n",
       "4              Yes               42.0   AM10361  \n",
       "..             ...                ...       ...  \n",
       "308             No                NaN   AM20731  \n",
       "309             No                NaN   AM20732  \n",
       "310             No                NaN   AM20733  \n",
       "311             No                NaN   AM20841  \n",
       "312             No                NaN   AM50351  \n",
       "\n",
       "[313 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2) Sort each of them by SubjectID\n",
    "# Could sort by multiple variables, and define if ascending or descending using list \n",
    "# matching the same order of variables\n",
    "# na_position forces NA values to first or last \n",
    "\n",
    "Comment2.sort_values(by=['SubjectID'], na_position=\"last\")\n",
    "Interview2.sort_values(by=['SubjectID'], na_position=\"last\")\n",
    "Fall2.sort_values(by=['SubjectID'], na_position=\"last\")\n",
    "MedHx2.sort_values(by=['SubjectID'], na_position=\"last\")\n",
    "SurgHx2.sort_values(by=['SubjectID'], na_position=\"last\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "64842f81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             IAComment SubjectID   SIDateiv  \\\n",
      "0    very easy-going - real nuisance bring pills in...   AM10353 2014-08-01   \n",
      "1                                                  NaN   AM10355 2014-10-05   \n",
      "2                                                  NaN   AM10357 2014-11-28   \n",
      "3                                                  NaN   AM10359 2014-05-23   \n",
      "4    very interested in research, no MRI because pa...   AM10361 2014-08-07   \n",
      "..                                                 ...       ...        ...   \n",
      "291  Nutrition and SF36 Questionnaires completed at...   AM20723 2016-04-07   \n",
      "292  Nutrition and SF36 Questionnaires completed at...   AM20731 2016-04-01   \n",
      "293  Nutrition and SF36 questionnaires completed at...   AM20732 2016-04-22   \n",
      "294  Nutrition and SF36 Questionnaires completed at...   AM20733 2016-04-27   \n",
      "295  Nutrition and SF36 Questionnaires completed at...   AM20841 2015-11-18   \n",
      "\n",
      "    SIInterviewerName  SIHourBegan  SIMinuteBegan  SIHourEnded  SIMinuteEnded  \\\n",
      "0      Kim Kardashian          2.0           45.0          1.0            0.0   \n",
      "1      Kim Kardashian          1.0           44.0          2.0           50.0   \n",
      "2      Kim Kardashian          2.0           11.0          3.0            0.0   \n",
      "3      Kim Kardashian         10.0           37.0         11.0           45.0   \n",
      "4           Sam Smith          1.0           31.0          2.0           35.0   \n",
      "..                ...          ...            ...          ...            ...   \n",
      "291    Kim Kardashian          8.0           17.0         11.0           50.0   \n",
      "292    Kim Kardashian         10.0           42.0         11.0           20.0   \n",
      "293    Kim Kardashian         10.0           42.0         11.0           15.0   \n",
      "294    Kim Kardashian         10.0           45.0         11.0           25.0   \n",
      "295    Kim Kardashian          4.0            5.0         11.0           45.0   \n",
      "\n",
      "     SIDXA  SIpQCT  ...  SDHospitalAd  SDAdreasonHD  SDAdreasonBC  \\\n",
      "0        1       1  ...            No             0             0   \n",
      "1        1       1  ...            No             0             0   \n",
      "2        1       1  ...            No             0             0   \n",
      "3        1       1  ...           Yes             0             0   \n",
      "4        1       1  ...            No             0             0   \n",
      "..     ...     ...  ...           ...           ...           ...   \n",
      "291      1       1  ...            No             0             0   \n",
      "292      1       1  ...            No             0             0   \n",
      "293      1       1  ...            No             0             0   \n",
      "294      1       1  ...            No             0             0   \n",
      "295      1       1  ...            No             0             0   \n",
      "\n",
      "     SDAdreasonCU SDAdreasonOC SDAdreasonOCSpec  SDAdreasonOA  \\\n",
      "0               0            0              NaN             0   \n",
      "1               0            0              NaN             0   \n",
      "2               0            0              NaN             0   \n",
      "3               0            0              NaN             1   \n",
      "4               0            0              NaN             0   \n",
      "..            ...          ...              ...           ...   \n",
      "291             0            0              NaN             0   \n",
      "292             0            0              NaN             0   \n",
      "293             0            0              NaN             0   \n",
      "294             0            0              NaN             0   \n",
      "295             0            0              NaN             0   \n",
      "\n",
      "                 SDAdreasonOASpec  SDHysterectomy  SDHysterectomyAge  \n",
      "0                             NaN              No                NaN  \n",
      "1                             NaN             Yes               50.0  \n",
      "2                             NaN              No                NaN  \n",
      "3    infection on antibiotic pump             Yes               40.0  \n",
      "4                             NaN             Yes               42.0  \n",
      "..                            ...             ...                ...  \n",
      "291                           NaN              No                NaN  \n",
      "292                           NaN              No                NaN  \n",
      "293                           NaN              No                NaN  \n",
      "294                           NaN              No                NaN  \n",
      "295                           NaN              No                NaN  \n",
      "\n",
      "[296 rows x 74 columns]\n"
     ]
    }
   ],
   "source": [
    "#3) Merge all of them together in an inner join as a new dataframe\n",
    "TablesJoin = Comment2.merge(Interview2, on=\"SubjectID\", how=\"inner\")\n",
    "TablesJoin = TablesJoin.merge(Fall2, on=\"SubjectID\", how=\"inner\")\n",
    "TablesJoin = TablesJoin.merge(MedHx2, on=\"SubjectID\", how=\"inner\")\n",
    "TablesJoin = TablesJoin.merge(SurgHx2, on=\"SubjectID\", how=\"inner\")\n",
    "print(TablesJoin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "151114f0",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "cannot subtract DatetimeArray from ndarray",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-028baefbfd49>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m# use suffix .dt.days after subtraction of dates\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;31m# requires dates to be a legitimate pythonic date - not string\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mTablesJoin\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"timesinceinv\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mTablesJoin\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'TodaysDate'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mTablesJoin\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"SIDateiv\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdays\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\ops\\common.py\u001b[0m in \u001b[0;36mnew_method\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m     63\u001b[0m         \u001b[0mother\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mitem_from_zerodim\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnew_method\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\arraylike.py\u001b[0m in \u001b[0;36m__sub__\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0munpack_zerodim_and_defer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"__sub__\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__sub__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 97\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_arith_method\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moperator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msub\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0munpack_zerodim_and_defer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"__rsub__\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m_arith_method\u001b[1;34m(self, other, op)\u001b[0m\n\u001b[0;32m   4996\u001b[0m         \u001b[0mlvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4997\u001b[0m         \u001b[0mrvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4998\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marithmetic_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4999\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5000\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_construct_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mres_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\ops\\array_ops.py\u001b[0m in \u001b[0;36marithmetic_op\u001b[1;34m(left, right, op)\u001b[0m\n\u001b[0;32m    183\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mshould_extension_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mTimedelta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    184\u001b[0m         \u001b[1;31m# Timedelta is included because numexpr will fail on it, see GH#31457\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 185\u001b[1;33m         \u001b[0mres_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    186\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    187\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\arrays\\datetimelike.py\u001b[0m in \u001b[0;36m__rsub__\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m   1285\u001b[0m             \u001b[1;31m# GH#19959 datetime - datetime is well-defined as timedelta,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1286\u001b[0m             \u001b[1;31m# but any other type - datetime is not well-defined.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1287\u001b[1;33m             raise TypeError(\n\u001b[0m\u001b[0;32m   1288\u001b[0m                 \u001b[1;34mf\"cannot subtract {type(self).__name__} from {type(other).__name__}\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1289\u001b[0m             )\n",
      "\u001b[1;31mTypeError\u001b[0m: cannot subtract DatetimeArray from ndarray"
     ]
    }
   ],
   "source": [
    "#4) Calculate time SINCE the interview date (SIDateiv) - using your \"today\"'s date - expressed in months  (call this  timesinceint)\n",
    "from datetime import date\n",
    "TablesJoin['TodaysDate'] = date.today()\n",
    "\n",
    "# use suffix .dt.days after subtraction of dates\n",
    "# requires dates to be a legitimate pythonic date - not string \n",
    "TablesJoin[\"timesinceinv\"] = (TablesJoin['TodaysDate'] - TablesJoin[\"SIDateiv\"]).dt.days\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8212c9dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#6) Subset this dataframe by selecting only cases where  timesinceint is >= mean of the value of  timesinceint across all participants (e.g. the 50% most recent participants) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7dfe5696",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubjectID</th>\n",
       "      <th>SDOsteoDiag</th>\n",
       "      <th>SDRheumaDiag</th>\n",
       "      <th>SDOADiag</th>\n",
       "      <th>SDLupusDiag</th>\n",
       "      <th>SDHyperthyDiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AM10353</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AM10355</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AM10357</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AM10359</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AM10361</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>AM20723</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>AM20731</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>AM20732</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>AM20733</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>AM20841</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>296 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    SubjectID SDOsteoDiag SDRheumaDiag SDOADiag SDLupusDiag SDHyperthyDiag\n",
       "0     AM10353          No           No      Yes          No             No\n",
       "1     AM10355         Yes           No       No          No             No\n",
       "2     AM10357         Yes           No       No          No             No\n",
       "3     AM10359         Yes          Yes      Yes          No             No\n",
       "4     AM10361          No           No       No          No             No\n",
       "..        ...         ...          ...      ...         ...            ...\n",
       "291   AM20723          No           No       No          No             No\n",
       "292   AM20731         Yes          Yes       No          No             No\n",
       "293   AM20732          No           No      Yes          No            Yes\n",
       "294   AM20733         Yes           No      Yes          No             No\n",
       "295   AM20841         Yes           No      Yes          No             No\n",
       "\n",
       "[296 rows x 6 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#7) Create another subset of this dataframe from 6) selecting only:  SubjectID and these variables: \n",
    "#SDOsteoDiag      SDRheumaDiag SDOADiag            SDLupusDiag      SDHyperthyDiag\n",
    "Subset = TablesJoin.loc[:,[\"SubjectID\", \"SDOsteoDiag\",'SDRheumaDiag','SDOADiag','SDLupusDiag','SDHyperthyDiag']]\n",
    "Subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "53d71176",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'Index' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-72-b1a6cfc8b73a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#8) Recode the yes and no's to 1's and 0's, respectively. Also recode missing data and Don't know to NA.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#Challenge yourself and use functions and loops rather than doing one at a time.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mlst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSubset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: 'Index' object is not callable"
     ]
    }
   ],
   "source": [
    "#8) Recode the yes and no's to 1's and 0's, respectively. Also recode missing data and Don't know to NA. \n",
    "#Challenge yourself and use functions and loops rather than doing one at a time. \n",
    "lst = list(Subset.columns)[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "21340285",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SDOsteoDiag</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Don't Know</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0        count\n",
       "SDOsteoDiag       \n",
       "Don't Know       2\n",
       "No             218\n",
       "Yes             73"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#9) Determine the frequencies of the response categories for these same variables that you've recoded\n",
    "pd.crosstab(index=Subset['SDOsteoDiag'], columns='count')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "193486c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SDRheumaDiag</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Don't Know</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0         count\n",
       "SDRheumaDiag       \n",
       "Don't Know        1\n",
       "No              272\n",
       "Yes              20"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(index=Subset['SDRheumaDiag'], columns='count')   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "fdc0f7ef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SDOADiag</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Don't Know</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0       count\n",
       "SDOADiag         \n",
       "Don't Know      2\n",
       "No            176\n",
       "Yes           116"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(index=Subset['SDOADiag'], columns='count')   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "01ed1e3c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SDLupusDiag</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing data</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0         count\n",
       "SDLupusDiag        \n",
       "No              290\n",
       "Yes               1\n",
       "missing data      1"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(index=Subset['SDLupusDiag'], columns='count')   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "46ed16ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SDHyperthyDiag</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0           count\n",
       "SDHyperthyDiag       \n",
       "No                287\n",
       "Yes                 5"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(index=Subset['SDHyperthyDiag'], columns='count')   \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}