reads_to_assemblies.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paths to data\n",
    "wd = '/Users/rosekantor/data/awtp2_metagenomics'\n",
    "wf = '/Users/rosekantor/work/awtp2/workflows'\n",
    "assem_dir = '/groups/banfield/projects/human/drinkingwater/assembly'\n",
    "reads_dir = '/groups/banfield/projects/human/drinkingwater/raw.d/trimmed_reads'\n",
    "mash_dir = '/groups/banfield/projects/human/drinkingwater/raw.d/mash_analysis'\n",
    "\n",
    "# paths to software\n",
    "bb = '/shared/software/bin/bbduk.sh -Xmx5g k=23 mink=11 hdist=1 tbo tpe t=2'\n",
    "sickle = '/shared/software/bin/sickle'\n",
    "fastqc = '/shared/software/bin/fastqc'\n",
    "adapters = '/shared/software/bbmap/v38.78/resources/adapters.fa'\n",
    "phiX = '/shared/software/bbmap/v38.78/resources/phix174_ill.ref.fa.gz'\n",
    "\n",
    "mash = '/shared/software/bin/mash'\n",
    "\n",
    "megahit = '/shared/software/bin/megahit'\n",
    "metaspades = '/shared/software/bin/spades/bin/metaspades.py'\n",
    "\n",
    "bt2 = '/shared/software/bin/bowtie2'\n",
    "ssam = '/shared/software/bin/shrinksam'\n",
    "\n",
    "anvip = 'anvi-profile --min-contig-length 1000 --skip-SNV-profiling -T 48'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DPWF', 'ECAWPC', 'Experimental', 'Full_Scale', 'Pipe_Loop'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import metadata\n",
    "tab = pd.read_csv(f'{wd}/metagenome_info_tables/metagenomics_sample_table.tsv', sep='\\t')\n",
    "tab['path_prefix'] = tab[['sample_code_partial', 'sample_id']].apply(lambda x: f'{assem_dir}/'+'/'.join(x)+'_', axis=1)\n",
    "tab['assem'] = tab[['sample_code_partial', 'sample_id']].apply(lambda x: f'{assem_dir}/'+'/'.join(x)+'_contigs_min1000.fa', axis=1)\n",
    "tab['r1'] = tab['sample_id'].apply(lambda x: f'{reads_dir}/'+x+'.PE.1.fastq.gz')\n",
    "tab['r2'] = tab['sample_id'].apply(lambda x: f'{reads_dir}/'+x+'.PE.2.fastq.gz')\n",
    "\n",
    "# change project_type for 'Loop_MWTP_inf' to be Full_Scale so that we can group by project_type later and MWTP will be with DWDS\n",
    "tab.loc[tab['location_code']=='Loop_MWTP_inf', ['project_type']] = 'Full_Scale'\n",
    "\n",
    "# there are no duplicates in sample_code_partial except for two controls. Renaming controls below:\n",
    "\n",
    "# add a column to tab with shorter names for the controls and NA for samples\n",
    "controls_names = pd.DataFrame.from_dict({'sample_id': ['KNLK_1', 'KNLK_2', 'KNLK_24', 'KNLK_3', 'KNLK_35', 'KNLK_4','KNLK_42', 'KNLK_57'],\n",
    "                                         'sample_name': ['neg_RO_bf', \n",
    "                                                          'neg_field_bf',\n",
    "                                                          'pos_ex_1',\n",
    "                                                          'neg_ex_1',\n",
    "                                                          'neg_field_1',\n",
    "                                                          'neg_field_2',\n",
    "                                                          'pos_ex_3',\n",
    "                                                          'pos_ex_2']})\n",
    "tab = tab.merge(controls_names, on='sample_id', how='left')\n",
    "\n",
    "# fill in na in sample_names with names from sample_code_partial\n",
    "tab.sample_name = tab.sample_name.fillna(tab.sample_code_partial)\n",
    "\n",
    "# what projects do we have?\n",
    "set(tab.project_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>read_names</th>\n",
       "      <th>sample_16S</th>\n",
       "      <th>order</th>\n",
       "      <th>band_in_gel_amplicons</th>\n",
       "      <th>sample_code_partial</th>\n",
       "      <th>Sequal_prep_elute_pooled</th>\n",
       "      <th>Duplicate.</th>\n",
       "      <th>need_optimization.</th>\n",
       "      <th>Has_Other_pos_sample</th>\n",
       "      <th>...</th>\n",
       "      <th>Extraction_Date</th>\n",
       "      <th>Ext_num_unique</th>\n",
       "      <th>Microconcentrated.</th>\n",
       "      <th>DNA_Ext_conc_ngperuL</th>\n",
       "      <th>kit</th>\n",
       "      <th>path_prefix</th>\n",
       "      <th>assem</th>\n",
       "      <th>r1</th>\n",
       "      <th>r2</th>\n",
       "      <th>sample_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KNLK_10</td>\n",
       "      <td>KNLK_10_S73</td>\n",
       "      <td>KNLK_SD021</td>\n",
       "      <td>73.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_282B_bulk_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/5/2018</td>\n",
       "      <td>42</td>\n",
       "      <td>Y</td>\n",
       "      <td>24.600</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_282B_bulk_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>KNLK_67</td>\n",
       "      <td>KNLK_67_S7</td>\n",
       "      <td>KNLK_SD159</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_282B_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>119</td>\n",
       "      <td>Y</td>\n",
       "      <td>31.800</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_282B_bulk_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>KNLK_41</td>\n",
       "      <td>KNLK_41_S60</td>\n",
       "      <td>KNLK_SD018</td>\n",
       "      <td>60.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_282B_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>120</td>\n",
       "      <td>Y</td>\n",
       "      <td>7.580</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_282B_bulk_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>KNLK_82</td>\n",
       "      <td>KNLK_82_S48</td>\n",
       "      <td>KNLK_SD141</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_282B_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>121</td>\n",
       "      <td>Y</td>\n",
       "      <td>2.650</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_282B_bulk_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>KNLK_61</td>\n",
       "      <td>KNLK_61_S46</td>\n",
       "      <td>KNLK_SD139</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_282B_bulk_6</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>122</td>\n",
       "      <td>Y</td>\n",
       "      <td>2.290</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_282B_bulk_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>KNLK_75</td>\n",
       "      <td>KNLK_75_S71</td>\n",
       "      <td>KNLK_SD153</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_286_bulk_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>109</td>\n",
       "      <td>Y</td>\n",
       "      <td>19.800</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_286_bulk_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>KNLK_71</td>\n",
       "      <td>KNLK_71_S39</td>\n",
       "      <td>KNLK_SD136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_286_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>111</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.530</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_286_bulk_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>KNLK_81</td>\n",
       "      <td>KNLK_81_S40</td>\n",
       "      <td>KNLK_SD137</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_286_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>112</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.740</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_286_bulk_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>KNLK_60</td>\n",
       "      <td>KNLK_60_S38</td>\n",
       "      <td>KNLK_SD135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_286_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>113</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.500</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_286_bulk_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>KNLK_27</td>\n",
       "      <td>KNLK_27_S35</td>\n",
       "      <td>KNLK_SD133</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_286_bulk_6</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>114</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.130</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_286_bulk_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>KNLK_17</td>\n",
       "      <td>KNLK_17_S42</td>\n",
       "      <td>KNLK_SD008</td>\n",
       "      <td>42.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_328_bulk_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>103</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.870</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_328_bulk_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>KNLK_18</td>\n",
       "      <td>KNLK_18_S50</td>\n",
       "      <td>KNLK_SD012</td>\n",
       "      <td>50.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_328_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>104</td>\n",
       "      <td>Y</td>\n",
       "      <td>3.140</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_328_bulk_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>KNLK_8</td>\n",
       "      <td>KNLK_8_S57</td>\n",
       "      <td>KNLK_SD015</td>\n",
       "      <td>57.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_328_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>105</td>\n",
       "      <td>Y</td>\n",
       "      <td>5.700</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_328_bulk_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>KNLK_36</td>\n",
       "      <td>KNLK_36_S20</td>\n",
       "      <td>KNLK_SD005</td>\n",
       "      <td>20.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_328_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>116</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.307</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_328_bulk_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>KNLK_53</td>\n",
       "      <td>KNLK_53_S69</td>\n",
       "      <td>KNLK_SD151</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_328_bulk_6</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/5/2018</td>\n",
       "      <td>43</td>\n",
       "      <td>Y</td>\n",
       "      <td>19.300</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_328_bulk_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNLK_15</td>\n",
       "      <td>KNLK_15_S26</td>\n",
       "      <td>KNLK_SD124</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_10</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>100</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.510</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>KNLK_48</td>\n",
       "      <td>KNLK_48_S29</td>\n",
       "      <td>KNLK_SD127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_11</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>101</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.650</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>KNLK_49</td>\n",
       "      <td>KNLK_49_S37</td>\n",
       "      <td>KNLK_SD134</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>106</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.310</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>KNLK_63</td>\n",
       "      <td>KNLK_63_S62</td>\n",
       "      <td>KNLK_SD020</td>\n",
       "      <td>62.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_6</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/5/2018</td>\n",
       "      <td>40</td>\n",
       "      <td>Y</td>\n",
       "      <td>8.860</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>KNLK_34</td>\n",
       "      <td>KNLK_34_S4</td>\n",
       "      <td>KNLK_SD121</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_8</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8/23/2018</td>\n",
       "      <td>108</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.577</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>KNLK_14</td>\n",
       "      <td>KNLK_14_S18</td>\n",
       "      <td>KNLK_SD003</td>\n",
       "      <td>18.0</td>\n",
       "      <td>+</td>\n",
       "      <td>DWDS_F_utility_bulk_9</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7/9/2018</td>\n",
       "      <td>99</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.252</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>DWDS_F_utility_bulk_9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id   read_names  sample_16S  order band_in_gel_amplicons  \\\n",
       "1    KNLK_10  KNLK_10_S73  KNLK_SD021   73.0                     +   \n",
       "63   KNLK_67   KNLK_67_S7  KNLK_SD159    NaN                     +   \n",
       "35   KNLK_41  KNLK_41_S60  KNLK_SD018   60.0                     +   \n",
       "80   KNLK_82  KNLK_82_S48  KNLK_SD141    NaN                     +   \n",
       "57   KNLK_61  KNLK_61_S46  KNLK_SD139    NaN                     +   \n",
       "72   KNLK_75  KNLK_75_S71  KNLK_SD153    NaN                     +   \n",
       "68   KNLK_71  KNLK_71_S39  KNLK_SD136    NaN                     +   \n",
       "79   KNLK_81  KNLK_81_S40  KNLK_SD137    NaN                     +   \n",
       "56   KNLK_60  KNLK_60_S38  KNLK_SD135    NaN                     +   \n",
       "19   KNLK_27  KNLK_27_S35  KNLK_SD133    NaN                     +   \n",
       "8    KNLK_17  KNLK_17_S42  KNLK_SD008   42.0                     +   \n",
       "9    KNLK_18  KNLK_18_S50  KNLK_SD012   50.0                     +   \n",
       "77    KNLK_8   KNLK_8_S57  KNLK_SD015   57.0                     +   \n",
       "29   KNLK_36  KNLK_36_S20  KNLK_SD005   20.0                     +   \n",
       "48   KNLK_53  KNLK_53_S69  KNLK_SD151    NaN                     +   \n",
       "6    KNLK_15  KNLK_15_S26  KNLK_SD124    NaN                     +   \n",
       "42   KNLK_48  KNLK_48_S29  KNLK_SD127    NaN                     +   \n",
       "43   KNLK_49  KNLK_49_S37  KNLK_SD134    NaN                     +   \n",
       "59   KNLK_63  KNLK_63_S62  KNLK_SD020   62.0                     +   \n",
       "27   KNLK_34   KNLK_34_S4  KNLK_SD121    NaN                     +   \n",
       "5    KNLK_14  KNLK_14_S18  KNLK_SD003   18.0                     +   \n",
       "\n",
       "       sample_code_partial Sequal_prep_elute_pooled  Duplicate.  \\\n",
       "1       DWDS_F_282B_bulk_1                        N       False   \n",
       "63      DWDS_F_282B_bulk_3                        N       False   \n",
       "35      DWDS_F_282B_bulk_4                        N       False   \n",
       "80      DWDS_F_282B_bulk_5                        N       False   \n",
       "57      DWDS_F_282B_bulk_6                        N       False   \n",
       "72       DWDS_F_286_bulk_1                        N       False   \n",
       "68       DWDS_F_286_bulk_3                        N       False   \n",
       "79       DWDS_F_286_bulk_4                        N       False   \n",
       "56       DWDS_F_286_bulk_5                        N       False   \n",
       "19       DWDS_F_286_bulk_6                        N       False   \n",
       "8        DWDS_F_328_bulk_1                        N       False   \n",
       "9        DWDS_F_328_bulk_3                        N       False   \n",
       "77       DWDS_F_328_bulk_4                        N       False   \n",
       "29       DWDS_F_328_bulk_5                        N       False   \n",
       "48       DWDS_F_328_bulk_6                        N       False   \n",
       "6   DWDS_F_utility_bulk_10                        N       False   \n",
       "42  DWDS_F_utility_bulk_11                        N       False   \n",
       "43   DWDS_F_utility_bulk_2                        N       False   \n",
       "59   DWDS_F_utility_bulk_6                        N       False   \n",
       "27   DWDS_F_utility_bulk_8                        N       False   \n",
       "5    DWDS_F_utility_bulk_9                        N       False   \n",
       "\n",
       "   need_optimization. Has_Other_pos_sample  ... Extraction_Date  \\\n",
       "1                   N                  NaN  ...        7/5/2018   \n",
       "63                  N                  NaN  ...       8/23/2018   \n",
       "35                  N                  NaN  ...       8/23/2018   \n",
       "80                  N                  NaN  ...       8/23/2018   \n",
       "57                  N                  NaN  ...       8/23/2018   \n",
       "72                  N                  NaN  ...       8/23/2018   \n",
       "68                  N                  NaN  ...       8/23/2018   \n",
       "79                  N                  NaN  ...       8/23/2018   \n",
       "56                  N                  NaN  ...       8/23/2018   \n",
       "19                  N                  NaN  ...       8/23/2018   \n",
       "8                   N                  NaN  ...        7/9/2018   \n",
       "9                   N                  NaN  ...        7/9/2018   \n",
       "77                  N                  NaN  ...        7/9/2018   \n",
       "29                  N                  NaN  ...       8/23/2018   \n",
       "48                  N                  NaN  ...        7/5/2018   \n",
       "6                   N                  NaN  ...        7/9/2018   \n",
       "42                  N                  NaN  ...        7/9/2018   \n",
       "43                  N                  NaN  ...        7/9/2018   \n",
       "59                  N                  NaN  ...        7/5/2018   \n",
       "27                  N                  NaN  ...       8/23/2018   \n",
       "5                   N                  NaN  ...        7/9/2018   \n",
       "\n",
       "   Ext_num_unique Microconcentrated. DNA_Ext_conc_ngperuL            kit  \\\n",
       "1              42                  Y               24.600  PowerSoil Pro   \n",
       "63            119                  Y               31.800  PowerSoil Pro   \n",
       "35            120                  Y                7.580  PowerSoil Pro   \n",
       "80            121                  Y                2.650  PowerSoil Pro   \n",
       "57            122                  Y                2.290  PowerSoil Pro   \n",
       "72            109                  Y               19.800  PowerSoil Pro   \n",
       "68            111                  Y                1.530  PowerSoil Pro   \n",
       "79            112                  Y                1.740  PowerSoil Pro   \n",
       "56            113                  Y                1.500  PowerSoil Pro   \n",
       "19            114                  Y                1.130  PowerSoil Pro   \n",
       "8             103                  Y                1.870  PowerSoil Pro   \n",
       "9             104                  Y                3.140  PowerSoil Pro   \n",
       "77            105                  Y                5.700  PowerSoil Pro   \n",
       "29            116                  Y                0.307  PowerSoil Pro   \n",
       "48             43                  Y               19.300  PowerSoil Pro   \n",
       "6             100                  Y                0.510  PowerSoil Pro   \n",
       "42            101                  Y                0.650  PowerSoil Pro   \n",
       "43            106                  Y                1.310  PowerSoil Pro   \n",
       "59             40                  Y                8.860  PowerSoil Pro   \n",
       "27            108                  Y                0.577  PowerSoil Pro   \n",
       "5              99                  Y                0.252  PowerSoil Pro   \n",
       "\n",
       "                                          path_prefix  \\\n",
       "1   /groups/banfield/projects/human/drinkingwater/...   \n",
       "63  /groups/banfield/projects/human/drinkingwater/...   \n",
       "35  /groups/banfield/projects/human/drinkingwater/...   \n",
       "80  /groups/banfield/projects/human/drinkingwater/...   \n",
       "57  /groups/banfield/projects/human/drinkingwater/...   \n",
       "72  /groups/banfield/projects/human/drinkingwater/...   \n",
       "68  /groups/banfield/projects/human/drinkingwater/...   \n",
       "79  /groups/banfield/projects/human/drinkingwater/...   \n",
       "56  /groups/banfield/projects/human/drinkingwater/...   \n",
       "19  /groups/banfield/projects/human/drinkingwater/...   \n",
       "8   /groups/banfield/projects/human/drinkingwater/...   \n",
       "9   /groups/banfield/projects/human/drinkingwater/...   \n",
       "77  /groups/banfield/projects/human/drinkingwater/...   \n",
       "29  /groups/banfield/projects/human/drinkingwater/...   \n",
       "48  /groups/banfield/projects/human/drinkingwater/...   \n",
       "6   /groups/banfield/projects/human/drinkingwater/...   \n",
       "42  /groups/banfield/projects/human/drinkingwater/...   \n",
       "43  /groups/banfield/projects/human/drinkingwater/...   \n",
       "59  /groups/banfield/projects/human/drinkingwater/...   \n",
       "27  /groups/banfield/projects/human/drinkingwater/...   \n",
       "5   /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                assem  \\\n",
       "1   /groups/banfield/projects/human/drinkingwater/...   \n",
       "63  /groups/banfield/projects/human/drinkingwater/...   \n",
       "35  /groups/banfield/projects/human/drinkingwater/...   \n",
       "80  /groups/banfield/projects/human/drinkingwater/...   \n",
       "57  /groups/banfield/projects/human/drinkingwater/...   \n",
       "72  /groups/banfield/projects/human/drinkingwater/...   \n",
       "68  /groups/banfield/projects/human/drinkingwater/...   \n",
       "79  /groups/banfield/projects/human/drinkingwater/...   \n",
       "56  /groups/banfield/projects/human/drinkingwater/...   \n",
       "19  /groups/banfield/projects/human/drinkingwater/...   \n",
       "8   /groups/banfield/projects/human/drinkingwater/...   \n",
       "9   /groups/banfield/projects/human/drinkingwater/...   \n",
       "77  /groups/banfield/projects/human/drinkingwater/...   \n",
       "29  /groups/banfield/projects/human/drinkingwater/...   \n",
       "48  /groups/banfield/projects/human/drinkingwater/...   \n",
       "6   /groups/banfield/projects/human/drinkingwater/...   \n",
       "42  /groups/banfield/projects/human/drinkingwater/...   \n",
       "43  /groups/banfield/projects/human/drinkingwater/...   \n",
       "59  /groups/banfield/projects/human/drinkingwater/...   \n",
       "27  /groups/banfield/projects/human/drinkingwater/...   \n",
       "5   /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                   r1  \\\n",
       "1   /groups/banfield/projects/human/drinkingwater/...   \n",
       "63  /groups/banfield/projects/human/drinkingwater/...   \n",
       "35  /groups/banfield/projects/human/drinkingwater/...   \n",
       "80  /groups/banfield/projects/human/drinkingwater/...   \n",
       "57  /groups/banfield/projects/human/drinkingwater/...   \n",
       "72  /groups/banfield/projects/human/drinkingwater/...   \n",
       "68  /groups/banfield/projects/human/drinkingwater/...   \n",
       "79  /groups/banfield/projects/human/drinkingwater/...   \n",
       "56  /groups/banfield/projects/human/drinkingwater/...   \n",
       "19  /groups/banfield/projects/human/drinkingwater/...   \n",
       "8   /groups/banfield/projects/human/drinkingwater/...   \n",
       "9   /groups/banfield/projects/human/drinkingwater/...   \n",
       "77  /groups/banfield/projects/human/drinkingwater/...   \n",
       "29  /groups/banfield/projects/human/drinkingwater/...   \n",
       "48  /groups/banfield/projects/human/drinkingwater/...   \n",
       "6   /groups/banfield/projects/human/drinkingwater/...   \n",
       "42  /groups/banfield/projects/human/drinkingwater/...   \n",
       "43  /groups/banfield/projects/human/drinkingwater/...   \n",
       "59  /groups/banfield/projects/human/drinkingwater/...   \n",
       "27  /groups/banfield/projects/human/drinkingwater/...   \n",
       "5   /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                   r2             sample_name  \n",
       "1   /groups/banfield/projects/human/drinkingwater/...      DWDS_F_282B_bulk_1  \n",
       "63  /groups/banfield/projects/human/drinkingwater/...      DWDS_F_282B_bulk_3  \n",
       "35  /groups/banfield/projects/human/drinkingwater/...      DWDS_F_282B_bulk_4  \n",
       "80  /groups/banfield/projects/human/drinkingwater/...      DWDS_F_282B_bulk_5  \n",
       "57  /groups/banfield/projects/human/drinkingwater/...      DWDS_F_282B_bulk_6  \n",
       "72  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_286_bulk_1  \n",
       "68  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_286_bulk_3  \n",
       "79  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_286_bulk_4  \n",
       "56  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_286_bulk_5  \n",
       "19  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_286_bulk_6  \n",
       "8   /groups/banfield/projects/human/drinkingwater/...       DWDS_F_328_bulk_1  \n",
       "9   /groups/banfield/projects/human/drinkingwater/...       DWDS_F_328_bulk_3  \n",
       "77  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_328_bulk_4  \n",
       "29  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_328_bulk_5  \n",
       "48  /groups/banfield/projects/human/drinkingwater/...       DWDS_F_328_bulk_6  \n",
       "6   /groups/banfield/projects/human/drinkingwater/...  DWDS_F_utility_bulk_10  \n",
       "42  /groups/banfield/projects/human/drinkingwater/...  DWDS_F_utility_bulk_11  \n",
       "43  /groups/banfield/projects/human/drinkingwater/...   DWDS_F_utility_bulk_2  \n",
       "59  /groups/banfield/projects/human/drinkingwater/...   DWDS_F_utility_bulk_6  \n",
       "27  /groups/banfield/projects/human/drinkingwater/...   DWDS_F_utility_bulk_8  \n",
       "5   /groups/banfield/projects/human/drinkingwater/...   DWDS_F_utility_bulk_9  \n",
       "\n",
       "[21 rows x 51 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab[tab.project_type == 'Full_Scale'].sort_values('sample_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>read_names</th>\n",
       "      <th>sample_16S</th>\n",
       "      <th>order</th>\n",
       "      <th>band_in_gel_amplicons</th>\n",
       "      <th>sample_code_partial</th>\n",
       "      <th>Sequal_prep_elute_pooled</th>\n",
       "      <th>Duplicate.</th>\n",
       "      <th>need_optimization.</th>\n",
       "      <th>Has_Other_pos_sample</th>\n",
       "      <th>...</th>\n",
       "      <th>Extraction_Date</th>\n",
       "      <th>Ext_num_unique</th>\n",
       "      <th>Microconcentrated.</th>\n",
       "      <th>DNA_Ext_conc_ngperuL</th>\n",
       "      <th>kit</th>\n",
       "      <th>path_prefix</th>\n",
       "      <th>assem</th>\n",
       "      <th>r1</th>\n",
       "      <th>r2</th>\n",
       "      <th>sample_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>KNLK_2</td>\n",
       "      <td>KNLK_2_S9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>-</td>\n",
       "      <td>EXP_0_field_biofilm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>/groups/banfield/projects/human/drinkingwater/...</td>\n",
       "      <td>neg_field_bf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id read_names sample_16S  order band_in_gel_amplicons  \\\n",
       "11    KNLK_2  KNLK_2_S9        NaN    9.0                     -   \n",
       "\n",
       "    sample_code_partial Sequal_prep_elute_pooled  Duplicate.  \\\n",
       "11  EXP_0_field_biofilm                      NaN       False   \n",
       "\n",
       "   need_optimization. Has_Other_pos_sample  ... Extraction_Date  \\\n",
       "11                  N                    N  ...             NaN   \n",
       "\n",
       "   Ext_num_unique Microconcentrated. DNA_Ext_conc_ngperuL  kit  \\\n",
       "11            NaN                NaN                  NaN  NaN   \n",
       "\n",
       "                                          path_prefix  \\\n",
       "11  /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                assem  \\\n",
       "11  /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                   r1  \\\n",
       "11  /groups/banfield/projects/human/drinkingwater/...   \n",
       "\n",
       "                                                   r2   sample_name  \n",
       "11  /groups/banfield/projects/human/drinkingwater/...  neg_field_bf  \n",
       "\n",
       "[1 rows x 51 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab[tab.sample_name=='neg_field_bf']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reads: trimming, QC, mash"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preliminary FastQC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# did this on the awtp samples, not everything.  Will check everything after trimming rather than before.\n",
    "# awtp =  tab[tab.project_type.isin(['DPWF', 'ECAWPC'])]\n",
    "\n",
    "# with open(f'{wd}/read_fastqc.sh', 'w') as f:\n",
    "#     for i in awtp.read_names:\n",
    "#         f.write(f'/opt/bin/bio/FastQC/fastqc -t 2 {i}.1.fastq.gz {i}.2.fastq.gz\\n')\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Manual examination of fastQC showed forward reads have adapter contamination at 3' end. Reverse reads have small amounts of low-quality sequence as indicated by over-represented sequences that were strings of G's. Trimming dealt with both of these successfully.\n",
    "\n",
    "A few samples look like they failed sequencing (high adapter contamination, weird per-base content and weird quality profiles)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trim bbmap, sickle, MASH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate through each filtered dataframe and make all the reads processing commands\n",
    "rawreads = '/groups/banfield/projects/human/drinkingwater/raw.d/KNKL'\n",
    "trimmedreads = '/groups/banfield/projects/human/drinkingwater/raw.d/trimmed_reads'\n",
    "trim_cmd = []\n",
    "mash_cmd = []\n",
    "for row in tab.itertuples():\n",
    "    r = row.read_names\n",
    "    s = row.sample_id\n",
    "    trimR = f'{bb} ref={adapters} ktrim=r ftm=5 -in1={rawreads}/{r}.1.fastq.gz -in2={rawreads}/{r}.2.fastq.gz -out1={rawreads}/{s}.1.noadapt.fastq.gz -out2={rawreads}/{s}.2.noadapt.fastq.gz'\n",
    "    #filtphiX = f'{bb} -Xmx5g t=2 k=31 hdist=1 -in1={s}.1.noadapt.fastq.gz -in2={s}.2.noadapt.fastq.gz out1={s}.1.nophix.fastq.gz out2={s}.2.nophix.fastq.gz ref={phiX} stats={s}_stats_phix.txt'\n",
    "    # based on testing, reads had no phiX, so this command isn't necessary\n",
    "    qtrim = f'sickle pe -l 75 -f {rawreads}/{s}.1.noadapt.fastq.gz -r {rawreads}/{s}.2.noadapt.fastq.gz -t sanger -o {trimmedreads}/{s}.PE.1.fastq -p {trimmedreads}/{s}.PE.2.fastq -s {trimmedreads}/{s}.SR.fastq'\n",
    "    clean = f'rm {rawreads}/{s}.1.noadapt.fastq.gz {rawreads}/{s}.2.noadapt.fastq.gz {trimmedreads}/{s}.SR.fastq'\n",
    "    qc = f'{fastqc} -t 2 {s}.PE.1.fastq {s}.PE.2.fastq'\n",
    "    gz1 = f'pigz -p 2 {trimmedreads}/{s}.PE.1.fastq'\n",
    "    gz2 = f'pigz -p 2 {trimmedreads}/{s}.PE.2.fastq'\n",
    "    cmd = [trimR, qtrim, clean, qc, gz1, gz2]\n",
    "    cmd = \"; \".join(cmd)\n",
    "    trim_cmd.append(cmd)\n",
    "    \n",
    "    # mash all v. all reads\n",
    "    cmd2 = f'cat {row.r1} {row.r2} | {mash} sketch -m 2 -r - -I {s} -s 10000 -o {mash_dir}/{s}'\n",
    "    mash_cmd.append(cmd2)\n",
    "    \n",
    "trim_cmd = pd.Series(trim_cmd, name='trim_cmd')\n",
    "mash_cmd = pd.Series(mash_cmd, name='mash_cmd')\n",
    "\n",
    "# add to tab\n",
    "tab['trim_cmd'] = trim_cmd\n",
    "tab['mash_cmd'] = mash_cmd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## mash dist commands\n",
    "`mash paste` combines multiple sketches into a single sketch.  The first arg is output name, followed by a list of all the sketch files you want to combine\n",
    "\n",
    "Command: `mash paste awtp2.msh *msh`\n",
    "\n",
    "`mash dist` can sketch on the fly or take a sketch as input.  Because we are doing all-vs-all we use the same msh file as the query and reference.\n",
    "\n",
    "Command: `mash dist awtp2.msh awtp2.msh`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assembly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Megahit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "megahit_cmd = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    r1 = row.r1\n",
    "    r2 = row.r2\n",
    "    assem_name = row.sample_code_partial\n",
    "    cmd = f'sbatch --wrap \"' \\\n",
    "          f'{megahit} -t 48 ' \\\n",
    "          f'-1 {row.r1} ' \\\n",
    "          f'-2 {row.r2} ' \\\n",
    "          f'-o {assem_dir}/{assem_name}\"'\n",
    "\n",
    "    megahit_cmd.append(cmd)\n",
    "megahit_cmd = pd.Series(megahit_cmd, name='megahit_cmd')\n",
    "tab['megahit_cmd'] = megahit_cmd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assembly post-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "awk = \"awk '{print $1}'\"\n",
    "\n",
    "postassem_cmd = [] # create empty list\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "\n",
    "    # replace the fasta headers that start \"k141\" with \"sample_id\" and rename file to sample_id_contigs.fa, simplify headers to just the first field\n",
    "    rehead = f\"sed 's/k141/{s}/g' {assem_dir}/{assem_name}/final.contigs.fa | {awk} > {assem_dir}/{assem_name}/{s}_contigs.fa\"\n",
    "\n",
    "    # get contig stats\n",
    "    cstats = f'contig_stats.pl -i {assem_dir}/{assem_name}/{s}_contigs.fa'\n",
    "\n",
    "    # filter for only contigs ≥1000 bp\n",
    "    min1000 = f'pullseq -i {assem_dir}/{assem_name}/{s}_contigs.fa --min 1000 > {assem_dir}/{assem_name}/{s}_contigs_min1000.fa'\n",
    "\n",
    "    # delete extra files from assembly\n",
    "    clean = f'rm -r {assem_dir}/{assem_name}/intermediate_contigs/ '\\\n",
    "            f'{assem_dir}/{assem_name}/checkpoints.txt '\\\n",
    "            f'{assem_dir}/{assem_name}/final.contigs.fa '\\\n",
    "            f'{assem_dir}/{assem_name}/done '\\\n",
    "            f'{assem_dir}/{assem_name}/options.json'\n",
    "\n",
    "    # make directory to store bowtie2 indices in\n",
    "    mdbt2 = f'mkdir {assem_dir}/{assem_name}/bt2/'\n",
    "\n",
    "    # index in prep for bowtie2 mapping\n",
    "    ind = f'bowtie2-build {assem_dir}/{assem_name}/{s}_contigs_min1000.fa {assem_dir}/{assem_name}/bt2/{s}_contigs_min1000.fa'\n",
    "\n",
    "    cmd = [rehead, cstats, min1000, clean, mdbt2, ind]\n",
    "    all_cmd = '; '.join(cmd) # separate all commands by semicolon (so they will be executed in order for each sample)\n",
    "    postassem_cmd.append(all_cmd) # append command to list\n",
    "    \n",
    "postassem_cmd = pd.Series(postassem_cmd) # from list to Series\n",
    "tab['postassem_cmd'] = postassem_cmd # add as a column to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Anvi'o process contigs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## update to add this to tab rather than separate table\n",
    "kaiju_path = '/groups/banfield/projects/human/drinkingwater/kaiju/bin'\n",
    "kaiju_nodes = '/groups/banfield/projects/human/drinkingwater/kaiju/nodes.dmp'\n",
    "kaiju_names = '/groups/banfield/projects/human/drinkingwater/kaiju/names.dmp'\n",
    "\n",
    "kaiju_nr = '/groups/banfield/projects/human/drinkingwater/kaiju/kaiju_db_nr_euk.fmi'\n",
    "kaiju = f'{kaiju_path}/kaiju -t {kaiju_nodes} -f {kaiju_nr}'\n",
    "kaiju_addtaxnames = f'{kaiju_path}/kaiju-addTaxonNames -t {kaiju_nodes} -n {kaiju_names} -r superkingdom,phylum,order,class,family,genus,species'\n",
    "\n",
    "contigs_df = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigs_min1000 = row.assem\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    gene_calls = f'{path_prefix}gene_calls.fa'\n",
    "    kaiju_out = f'{path_prefix}kaiju.out'\n",
    "    kaiju_processed = f'{path_prefix}genes_kaiju.txt'\n",
    "    \n",
    "    ## local initialize\n",
    "    make_cdb = f'anvi-gen-contigs-database -f {contigs_min1000} -o {contigsDB} -n {assem_name}'\n",
    "    get_genes = f'anvi-get-sequences-for-gene-calls -c {contigsDB} -o {gene_calls}'\n",
    "    makecdb_cmd = f'{make_cdb}; {get_genes}'\n",
    "    \n",
    "    ## cluster\n",
    "    anvhmms = f'anvi-run-hmms -c {contigsDB} -T 48'\n",
    "    anvscg = f'anvi-run-scg-taxonomy -c {contigsDB} -T 48'\n",
    "    #anvcogs = f'anvi-run-ncbi-cogs -c {contigsDB} -T 48'\n",
    "    run_kaiju = f'{kaiju} -i {gene_calls} -v -z 48 > {kaiju_out}' \n",
    "    analyzecdbCluster_cmd = f'sbatch --wrap \"{anvhmms}; {anvscg}; {run_kaiju}\"'\n",
    "    \n",
    "    ## local after cluster jobs\n",
    "    process_kaiju = f'{kaiju_addtaxnames} -i {kaiju_out} -o {kaiju_processed}' \n",
    "    import_kaiju = f'anvi-import-taxonomy-for-genes -i {kaiju_processed} -c {contigsDB} -p kaiju --just-do-it'\n",
    "    cdbAddTaxonomy_cmd = f'{process_kaiju}; {import_kaiju}'\n",
    "        \n",
    "    ## append all commands to table\n",
    "    contigs_cmd = [s, makecdb_cmd, analyzecdbCluster_cmd, cdbAddTaxonomy_cmd]\n",
    "    contigs_df.append(contigs_cmd)\n",
    "\n",
    "rownames = ['sample_id', 'makecdb_cmd', 'analyzecdbCluster_cmd', 'cdbAddTaxonomy_cmd']\n",
    "contigs_df = pd.DataFrame.from_records(contigs_df, columns=rownames)\n",
    "tab = tab.merge(contigs_df, on='sample_id')\n",
    "\n",
    "# Later annotations\n",
    "# blast vs CARD # do this later and use USEARCH\n",
    "# PFAM and TIGRFAM\n",
    "#anvi-import-functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write files\n",
    "project = 'DPWF' # 'Full_Scale'\n",
    "#tab[tab.project_type==project].makecdb_cmd.to_csv(f'{wf}/makecdb_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#tab[tab.project_type==project].analyzecdbCluster_cmd.to_csv(f'{wf}/analyzecdbCluster_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#tab[tab.project_type==project].cdbAddTaxonomy_cmd.to_csv(f'{wf}/cdbAddTaxonomy_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Contigs summary\n",
    "anvi-display-contigs-stats --report-as-text -o contig_stats.txt *contigs.db"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mapping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate decontamination mapping commands df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# within-project mapping\n",
    "project = 'DPWF' # 'Full_Scale'\n",
    "df = tab[tab.project_type==project]\n",
    "\n",
    "# get all rows of tab that are from the project of interest AND controls\n",
    "df_cont = tab[tab.project_type.isin([project, 'Experimental'])]\n",
    "\n",
    "mapping_df = []\n",
    "for srow in df.itertuples():  \n",
    "    s = srow.sample_id\n",
    "    assem_name = srow.sample_name\n",
    "    assem = srow.assem\n",
    "    path_prefix = srow.path_prefix\n",
    "    bt2ind = f'{assem_dir}/{assem_name}/bt2/{s}_contigs_min1000.fa'\n",
    "    \n",
    "    # vs other samples and controls\n",
    "    for rrow in df_cont.itertuples():\n",
    "        r = rrow.sample_id\n",
    "        rcode = rrow.sample_name\n",
    "        r1 = rrow.r1\n",
    "        r2 = rrow.r2\n",
    "        bam = f'{assem}-vs-{r}.bam'\n",
    "        filtered_bam_raw = f'{assem}-vs-{r}.filtered.raw.bam'\n",
    "        filtered_bam = f'{assem}-vs-{r}.filtered.bam'\n",
    "        \n",
    "        # mapping directly to bam\n",
    "        map_cmd = f'{bt2} -p 48 -x {bt2ind} -1 {r1} -2 {r2} --reorder | {ssam} -v | sambam > {bam}'\n",
    "        map_qcmd = f'sbatch --wrap \"{map_cmd}\"' \n",
    "        \n",
    "        # filter mapping - for decontamination, we want to use filtered mappings\n",
    "        filter_mapping = f'sbatch --wrap \"reformat.sh in={bam} out={filtered_bam_raw} editfilter=2 threads=48\"'\n",
    "        \n",
    "        # process mapping\n",
    "        sort = f'samtools sort -m 5G {filtered_bam_raw} > {filtered_bam}'\n",
    "        index = f'samtools index {filtered_bam}'\n",
    "        clean = f'rm {filtered_bam_raw}'\n",
    "        process_mapping = [sort, index, clean]\n",
    "        process_mapping = '; '.join(process_mapping) # make into a single command line\n",
    "        \n",
    "        # anvi-profile\n",
    "        profile_out = f'{assem_dir}/{assem_name}/anvio_data/{r}_profile' # check this\n",
    "        anvip_cmd = f'anvi-profile --min-contig-length 1000 --skip-SNV-profiling -T 48 -i {filtered_bam} -c {path_prefix}contigs.db -o {profile_out} -S {rcode}'\n",
    "        anvip_qcmd = f'sbatch --wrap \"{anvip_cmd}\"'\n",
    "        \n",
    "        ## append all commands to table\n",
    "        all_cmds = [s, project, assem_name, r, rcode, map_qcmd, filter_mapping, process_mapping, anvip_qcmd]\n",
    "        mapping_df.append(all_cmds)\n",
    "\n",
    "rownames = ['sample_id', 'project_type', 'sample_code', 'read_id', 'read_code', 'map_qcmd', 'filter_mapping', 'process_mapping', 'anvip_qcmd']\n",
    "mapping_df = pd.DataFrame.from_records(mapping_df, columns=rownames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get mapping commands for decontamination "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (excluding pos_ex_3 because 2 pos controls is enough)\n",
    "decontam_mappings = mapping_df[(mapping_df.read_code.str.contains('neg') | mapping_df.read_code.str.contains('pos')) \\\n",
    "                               & (mapping_df.read_code != 'pos_ex_3') \\\n",
    "                               & (mapping_df.read_code != 'neg_RO_bf') \\\n",
    "                               | (mapping_df.read_code == mapping_df.sample_code) # self mapping\n",
    "                              ]\n",
    "#decontam_mappings.map_qcmd.to_csv(f'{wf}/decontam_map_qcmd_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#decontam_mappings.filter_mapping.to_csv(f'{wf}/decontam_filter_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#decontam_mappings.process_mapping.to_csv(f'{wf}/decontam_process_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#decontam_mappings.anvip_qcmd.to_csv(f'{wf}/decontam_anvip_qcmd_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge and find contaminants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "anvi_merge = []\n",
    "find_contams = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    \n",
    "    anvim_cmd = f'anvi-merge {assem_dir}/{assem_name}/anvio_data/*_profile/PROFILE.db -c {contigsDB} -o {path_prefix}merged_decontam_profiles'\n",
    "    anvi_merge.append(anvim_cmd)\n",
    "    \n",
    "    profileDB = f'{path_prefix}merged_decontam_profiles/PROFILE.db'\n",
    "    find_contams_cmd = f'/home/rkantor/scripts/find_contams.py -p {profileDB} -c {contigsDB} -o contam -n {assem_name}'\n",
    "    find_contams.append(find_contams_cmd)\n",
    "    \n",
    "tab['anvi_merge_decontam'] = pd.Series(anvi_merge)\n",
    "tab['find_contams'] = pd.Series(find_contams)\n",
    "\n",
    "# write comands\n",
    "project = 'Full_Scale'\n",
    "anvim = tab[tab.project_type == project].anvi_merge_decontam\n",
    "find_contams = tab[tab.project_type == project].find_contams\n",
    "\n",
    "#anvim.to_csv(f'{wf}/decontam_anvim_{project}.sh',sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#find_contams.to_csv(f'{wf}/decontam_find_contams_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete bam files (except self-mappings)\n",
    "# bams_list = '/Users/rosekantor/work/awtp2/workflows/bams_list_DWDS.txt' #'/Users/rosekantor/work/awtp2/workflows/bams_list_AWTP2.txt'\n",
    "# bams = pd.read_csv(bams_list, names=['file'])\n",
    "# bams.index = bams.file\n",
    "# bams = bams.file.str.split(pat='_contigs_min1000.fa-vs-', expand=True)\n",
    "# bams.columns = ['samp', 'reads']\n",
    "# bams.samp = bams.samp.str.split('/', expand=True)[1]\n",
    "# bams.reads = bams.reads.str.split('.', expand=True)[0]\n",
    "# bams['file_name'] = bams.index\n",
    "\n",
    "# with open('/Users/rosekantor/work/awtp2/workflows/remove_bams_dwds.sh', 'w') as f:\n",
    "#     for row in bams.itertuples():\n",
    "#         if row.samp != row.reads:\n",
    "#             f.write(f'rm {row.file_name}\\n')\n",
    "#             f.write(f'rm {row.file_name}.bai\\n')\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split all contig dbs to visualize all contams removed and decide if it's okay\n",
    "cmds = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    profileDB = f'{path_prefix}merged_decontam_profiles/PROFILE.db'\n",
    "    collection = f'/groups/banfield/projects/human/drinkingwater/assembly/decontam_work/results/{assem_name}_contam_collection.txt'\n",
    "    new_anvi_data = f'{path_prefix}merged_decontam_split'\n",
    "    anviimp = f'anvi-import-collection -c {contigsDB} -p {profileDB} -C contam {collection}'\n",
    "    anvisplit = f'anvi-split -c {contigsDB} -p {profileDB} -C contam -o {new_anvi_data}'\n",
    "    \n",
    "    cmds.append('; '.join([anviimp, anvisplit]))\n",
    "    \n",
    "tab['split_contams'] = pd.Series(cmds)\n",
    "\n",
    "project = 'DPWF'\n",
    "split_contams = tab[tab.project_type==project].split_contams\n",
    "# split_contams.to_csv(f'{wf}/decontam_split_contams_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Testing\n",
    "## import the contams bin as a collection and then split it out as a new profile and cluster it to prep for anvi-interactive\n",
    "# print(f'anvi-import-collection -c KNLK_58_contigs.db -p KNLK_58_merged_profiles_filtered/PROFILE.db -C contam_f contam_f_collection.txt')\n",
    "## split out as new anvio db\n",
    "# print(f'anvi-split -c KNLK_58_contigs.db -p KNLK_58_merged_profiles_filtered/PROFILE.db -C contam_f -o KNLK_58_contam_f/')\n",
    "## add layers from the output of find_contams.py\n",
    "# print(f'anvi-import-misc-data -p KNLK_58_contam_f/contam/PROFILE.db -t items contam_f_scores.txt')\n",
    "## anvi-interactive to manually inspect - does it look different with filtered mapping?\n",
    "# print('anvi-interactive -p KNLK_58_contam_f/contam/PROFILE.db -c KNLK_58_contam_f/contam/CONTIGS.db')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross-mapping for binning\n",
    "\n",
    "**Full_Scale samples 282B have been run**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# within-project mapping\n",
    "project = 'Full_Scale' # 'Full_Scale'  #DPWF\n",
    "df = tab[tab.project_type==project]\n",
    "\n",
    "# get all rows of tab that are from the project of interest AND controls\n",
    "df_cont = tab[tab.project_type.isin([project])]\n",
    "\n",
    "xmapping_df = []\n",
    "for srow in df.itertuples():  \n",
    "    s = srow.sample_id\n",
    "    assem_name = srow.sample_name\n",
    "    assem = srow.assem\n",
    "    path_prefix = srow.path_prefix\n",
    "    bt2ind = f'{assem_dir}/{assem_name}/bt2/{s}_contigs_min1000.fa'\n",
    "    \n",
    "    # vs other samples and controls\n",
    "    for rrow in df_cont.itertuples():\n",
    "        r = rrow.sample_id\n",
    "        rcode = rrow.sample_name\n",
    "        r1 = rrow.r1\n",
    "        r2 = rrow.r2\n",
    "        bam = f'{assem}-vs-{r}.bam'\n",
    "        sorted_bam = f'{assem}-vs-{r}.sorted.bam'\n",
    "        \n",
    "        # mapping directly to bam\n",
    "        map_cmd = f'{bt2} -p 48 -x {bt2ind} -1 {r1} -2 {r2} --reorder | {ssam} -v | sambam > {bam}'\n",
    "        map_qcmd = f'sbatch --wrap \"{map_cmd}\"' \n",
    "        \n",
    "        # process mapping\n",
    "        sort = f'samtools sort -m 5G {bam} > {sorted_bam}'\n",
    "        index = f'samtools index {sorted_bam}'\n",
    "        clean = f'rm {bam}'\n",
    "        process_mapping = [sort, index, clean]\n",
    "        process_mapping = '; '.join(process_mapping) # make into a single command line\n",
    "        \n",
    "        # anvi-profile\n",
    "        profile_out = f'{assem_dir}/{assem_name}/anvio_data/{r}_profile' # check this\n",
    "        anvip_cmd = f'anvi-profile --min-contig-length 1000 -T 48 -i {sorted_bam} -c {path_prefix}contigs.db -o {profile_out} -S {rcode}'\n",
    "        anvip_qcmd = f'sbatch --wrap \"{anvip_cmd}\"'\n",
    "        \n",
    "        ## append all commands to table\n",
    "        all_cmds = [s, project, assem_name, r, rcode, map_qcmd, filter_mapping, process_mapping, anvip_qcmd]\n",
    "        xmapping_df.append(all_cmds)\n",
    "\n",
    "rownames = ['sample_id', 'project_type', 'sample_code', 'read_id', 'read_code', 'map_qcmd', 'filter_mapping', 'process_mapping', 'anvip_qcmd']\n",
    "xmapping_df = pd.DataFrame.from_records(xmapping_df, columns=rownames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first move all previously made anvio profiles into a decontam folder\n",
    "# for i in $(ls -d DWDS*/); do mkdir $i/anvio_data/decontam; mv $i/anvio_data/KNLK*/ $i/anvio_data/decontam/; done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# group by sample_code, save individually for ease of running jobs on cluster\n",
    "for location, df in xmapping_df.groupby('sample_code', as_index=False):\n",
    "    sub_df = df[df.read_code != df.sample_code] # self-mapping is already complete for DPWF and DWDS\n",
    "#     sub_df.map_qcmd.to_csv(f'{wf}/m1.map_qcmd_{project}_{location}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#     df.process_mapping.to_csv(f'{wf}/m2.process_{project}_{location}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n",
    "#     df.anvip_qcmd.to_csv(f'{wf}/m3.anvip_qcmd_{project}_{location}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "anvi_merge = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    anvim_cmd = f'anvi-merge {assem_dir}/{assem_name}/anvio_data/*_profile/PROFILE.db -c {contigsDB} -o {path_prefix}merged'\n",
    "    anvi_merge.append(anvim_cmd)    \n",
    "tab['anvi_merge_binning'] = pd.Series(anvi_merge)\n",
    "\n",
    "# write comands\n",
    "project = 'Full_Scale'\n",
    "anvim = tab[tab.project_type == project].anvi_merge_binning\n",
    "# anvim.to_csv(f'{wf}/anvim_{project}.sh',sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_biofilm_control_1/KNLK_1_'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab.path_prefix.values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "concoct_cmds = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    profileDB = f'{path_prefix}merged/PROFILE.db'\n",
    "    threads = 10\n",
    "\n",
    "    get_cococt_input = f'anvi-export-splits-and-coverages -p {profileDB} -c {contigsDB} -o {assem_dir}/{assem_name}/ -O {s}'\n",
    "    run_concoct = f'OMP_NUM_THREADS=10 OMP_THREAD_LIMIT={threads} concoct --coverage_file {assem_dir}/{assem_name}/{s}-COVs.txt --composition_file {assem_dir}/{assem_name}/{s}-SPLITS.fa -b {assem_dir}/{assem_name}/{s}_concoct -r 150 -c 20 -t {threads}'\n",
    "    \n",
    "    #this line doesn't work yet\n",
    "    #edit_concoct_out = f'''ruby -F\",\" -lane 'puts $F.join(\"\\tbin_\")' {path_prefix}concoct_clustering_gt1000.csv > {path_prefix}concoct_clustering_gt1000.tsv'''\n",
    "    \n",
    "    import_bins = f'anvi-import-collection -c {contigsDB} -p {profileDB} -C concoct {path_prefix}concoct_clustering_gt1000.tsv'\n",
    "    cmd = '; '.join([get_cococt_input, run_concoct, import_bins]) # edit_concoct_out,\n",
    "    concoct_cmds.append(cmd)\n",
    "tab['concoct_cmd'] = pd.Series(concoct_cmds)\n",
    "\n",
    "# write comands\n",
    "project = 'Full_Scale'\n",
    "concoct_cmd = tab[tab.project_type == project].concoct_cmd\n",
    "#concoct_cmd[78]\n",
    "\n",
    "concoct_cmd.to_csv(f'{wf}/concoct_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ruby -F\",\" -lane 'puts $F.join(\"\\tbin_\")' /groups/banfield/projects/human/drinkingwater/assembly/DWDS_F_282B_bulk_1/KNLK_10_concoct_clustering_gt1000.csv > /groups/banfield/projects/human/drinkingwater/assembly/DWDS_F_282B_bulk_1/KNLK_10_concoct_clustering_gt1000.tsv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "s = # Binning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "concoct --coverage_file /groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3//KNLK_58-COVs.txt --composition_file /groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3//KNLK_58-SPLITS.fa -b /groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3//KNLK_58_concoct -r 150 -c 24 -t 10\n"
     ]
    }
   ],
   "source": [
    "s = 'KNLK_58'\n",
    "assem_dir = '/groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3/'\n",
    "profileDB = '/groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3/KNLK_58_merged/PROFILE.db'\n",
    "contigsDB = '/groups/banfield/projects/human/drinkingwater/assembly/AWTP_2_RO2_bulk_3/KNLK_58_contigs.db'\n",
    "get_cococt_input = f'anvi-export-splits-and-coverages -p {profileDB} -c {contigsDB} -o {assem_dir} -O {s} --splits-mode'\n",
    "run_concoct = f'concoct --coverage_file {assem_dir}/{s}-COVs.txt --composition_file {assem_dir}/{s}-SPLITS.fa -b {assem_dir}/{s}_concoct -r 150 -c 24 -t 10'\n",
    "print(run_concoct)\n",
    "\n",
    "## process the output with 1-liner to make it importable into anvio\n",
    "# ruby -F',' -lane 'puts $F.join(\"\\tbin_\")' KNLK_58_concoct_clustering_gt1000.csv > KNLK_58_concoct_clustering_gt1000.tsv\n",
    "# output collection file: KNLK_58_concoct_clustering_gt1000.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['sample_id', 'read_names', 'sample_16S', 'order',\n",
       "       'band_in_gel_amplicons', 'sample_code_partial',\n",
       "       'Sequal_prep_elute_pooled', 'Duplicate.', 'need_optimization.',\n",
       "       'Has_Other_pos_sample', 'Pool_for_16S', 'F_primer', 'R_primer',\n",
       "       'F_R_primer', 'PCR_rxn_DNA_Total_ng', 'template_dilution_factor',\n",
       "       'DNA_Ext_conc_ng.uL', 'column', 'row', 'Genomic_DNA_size_1_bp',\n",
       "       'Genomic_DNA_RLU_1', 'Genomic_DNA_size_2_bp', 'Genomic_DNA_RLU_2',\n",
       "       'Genomic_DNA_2_plus_peaks', 'Genomic_DNA_3_plus_peaks', 'Notes_prep',\n",
       "       'count_raw_reads', 'barcode_forward', 'barcode_reverse', 'plate',\n",
       "       'well', 'metagenomic_sample', 'sample_code_full_and_batch',\n",
       "       'sample_date', 'batch_sample_date', 'project_type', 'sample_or_control',\n",
       "       'location_code', 'sample_type', 'sampler_name', 'Ext_batch',\n",
       "       'Extraction_Date', 'Ext_num_unique', 'Microconcentrated.',\n",
       "       'DNA_Ext_conc_ngperuL', 'kit', 'path_prefix', 'assem', 'r1', 'r2',\n",
       "       'sample_name', 'trim_cmd', 'mash_cmd', 'megahit_cmd', 'postassem_cmd',\n",
       "       'makecdb_cmd', 'analyzecdbCluster_cmd', 'cdbAddTaxonomy_cmd',\n",
       "       'count_mapped', 'anvi_merge_binning'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOT DONE\n",
    "# remove contams with anvi-split (figure out how to invert it first)\n",
    "# split all contig dbs to visualize all contams removed and decide if it's okay\n",
    "cmds = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem_name = row.sample_code_partial\n",
    "    path_prefix = row.path_prefix\n",
    "    contigsDB = f'{path_prefix}contigs.db'\n",
    "    profileDB = f'{path_prefix}merged_decontam_profiles/PROFILE.db'\n",
    "    collection = f'/groups/banfield/projects/human/drinkingwater/assembly/decontam_work/results/{assem_name}_contam_collection.txt'\n",
    "    new_anvi_data = f'{path_prefix}merged_decontam_split'\n",
    "    anviimp = f'anvi-import-collection -c {contigsDB} -p {profileDB} -C contam {collection}'\n",
    "    \n",
    "    # FIGURE OUT HOW TO INVERT THE COLLECTION, SPLIT THE UNBINNED\n",
    "    #anvisplit = f'anvi-split -c {contigsDB} -p {profileDB} --skip-hierarchical-clustering -C contam -o {new_anvi_data}'\n",
    "    \n",
    "    cmds.append('; '.join([anviimp, anvisplit]))\n",
    "    \n",
    "tab['split_contams'] = pd.Series(cmds)\n",
    "\n",
    "project = 'DPWF'\n",
    "split_contams = tab[tab.project_type==project].split_contams\n",
    "split_contams.to_csv(f'{wf}/decontam_split_contams_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize metagenomes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DONE - both proj # trimmed reads: seqkit stats *fastq.gz -T > trimmed_reads_stats.txt\n",
    "# DONE - both proj # megahit assembly, log file: grep 'contigs, total' */log > analysis_assemblies/assembly_stats.megahit.042720.txt\n",
    "# DONE - both proj # read mapping: samtools view -c -F 260\n",
    "# DONE - both proj # decontam data: compiled reports from find_contams.py\n",
    "# assembly min1000 after decontam: anvio\n",
    "# read-mapping after decontam: maybe there's a fancy samtools view -c that lets you exclude a list of contigs \n",
    "                            # (or use anvio after anvi-split- check that they are the same?)\n",
    "\n",
    "# % reads mapping to min1000 after decontam (# reads mapped to non contam contigs) / (total reads - reads mapped to contam contigs)\n",
    "\n",
    "#summarize commands:\n",
    "#anvi-display-contigs-stats --report-as-text -o contig_stats.txt *contigs.db\n",
    "project = 'Full_Scale'\n",
    "count_mapped = []\n",
    "for row in tab.itertuples():\n",
    "    s = row.sample_id\n",
    "    assem = row.assem\n",
    "    self_mapping = f'{assem}-vs-{s}.bam'\n",
    "    cmd = f'samtools view -c -F 260 {self_mapping} >> /groups/banfield/projects/human/drinkingwater/assembly/analysis_assemblies/reads_mapped_{project}.txt'\n",
    "    count_mapped.append(cmd)\n",
    "tab['count_mapped'] = count_mapped\n",
    "tab[tab.project_type == project].count_mapped.to_csv(f'{wf}/count_mapped_{project}.sh', sep='\\t', header=False, index=False, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resfam HMMsearches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in sample_1 sample_9 sample_33 sample_37 sample_38 sample_39 sample_65 sample_66 sample_67 sample_185 sample_187 sample_193\n",
    "do\n",
    "\n",
    "hmmsearch --tblout resfams.vs.$i.out --cut_ga --cpu 4 ~/ref_databases/resfams/Resfams.hmm /data4/other/awtp1/assembly/$i/idba_ud/prokka_annotation/prokka.faa\n",
    "rename.py -f resfams.vs.$i.out -d /data4/other/awtp1/assembly/$i/idba_ud/prokka_annotation/$i.prokka_ids-to-geneids.txt > resfams.vs.$i.renamed.out\n",
    "add_bins.py -s all_scaf2bin.txt -t resfams.vs.$i.renamed.out -g gene > resfams.vs.$i.binned.out\n",
    "\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# just running this for DPWF for now\n",
    "hmmsearch = '/usr/bin/hmmsearch'\n",
    "assem_dir = '/data2/other/knelson/assembly'\n",
    "resfams = '/home/rkantor/ref_databases/functional_genes_hmms/resfams/Resfams.hmm'\n",
    "with open(f'{wd}/workflows/resfams_DPWF.sh', 'w') as f:\n",
    "    for row in tab[tab.project_type=='DPWF'].itertuples():\n",
    "        assem_name = row.sample_code_partial\n",
    "        s = row.sample_id\n",
    "        cmd = f'{hmmsearch} --tblout {assem_dir}/{assem_name}/resfams.vs.{s}.out --cut_ga --cpu 4 {resfams} {assem_dir}/{assem_name}/{s}_contigs_min1000.fa.genes.faa'\n",
    "        f.write(cmd + '\\n')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#collecting resfams hits and running uclust on them to identify identical ones:\n",
    "assem_dir = '/data2/other/knelson/assembly'\n",
    "with open(f'{wd}/workflows/resfams_collect_DPWF.sh', 'w') as f:\n",
    "    for row in tab[tab.project_type=='DPWF'].itertuples():\n",
    "        \n",
    "        assem_name = row.sample_code_partial\n",
    "        s = row.sample_id\n",
    "        min1000 = f'{assem_dir}/{assem_name}/{s}_contigs_min1000.fa.genes.faa'\n",
    "        \n",
    "        resout = f'{assem_dir}/{assem_name}/resfams.vs.{s}.out'\n",
    "        collect_hits = f\"grep -v '^#' {resout} | \"\\\n",
    "                       \"awk '{print $1}' | \"\\\n",
    "                       f'pullseq -N -i {min1000} > {assem_dir}/{assem_name}/{s}.resfams_hits.faa'\n",
    "        f.write(collect_hits + '\\n')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cat AWTP_2_*/*resfam*faa > resfam_awtp2_hits_all.faa\n",
    "# usearch -sortbylength resfam_awtp2_hits_all.faa -fastaout resfam_awtp2_hits_all.sorted.faa\n",
    "# usearch -cluster_fast resfam_awtp2_hits_all.sorted.faa -id 0.99 -uc resfam_awtp2_hits_all.uc -threads 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>read_names</th>\n",
       "      <th>sample_16S</th>\n",
       "      <th>order</th>\n",
       "      <th>band_in_gel_amplicons</th>\n",
       "      <th>sample_code_partial</th>\n",
       "      <th>Sequal_prep_elute_pooled</th>\n",
       "      <th>Duplicate.</th>\n",
       "      <th>need_optimization.</th>\n",
       "      <th>Has_Other_pos_sample</th>\n",
       "      <th>...</th>\n",
       "      <th>sample_or_control</th>\n",
       "      <th>location_code</th>\n",
       "      <th>sample_type</th>\n",
       "      <th>sampler_name</th>\n",
       "      <th>Ext_batch</th>\n",
       "      <th>Extraction_Date</th>\n",
       "      <th>Ext_num_unique</th>\n",
       "      <th>Microconcentrated.</th>\n",
       "      <th>DNA_Ext_conc_ngperuL</th>\n",
       "      <th>kit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>KNLK_11</td>\n",
       "      <td>KNLK_11_S81</td>\n",
       "      <td>KNLK_SD028</td>\n",
       "      <td>81.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_BAC_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>BAC</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>2</td>\n",
       "      <td>6/16/2018</td>\n",
       "      <td>15</td>\n",
       "      <td>N</td>\n",
       "      <td>169.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>KNLK_13</td>\n",
       "      <td>KNLK_13_S10</td>\n",
       "      <td>KNLK_SD177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_biofilm_sep_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_BF_separator</td>\n",
       "      <td>biofilm</td>\n",
       "      <td>Rose_Kantor</td>\n",
       "      <td>19</td>\n",
       "      <td>9/27/2018</td>\n",
       "      <td>A69</td>\n",
       "      <td>N</td>\n",
       "      <td>0.071</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>KNLK_16</td>\n",
       "      <td>KNLK_16_S34</td>\n",
       "      <td>KNLK_SD132</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_MF_comb_bulk_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>MF_combined</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>18</td>\n",
       "      <td>9/8/2018</td>\n",
       "      <td>171</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.070</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>KNLK_23</td>\n",
       "      <td>KNLK_23_S3</td>\n",
       "      <td>KNLK_SD179</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_2stage</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>RO1</td>\n",
       "      <td>9/13/2018</td>\n",
       "      <td>A53</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.399</td>\n",
       "      <td>Powersoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>KNLK_28</td>\n",
       "      <td>KNLK_28_S43</td>\n",
       "      <td>KNLK_SD009</td>\n",
       "      <td>43.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_MF_comb_bulk_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>MF_combined</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>18</td>\n",
       "      <td>9/8/2018</td>\n",
       "      <td>172</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.910</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>KNLK_32</td>\n",
       "      <td>KNLK_32_S75</td>\n",
       "      <td>KNLK_SD023</td>\n",
       "      <td>75.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_BAC_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>BAC</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>Test_March11</td>\n",
       "      <td>3/11/2018</td>\n",
       "      <td>W5</td>\n",
       "      <td>supernatant</td>\n",
       "      <td>27.000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>KNLK_33</td>\n",
       "      <td>KNLK_33_S83</td>\n",
       "      <td>KNLK_SD030</td>\n",
       "      <td>83.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_inf_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>WW_3ary</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>1</td>\n",
       "      <td>6/15/2018</td>\n",
       "      <td>4</td>\n",
       "      <td>N</td>\n",
       "      <td>518.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>KNLK_38</td>\n",
       "      <td>KNLK_38_S36</td>\n",
       "      <td>KNLK_SD006</td>\n",
       "      <td>36.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_biofilm_sep_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_BF_separator</td>\n",
       "      <td>biofilm</td>\n",
       "      <td>Rose_Kantor</td>\n",
       "      <td>RO1</td>\n",
       "      <td>9/13/2018</td>\n",
       "      <td>A57</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.170</td>\n",
       "      <td>Powersoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>KNLK_43</td>\n",
       "      <td>KNLK_43_S76</td>\n",
       "      <td>KNLK_SD024</td>\n",
       "      <td>76.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_biofilm_ret_scrape_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_BF_retentate_scrapings</td>\n",
       "      <td>biofilm</td>\n",
       "      <td>Rose_Kantor</td>\n",
       "      <td>19</td>\n",
       "      <td>9/27/2018</td>\n",
       "      <td>199</td>\n",
       "      <td>N</td>\n",
       "      <td>30.500</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>KNLK_44</td>\n",
       "      <td>KNLK_44_S84</td>\n",
       "      <td>KNLK_SD031</td>\n",
       "      <td>84.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_inf_bulk_5</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>WW_3ary</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>1</td>\n",
       "      <td>6/15/2018</td>\n",
       "      <td>5</td>\n",
       "      <td>N</td>\n",
       "      <td>542.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>KNLK_46</td>\n",
       "      <td>KNLK_46_S13</td>\n",
       "      <td>KNLK_SD181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_bulk_6</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_2stage</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>RO1</td>\n",
       "      <td>9/13/2018</td>\n",
       "      <td>A55</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.187</td>\n",
       "      <td>Powersoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>KNLK_51</td>\n",
       "      <td>KNLK_51_S53</td>\n",
       "      <td>KNLK_SD142</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_MF_comb_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>MF_combined</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>18</td>\n",
       "      <td>9/8/2018</td>\n",
       "      <td>173</td>\n",
       "      <td>Y</td>\n",
       "      <td>4.850</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>KNLK_55</td>\n",
       "      <td>KNLK_55_S85</td>\n",
       "      <td>KNLK_SD032</td>\n",
       "      <td>85.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_biofilm_ret_scrape_1</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_BF_retentate_scrapings</td>\n",
       "      <td>biofilm</td>\n",
       "      <td>Rose_Kantor</td>\n",
       "      <td>2</td>\n",
       "      <td>6/16/2018</td>\n",
       "      <td>9</td>\n",
       "      <td>N</td>\n",
       "      <td>600.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>KNLK_58</td>\n",
       "      <td>KNLK_58_S22</td>\n",
       "      <td>KNLK_SD182</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_2stage</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>RO1</td>\n",
       "      <td>9/13/2018</td>\n",
       "      <td>A51</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.345</td>\n",
       "      <td>Powersoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>KNLK_66</td>\n",
       "      <td>KNLK_66_S86</td>\n",
       "      <td>KNLK_SD033</td>\n",
       "      <td>86.0</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_inf_bulk_3</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>WW_3ary</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>1</td>\n",
       "      <td>6/15/2018</td>\n",
       "      <td>3</td>\n",
       "      <td>N</td>\n",
       "      <td>680.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>KNLK_76</td>\n",
       "      <td>KNLK_76_S79</td>\n",
       "      <td>KNLK_SD160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_BAC_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>BAC</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>2</td>\n",
       "      <td>6/16/2018</td>\n",
       "      <td>14</td>\n",
       "      <td>N</td>\n",
       "      <td>85.600</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>KNLK_77</td>\n",
       "      <td>KNLK_77_S8</td>\n",
       "      <td>KNLK_SD161</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_inf_bulk_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>WW_3ary</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>1</td>\n",
       "      <td>6/15/2018</td>\n",
       "      <td>2</td>\n",
       "      <td>N</td>\n",
       "      <td>158.000</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>KNLK_80</td>\n",
       "      <td>KNLK_80_S32</td>\n",
       "      <td>KNLK_SD130</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_RO2_bulk_4</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>RO_2stage</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>Test_June2</td>\n",
       "      <td>6/2/2018</td>\n",
       "      <td>W12</td>\n",
       "      <td>supernatant</td>\n",
       "      <td>0.603</td>\n",
       "      <td>Powersoil Pro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>KNLK_9</td>\n",
       "      <td>KNLK_9_S65</td>\n",
       "      <td>KNLK_SD147</td>\n",
       "      <td>NaN</td>\n",
       "      <td>+</td>\n",
       "      <td>AWTP_2_BAC_bulk_2</td>\n",
       "      <td>N</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sample</td>\n",
       "      <td>BAC</td>\n",
       "      <td>DEUF</td>\n",
       "      <td>Scott_Miller</td>\n",
       "      <td>2</td>\n",
       "      <td>6/16/2018</td>\n",
       "      <td>13</td>\n",
       "      <td>N</td>\n",
       "      <td>17.500</td>\n",
       "      <td>PowerSoil Pro</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id   read_names  sample_16S  order band_in_gel_amplicons  \\\n",
       "2    KNLK_11  KNLK_11_S81  KNLK_SD028   81.0                     +   \n",
       "4    KNLK_13  KNLK_13_S10  KNLK_SD177    NaN                     +   \n",
       "7    KNLK_16  KNLK_16_S34  KNLK_SD132    NaN                     +   \n",
       "15   KNLK_23   KNLK_23_S3  KNLK_SD179    NaN                     +   \n",
       "20   KNLK_28  KNLK_28_S43  KNLK_SD009   43.0                     +   \n",
       "25   KNLK_32  KNLK_32_S75  KNLK_SD023   75.0                     +   \n",
       "26   KNLK_33  KNLK_33_S83  KNLK_SD030   83.0                     +   \n",
       "31   KNLK_38  KNLK_38_S36  KNLK_SD006   36.0                     +   \n",
       "37   KNLK_43  KNLK_43_S76  KNLK_SD024   76.0                     +   \n",
       "38   KNLK_44  KNLK_44_S84  KNLK_SD031   84.0                     +   \n",
       "40   KNLK_46  KNLK_46_S13  KNLK_SD181    NaN                     +   \n",
       "46   KNLK_51  KNLK_51_S53  KNLK_SD142    NaN                     +   \n",
       "50   KNLK_55  KNLK_55_S85  KNLK_SD032   85.0                     +   \n",
       "53   KNLK_58  KNLK_58_S22  KNLK_SD182    NaN                     +   \n",
       "62   KNLK_66  KNLK_66_S86  KNLK_SD033   86.0                     +   \n",
       "73   KNLK_76  KNLK_76_S79  KNLK_SD160    NaN                     +   \n",
       "74   KNLK_77   KNLK_77_S8  KNLK_SD161    NaN                     +   \n",
       "78   KNLK_80  KNLK_80_S32  KNLK_SD130    NaN                     +   \n",
       "85    KNLK_9   KNLK_9_S65  KNLK_SD147    NaN                     +   \n",
       "\n",
       "                sample_code_partial Sequal_prep_elute_pooled  Duplicate.  \\\n",
       "2                 AWTP_2_BAC_bulk_5                        N       False   \n",
       "4          AWTP_2_RO2_biofilm_sep_2                        N       False   \n",
       "7             AWTP_2_MF_comb_bulk_1                        N       False   \n",
       "15                AWTP_2_RO2_bulk_5                        N       False   \n",
       "20            AWTP_2_MF_comb_bulk_2                        N       False   \n",
       "25                AWTP_2_BAC_bulk_3                        N       False   \n",
       "26                AWTP_2_inf_bulk_4                        N       False   \n",
       "31         AWTP_2_RO2_biofilm_sep_1                        N       False   \n",
       "37  AWTP_2_RO2_biofilm_ret_scrape_2                        N       False   \n",
       "38                AWTP_2_inf_bulk_5                        N       False   \n",
       "40                AWTP_2_RO2_bulk_6                        N       False   \n",
       "46            AWTP_2_MF_comb_bulk_3                        N       False   \n",
       "50  AWTP_2_RO2_biofilm_ret_scrape_1                        N       False   \n",
       "53                AWTP_2_RO2_bulk_3                        N       False   \n",
       "62                AWTP_2_inf_bulk_3                        N       False   \n",
       "73                AWTP_2_BAC_bulk_4                        N       False   \n",
       "74                AWTP_2_inf_bulk_2                        N       False   \n",
       "78                AWTP_2_RO2_bulk_4                        N       False   \n",
       "85                AWTP_2_BAC_bulk_2                        N       False   \n",
       "\n",
       "   need_optimization. Has_Other_pos_sample  ... sample_or_control  \\\n",
       "2                   N                  NaN  ...            sample   \n",
       "4                   N                  NaN  ...            sample   \n",
       "7                   N                  NaN  ...            sample   \n",
       "15                  N                  NaN  ...            sample   \n",
       "20                  N                  NaN  ...            sample   \n",
       "25                  N                  NaN  ...            sample   \n",
       "26                  N                  NaN  ...            sample   \n",
       "31                  N                  NaN  ...            sample   \n",
       "37                  N                  NaN  ...            sample   \n",
       "38                  N                  NaN  ...            sample   \n",
       "40                  N                  NaN  ...            sample   \n",
       "46                  N                  NaN  ...            sample   \n",
       "50                  N                  NaN  ...            sample   \n",
       "53                  N                  NaN  ...            sample   \n",
       "62                  N                  NaN  ...            sample   \n",
       "73                  N                  NaN  ...            sample   \n",
       "74                  N                  NaN  ...            sample   \n",
       "78                  N                  NaN  ...            sample   \n",
       "85                  N                  NaN  ...            sample   \n",
       "\n",
       "                location_code sample_type  sampler_name     Ext_batch  \\\n",
       "2                         BAC        DEUF  Scott_Miller             2   \n",
       "4             RO_BF_separator     biofilm   Rose_Kantor            19   \n",
       "7                 MF_combined        DEUF  Scott_Miller            18   \n",
       "15                  RO_2stage        DEUF  Scott_Miller           RO1   \n",
       "20                MF_combined        DEUF  Scott_Miller            18   \n",
       "25                        BAC        DEUF  Scott_Miller  Test_March11   \n",
       "26                    WW_3ary        DEUF  Scott_Miller             1   \n",
       "31            RO_BF_separator     biofilm   Rose_Kantor           RO1   \n",
       "37  RO_BF_retentate_scrapings     biofilm   Rose_Kantor            19   \n",
       "38                    WW_3ary        DEUF  Scott_Miller             1   \n",
       "40                  RO_2stage        DEUF  Scott_Miller           RO1   \n",
       "46                MF_combined        DEUF  Scott_Miller            18   \n",
       "50  RO_BF_retentate_scrapings     biofilm   Rose_Kantor             2   \n",
       "53                  RO_2stage        DEUF  Scott_Miller           RO1   \n",
       "62                    WW_3ary        DEUF  Scott_Miller             1   \n",
       "73                        BAC        DEUF  Scott_Miller             2   \n",
       "74                    WW_3ary        DEUF  Scott_Miller             1   \n",
       "78                  RO_2stage        DEUF  Scott_Miller    Test_June2   \n",
       "85                        BAC        DEUF  Scott_Miller             2   \n",
       "\n",
       "    Extraction_Date  Ext_num_unique  Microconcentrated. DNA_Ext_conc_ngperuL  \\\n",
       "2         6/16/2018              15                   N              169.000   \n",
       "4         9/27/2018             A69                   N                0.071   \n",
       "7          9/8/2018             171                   Y                1.070   \n",
       "15        9/13/2018             A53                   Y                0.399   \n",
       "20         9/8/2018             172                   Y                1.910   \n",
       "25        3/11/2018              W5         supernatant               27.000   \n",
       "26        6/15/2018               4                   N              518.000   \n",
       "31        9/13/2018             A57                   Y                1.170   \n",
       "37        9/27/2018             199                   N               30.500   \n",
       "38        6/15/2018               5                   N              542.000   \n",
       "40        9/13/2018             A55                   Y                0.187   \n",
       "46         9/8/2018             173                   Y                4.850   \n",
       "50        6/16/2018               9                   N              600.000   \n",
       "53        9/13/2018             A51                   Y                0.345   \n",
       "62        6/15/2018               3                   N              680.000   \n",
       "73        6/16/2018              14                   N               85.600   \n",
       "74        6/15/2018               2                   N              158.000   \n",
       "78         6/2/2018             W12         supernatant                0.603   \n",
       "85        6/16/2018              13                   N               17.500   \n",
       "\n",
       "              kit  \n",
       "2   PowerSoil Pro  \n",
       "4             NaN  \n",
       "7   PowerSoil Pro  \n",
       "15  Powersoil Pro  \n",
       "20  PowerSoil Pro  \n",
       "25            NaN  \n",
       "26  PowerSoil Pro  \n",
       "31  Powersoil Pro  \n",
       "37  PowerSoil Pro  \n",
       "38  PowerSoil Pro  \n",
       "40  Powersoil Pro  \n",
       "46  PowerSoil Pro  \n",
       "50  PowerSoil Pro  \n",
       "53  Powersoil Pro  \n",
       "62  PowerSoil Pro  \n",
       "73  PowerSoil Pro  \n",
       "74  PowerSoil Pro  \n",
       "78  Powersoil Pro  \n",
       "85  PowerSoil Pro  \n",
       "\n",
       "[19 rows x 46 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab[tab.project_type=='DPWF']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BLAST of ARGs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "ref = '~/ref_databases/args/sul1_model_from_card.fasta'\n",
    "\n",
    "with open(f'{wd}/workflows/args_blast_DPWF.sh', 'w') as f:\n",
    "    for row in tab[tab.project_type=='DPWF'].itertuples(): \n",
    "        assem_name = row.sample_code_partial\n",
    "        s = row.sample_id\n",
    "        min1000 = f'{assem_dir}/{assem_name}/{s}_contigs_min1000.fa.genes.faa'\n",
    "        \n",
    "        blastout = f'{min1000}-vs-sul1.out'\n",
    "        cmd = f'usearch -ublast {min1000} -db {ref} -evalue 1e-5 -blast6out {blastout} -threads 2'\n",
    "        f.write(cmd + '\\n')\n",
    "f.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}