PART_1_GW9_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Purpose\n",
    "\n",
    "Characterize the sequences in Nef RM9 epitope. Processes a folder hierarchy of FASTQ files and saves Nef RM9 sequences, counts and statistics files to a corresponding folder hierarchy. This script will generate a tsv file with nucleotide sequences and the number of times they are present in the data set. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify dataset arguments\n",
    "\n",
    "Specify variables that need to be changed depending on the dataset and machine configuration. After these parameters are specified, the rest of the notebook should be runnable automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to FASTQ files to process\n",
    "# expects R1 and R2 paired samples\n",
    "FASTQ_FOLDER_PATH = 'Raw_FASTQs/'\n",
    "\n",
    "# path to folder containing barcode counts, intermediate files, and statistics\n",
    "OUTPUT_PATH = 'GW9_counts/'\n",
    "\n",
    "# path to bbmap\n",
    "BBMAP_PATH = 'bin/bbmap/'\n",
    "\n",
    "# path to reference file for read orientation\n",
    "REF_FASTA = 'ref/M33262.fasta'\n",
    "\n",
    "# 20bp of upstream and downstream sequences flanking Gag GW9\n",
    "UPSTREAM_FLANKING = 'TTGCAGCAGCCCAACAGAGG'\n",
    "DOWNSTREAM_FLANKING = 'AATTGTGGGAAAGAGGGACA'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions\n",
    "\n",
    "Define functions that will be used in the workflow to count barcodes from FASTQ files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import logger\n",
    "\n",
    "def print_status(status):\n",
    "    '''print timestamped status update'''\n",
    "    \n",
    "    from datetime import datetime\n",
    "    import logging\n",
    "    \n",
    "    log = logging.getLogger(__name__)\n",
    "    \n",
    "    print('--[' + datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '] ' + status + '--')\n",
    "    log.info(status)\n",
    "\n",
    "def create_temp_file():\n",
    "    '''create named temporary file\n",
    "    return temporary file object [0] and path to temporary file [1]'''\n",
    "\n",
    "    import pathlib\n",
    "    import tempfile\n",
    "    \n",
    "    temp = tempfile.NamedTemporaryFile()    \n",
    "    return [temp, temp.name]\n",
    "\n",
    "def create_directory(dir_path):\n",
    "    '''create directory at specified location if one does not already exist'''\n",
    "    \n",
    "    import os\n",
    "    \n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "        \n",
    "    return dir_path\n",
    "\n",
    "def copy_and_overwrite(from_path, to_path):\n",
    "    '''duplicate folder and file hierarchy'''\n",
    "    import os\n",
    "    import shutil\n",
    "     \n",
    "    print_status('Copying files from ' + from_path + ' to ' + to_path)\n",
    "    if os.path.exists(to_path):\n",
    "        shutil.rmtree(to_path)\n",
    "    shutil.copytree(from_path, to_path)\n",
    "    \n",
    "def find_files_in_path(search_path):\n",
    "    '''get list of files matching files in search_path'''\n",
    "    import glob\n",
    "    \n",
    "    f = glob.glob(search_path, recursive=True)\n",
    "    \n",
    "    return f\n",
    "\n",
    "def derive_file_name(source_file, find_string, replace_string):\n",
    "    '''create filename string '''\n",
    "    \n",
    "    f = source_file.replace(find_string, replace_string)\n",
    "    \n",
    "    return f\n",
    "\n",
    "def run_command(cmd_list, stdout_file = None, stderr_file = None):\n",
    "    '''run command with subprocess.call\n",
    "    if stdout or stderr arguments are passed, save to specified file\n",
    "    '''\n",
    "    \n",
    "    import subprocess\n",
    "    \n",
    "    print_status(' '.join(cmd_list)) # print status\n",
    "    \n",
    "    # if neither stdout or stderr specified\n",
    "    if stdout_file is None and stderr_file is None:\n",
    "        print(cmd_list)\n",
    "        subprocess.call(cmd_list)\n",
    "     \n",
    "    # if only stdout is specified\n",
    "    elif stdout_file is not None and stderr_file is None:\n",
    "        with open(stdout_file, 'w') as so:\n",
    "            subprocess.call(cmd_list, stdout = so)\n",
    "     \n",
    "    # if only stderr is specified\n",
    "    elif stdout_file is None and stderr_file is not None:\n",
    "        with open(stderr_file, 'w') as se:\n",
    "            subprocess.call(cmd_list, stderr = se)\n",
    "     \n",
    "    # if both stdout and stderr are specified\n",
    "    elif stdout_file is not None and stderr_file is not None:\n",
    "        with open(stdout_file, 'w') as so:\n",
    "            with open(stderr_file, 'w') as se:\n",
    "                subprocess.call(cmd_list, stdout = so, stderr = se)\n",
    "    \n",
    "    else: pass\n",
    "    \n",
    "def split_path(source_file):\n",
    "    '''get path to enclosing folder and basename for source_file'''\n",
    "    \n",
    "    import os\n",
    "    \n",
    "    source_file_path = os.path.dirname(source_file)\n",
    "    source_file_basename = os.path.basename(source_file)\n",
    "    \n",
    "    return [source_file_path, source_file_basename]\n",
    "\n",
    "def create_barcode_tsv(two_column_barcode_count_file, three_column_barcode_count_file, sample_name):\n",
    "    '''add a column with sample_name to barcode count TSV output by kmercountexact'''\n",
    "    \n",
    "    import pandas as pd \n",
    "    \n",
    "    # import file containing counts to pandas dataframe    \n",
    "    df = pd.read_csv(two_column_barcode_count_file, sep='\\t', names=['barcode_sequence', 'barcode_count'])\n",
    "    \n",
    "    # add column with sample name\n",
    "    df.insert(0, 'sample_name', sample_name)\n",
    "    \n",
    "    # export CSV with sample names added\n",
    "    df.to_csv(three_column_barcode_count_file, sep='\\t', index=False)\n",
    "    \n",
    "def remove_file(source_file):\n",
    "    '''remove file'''\n",
    "    \n",
    "    import os\n",
    "    \n",
    "    os.remove(source_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Workflow\n",
    "\n",
    "Count numbers of barcodes in FASTQ sequences by:\n",
    "\n",
    "1. Create folders to store barcode counts, intermediate files, and tools statistics\n",
    "1. Make dictionary of samples to process\n",
    "1. Iterate over every sample to process and:\n",
    "1. Quality trimming and merging R1 and R2 FASTQ sequences\n",
    "1. Extracting reads that contain the barcode\n",
    "1. Orient reads in same direction\n",
    "1. Trim reads to contain only the barcode\n",
    "1. Count number of times each barcode occurs\n",
    "1. Create TSV file with barcode counts for each sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.\n",
    "\n",
    "Checks if the output folder designated by the user exists, and creates it if not. The function then generates the output folder structure based on the input folder structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:16:42] Copying files from Raw_FASTQs/ to GW9_counts/--\n"
     ]
    }
   ],
   "source": [
    "## 1. Create folders to store barcode counts, intermediate files, and tools statistics ##\n",
    "\n",
    "copy_and_overwrite(from_path = FASTQ_FOLDER_PATH, to_path = OUTPUT_PATH)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. \n",
    "\n",
    "Creates a dictionary of sample names, where the key is the sample directory path and the value is the file name sans forward/reverse designator and file extension."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 2. Make dictionary of samples to process ##\n",
    "\n",
    "FASTQ_R1 = find_files_in_path(OUTPUT_PATH + '/**/*_R1_001.fastq.gz')\n",
    "\n",
    "SAMPLES = {}\n",
    "\n",
    "for i in FASTQ_R1:\n",
    "    \n",
    "    # get path to directory containing sample FASTQ\n",
    "    SAMPLE_DIR = split_path(i)[0]\n",
    "    \n",
    "    # get sample name\n",
    "    FASTQ_R1_BASENAME = split_path(i)[1]\n",
    "    SAMPLE_NAME = derive_file_name(FASTQ_R1_BASENAME, '_R1_001.fastq.gz', '')\n",
    "    \n",
    "    SAMPLES[i] = (SAMPLE_DIR, SAMPLE_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. \n",
    "\n",
    "Iterates over every sample in the dictionary to create a stats file and...\n",
    "\n",
    "### 4. \n",
    "\n",
    "Merges and quality trims the forward and reverse FASTQ reads using bbmerge. \n",
    "\n",
    "### 5. \n",
    "\n",
    "- Extracts reads containing the user-defined upstream flanking sequence, allowing for a 1 bp mismatch.\n",
    "- From those reads, extracts reads containing the user-defined downstream flanking sequence, again allowing for a 1 bp mismatch.\n",
    "\n",
    "### 6. \n",
    "\n",
    "- Maps reads to SIV virus reference using bbmap.\n",
    "- Using bbmap's splitsam utility, identifies plus and minus strands in the output sam file. Regenerates FASTQ files for plus and minus reads, and takes the reverse complement of the minus strands before concatenating the two files into a single FASTQ.\n",
    "\n",
    "### 7.\n",
    "\n",
    "Using bbduk, removes the upstream and downstream flanking sequences, again allowing for a 1 bp mismatch. By orienting the reads in Step 6, the upstream sequence can be left-trimmed and the downstream sequence can be right-trimmed without needing to account for reverse complements.\n",
    "\n",
    "### 8. \n",
    "\n",
    "Counts the number of times each 24mer barcode appears in each sample. By limiting to 24 bp, any PCR chimeras containing > 24 bp between flanking sequences are removed.\n",
    "\n",
    "### 9.\n",
    "\n",
    "Outputs a TSV file with barcode counts for each sequence, and removes temporary files created in the above steps. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:17:15] bin/bbmap/bbmerge.sh in=GW9_counts/966-53_S53_L001_R1_001.fastq.gz in2=GW9_counts/966-53_S53_L001_R2_001.fastq.gz out=GW9_counts/966-53_S53_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:17:32] bin/bbmap/bbduk.sh in=GW9_counts/966-53_S53_L001.merged.fastq.gz outm=GW9_counts/966-53_S53_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:17:35] bin/bbmap/bbduk.sh in=GW9_counts/966-53_S53_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-53_S53_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:17:36] bin/bbmap/bbmap.sh in=GW9_counts/966-53_S53_L001.containing_barcode.fastq.gz outm=GW9_counts/966-53_S53_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:17:42] bin/bbmap/splitsam.sh GW9_counts/966-53_S53_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp_6po49i3.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp86m10wc1.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpya0a4s3b.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-53_S53_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp_6po49i3.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp86m10wc1.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpya0a4s3b.sam']\n",
      "--[2019-09-30 16:17:43] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp_6po49i3.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpqgwcytj2.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp_6po49i3.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpqgwcytj2.fastq.gz']\n",
      "--[2019-09-30 16:17:43] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp86m10wc1.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpyc_pp7ds.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp86m10wc1.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpyc_pp7ds.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:17:44] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpqgwcytj2.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpyc_pp7ds.fastq.gz--\n",
      "--[2019-09-30 16:17:44] bin/bbmap/bbduk.sh in=GW9_counts/966-53_S53_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpl55zf1l1.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:17:44] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpl55zf1l1.fastq.gz out=GW9_counts/966-53_S53_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:17:45] bin/bbmap/kmercountexact.sh in=GW9_counts/966-53_S53_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdhqc44bw.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-53_S53_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdhqc44bw.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:17:45] bin/bbmap/bbmerge.sh in=GW9_counts/966-54_S54_L001_R1_001.fastq.gz in2=GW9_counts/966-54_S54_L001_R2_001.fastq.gz out=GW9_counts/966-54_S54_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:17:58] bin/bbmap/bbduk.sh in=GW9_counts/966-54_S54_L001.merged.fastq.gz outm=GW9_counts/966-54_S54_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:01] bin/bbmap/bbduk.sh in=GW9_counts/966-54_S54_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-54_S54_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:01] bin/bbmap/bbmap.sh in=GW9_counts/966-54_S54_L001.containing_barcode.fastq.gz outm=GW9_counts/966-54_S54_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:18:09] bin/bbmap/splitsam.sh GW9_counts/966-54_S54_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9msxdec7.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpa2f59fao.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbkwrv06l.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-54_S54_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9msxdec7.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpa2f59fao.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbkwrv06l.sam']\n",
      "--[2019-09-30 16:18:09] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9msxdec7.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmvw1k814.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9msxdec7.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmvw1k814.fastq.gz']\n",
      "--[2019-09-30 16:18:10] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpa2f59fao.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmprcepzq4u.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpa2f59fao.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmprcepzq4u.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:18:10] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmvw1k814.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmprcepzq4u.fastq.gz--\n",
      "--[2019-09-30 16:18:10] bin/bbmap/bbduk.sh in=GW9_counts/966-54_S54_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpf1_uc_ke.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:11] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpf1_uc_ke.fastq.gz out=GW9_counts/966-54_S54_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:18:11] bin/bbmap/kmercountexact.sh in=GW9_counts/966-54_S54_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp0rvcl8uz.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-54_S54_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp0rvcl8uz.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:18:12] bin/bbmap/bbmerge.sh in=GW9_counts/966-55_S55_L001_R1_001.fastq.gz in2=GW9_counts/966-55_S55_L001_R2_001.fastq.gz out=GW9_counts/966-55_S55_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:18:27] bin/bbmap/bbduk.sh in=GW9_counts/966-55_S55_L001.merged.fastq.gz outm=GW9_counts/966-55_S55_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:30] bin/bbmap/bbduk.sh in=GW9_counts/966-55_S55_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-55_S55_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:30] bin/bbmap/bbmap.sh in=GW9_counts/966-55_S55_L001.containing_barcode.fastq.gz outm=GW9_counts/966-55_S55_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:18:38] bin/bbmap/splitsam.sh GW9_counts/966-55_S55_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7uwmzsql.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxw2k46sw.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmptp7027b8.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-55_S55_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7uwmzsql.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxw2k46sw.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmptp7027b8.sam']\n",
      "--[2019-09-30 16:18:38] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7uwmzsql.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpv3i4lyde.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7uwmzsql.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpv3i4lyde.fastq.gz']\n",
      "--[2019-09-30 16:18:38] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxw2k46sw.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpz2kykt_k.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxw2k46sw.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpz2kykt_k.fastq.gz', 'rcomp=t']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:18:39] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpv3i4lyde.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpz2kykt_k.fastq.gz--\n",
      "--[2019-09-30 16:18:39] bin/bbmap/bbduk.sh in=GW9_counts/966-55_S55_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp52vmiu40.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:39] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp52vmiu40.fastq.gz out=GW9_counts/966-55_S55_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:18:40] bin/bbmap/kmercountexact.sh in=GW9_counts/966-55_S55_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpj7o4cvz9.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-55_S55_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpj7o4cvz9.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:18:40] bin/bbmap/bbmerge.sh in=GW9_counts/20065_S3_L001_R1_001.fastq.gz in2=GW9_counts/20065_S3_L001_R2_001.fastq.gz out=GW9_counts/20065_S3_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:18:53] bin/bbmap/bbduk.sh in=GW9_counts/20065_S3_L001.merged.fastq.gz outm=GW9_counts/20065_S3_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:55] bin/bbmap/bbduk.sh in=GW9_counts/20065_S3_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/20065_S3_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:18:56] bin/bbmap/bbmap.sh in=GW9_counts/20065_S3_L001.containing_barcode.fastq.gz outm=GW9_counts/20065_S3_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:19:03] bin/bbmap/splitsam.sh GW9_counts/20065_S3_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp07z2bofp.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9hos_uay.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmproymgws3.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/20065_S3_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp07z2bofp.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9hos_uay.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmproymgws3.sam']\n",
      "--[2019-09-30 16:19:03] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp07z2bofp.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpeei6vntk.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp07z2bofp.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpeei6vntk.fastq.gz']\n",
      "--[2019-09-30 16:19:03] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9hos_uay.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3aaswl1b.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9hos_uay.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3aaswl1b.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:19:04] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpeei6vntk.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3aaswl1b.fastq.gz--\n",
      "--[2019-09-30 16:19:04] bin/bbmap/bbduk.sh in=GW9_counts/20065_S3_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfsywkii9.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:04] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfsywkii9.fastq.gz out=GW9_counts/20065_S3_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:19:05] bin/bbmap/kmercountexact.sh in=GW9_counts/20065_S3_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq_6qpxm2.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/20065_S3_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq_6qpxm2.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:19:05] bin/bbmap/bbmerge.sh in=GW9_counts/966-62_S62_L001_R1_001.fastq.gz in2=GW9_counts/966-62_S62_L001_R2_001.fastq.gz out=GW9_counts/966-62_S62_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:19:22] bin/bbmap/bbduk.sh in=GW9_counts/966-62_S62_L001.merged.fastq.gz outm=GW9_counts/966-62_S62_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:25] bin/bbmap/bbduk.sh in=GW9_counts/966-62_S62_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-62_S62_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:25] bin/bbmap/bbmap.sh in=GW9_counts/966-62_S62_L001.containing_barcode.fastq.gz outm=GW9_counts/966-62_S62_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:19:33] bin/bbmap/splitsam.sh GW9_counts/966-62_S62_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfh1wvk5a.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvvotopev.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmps7e6t9ve.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-62_S62_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfh1wvk5a.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvvotopev.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmps7e6t9ve.sam']\n",
      "--[2019-09-30 16:19:34] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfh1wvk5a.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpoy3peb60.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpfh1wvk5a.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpoy3peb60.fastq.gz']\n",
      "--[2019-09-30 16:19:34] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvvotopev.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ozrlkvt.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvvotopev.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ozrlkvt.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:19:35] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpoy3peb60.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ozrlkvt.fastq.gz--\n",
      "--[2019-09-30 16:19:35] bin/bbmap/bbduk.sh in=GW9_counts/966-62_S62_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkszwr6xk.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:35] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkszwr6xk.fastq.gz out=GW9_counts/966-62_S62_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:19:36] bin/bbmap/kmercountexact.sh in=GW9_counts/966-62_S62_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp85r5nfq8.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-62_S62_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp85r5nfq8.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:19:36] bin/bbmap/bbmerge.sh in=GW9_counts/966-60_S60_L001_R1_001.fastq.gz in2=GW9_counts/966-60_S60_L001_R2_001.fastq.gz out=GW9_counts/966-60_S60_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:19:50] bin/bbmap/bbduk.sh in=GW9_counts/966-60_S60_L001.merged.fastq.gz outm=GW9_counts/966-60_S60_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:52] bin/bbmap/bbduk.sh in=GW9_counts/966-60_S60_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-60_S60_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:19:53] bin/bbmap/bbmap.sh in=GW9_counts/966-60_S60_L001.containing_barcode.fastq.gz outm=GW9_counts/966-60_S60_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:20:00] bin/bbmap/splitsam.sh GW9_counts/966-60_S60_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsqyi2hxs.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3ugkejxp.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpamr17ykn.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-60_S60_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsqyi2hxs.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3ugkejxp.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpamr17ykn.sam']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:20:00] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsqyi2hxs.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphtuquyml.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsqyi2hxs.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphtuquyml.fastq.gz']\n",
      "--[2019-09-30 16:20:01] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3ugkejxp.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq5iu24jm.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3ugkejxp.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq5iu24jm.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:20:01] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphtuquyml.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq5iu24jm.fastq.gz--\n",
      "--[2019-09-30 16:20:01] bin/bbmap/bbduk.sh in=GW9_counts/966-60_S60_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn7tevgea.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:02] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn7tevgea.fastq.gz out=GW9_counts/966-60_S60_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:20:02] bin/bbmap/kmercountexact.sh in=GW9_counts/966-60_S60_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp702j0bc2.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-60_S60_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp702j0bc2.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:20:03] bin/bbmap/bbmerge.sh in=GW9_counts/848-03_S3_L001_R1_001.fastq.gz in2=GW9_counts/848-03_S3_L001_R2_001.fastq.gz out=GW9_counts/848-03_S3_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:20:30] bin/bbmap/bbduk.sh in=GW9_counts/848-03_S3_L001.merged.fastq.gz outm=GW9_counts/848-03_S3_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:34] bin/bbmap/bbduk.sh in=GW9_counts/848-03_S3_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/848-03_S3_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:34] bin/bbmap/bbmap.sh in=GW9_counts/848-03_S3_L001.containing_barcode.fastq.gz outm=GW9_counts/848-03_S3_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:20:41] bin/bbmap/splitsam.sh GW9_counts/848-03_S3_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5xgxknew.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp81tqta86.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgwezgvgn.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/848-03_S3_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5xgxknew.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp81tqta86.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgwezgvgn.sam']\n",
      "--[2019-09-30 16:20:41] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5xgxknew.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp6lbrfb7k.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5xgxknew.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp6lbrfb7k.fastq.gz']\n",
      "--[2019-09-30 16:20:42] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp81tqta86.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp0d2pso57.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp81tqta86.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp0d2pso57.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:20:42] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp6lbrfb7k.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp0d2pso57.fastq.gz--\n",
      "--[2019-09-30 16:20:42] bin/bbmap/bbduk.sh in=GW9_counts/848-03_S3_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpetsp9enj.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:43] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpetsp9enj.fastq.gz out=GW9_counts/848-03_S3_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:20:43] bin/bbmap/kmercountexact.sh in=GW9_counts/848-03_S3_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpz_aahx37.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/848-03_S3_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpz_aahx37.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:20:43] bin/bbmap/bbmerge.sh in=GW9_counts/20063_S1_L001_R1_001.fastq.gz in2=GW9_counts/20063_S1_L001_R2_001.fastq.gz out=GW9_counts/20063_S1_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:20:51] bin/bbmap/bbduk.sh in=GW9_counts/20063_S1_L001.merged.fastq.gz outm=GW9_counts/20063_S1_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:52] bin/bbmap/bbduk.sh in=GW9_counts/20063_S1_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/20063_S1_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:20:53] bin/bbmap/bbmap.sh in=GW9_counts/20063_S1_L001.containing_barcode.fastq.gz outm=GW9_counts/20063_S1_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:20:58] bin/bbmap/splitsam.sh GW9_counts/20063_S1_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp4icy3l8m.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpcxas_acy.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpksex_6u4.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/20063_S1_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp4icy3l8m.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpcxas_acy.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpksex_6u4.sam']\n",
      "--[2019-09-30 16:20:59] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp4icy3l8m.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgad4b51r.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp4icy3l8m.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgad4b51r.fastq.gz']\n",
      "--[2019-09-30 16:20:59] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpcxas_acy.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2z3fx417.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpcxas_acy.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2z3fx417.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:20:59] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgad4b51r.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2z3fx417.fastq.gz--\n",
      "--[2019-09-30 16:20:59] bin/bbmap/bbduk.sh in=GW9_counts/20063_S1_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpi_el0m2s.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:21:00] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpi_el0m2s.fastq.gz out=GW9_counts/20063_S1_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:21:00] bin/bbmap/kmercountexact.sh in=GW9_counts/20063_S1_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpf6xr_4s8.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/20063_S1_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpf6xr_4s8.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:21:01] bin/bbmap/bbmerge.sh in=GW9_counts/966-61_S61_L001_R1_001.fastq.gz in2=GW9_counts/966-61_S61_L001_R2_001.fastq.gz out=GW9_counts/966-61_S61_L001.merged.fastq.gz qtrim=t ow=t--\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:21:18] bin/bbmap/bbduk.sh in=GW9_counts/966-61_S61_L001.merged.fastq.gz outm=GW9_counts/966-61_S61_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:21:21] bin/bbmap/bbduk.sh in=GW9_counts/966-61_S61_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-61_S61_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:21:21] bin/bbmap/bbmap.sh in=GW9_counts/966-61_S61_L001.containing_barcode.fastq.gz outm=GW9_counts/966-61_S61_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:21:28] bin/bbmap/splitsam.sh GW9_counts/966-61_S61_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpb5nc_2th.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdjyamxlp.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuj7ch7nu.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-61_S61_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpb5nc_2th.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdjyamxlp.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuj7ch7nu.sam']\n",
      "--[2019-09-30 16:21:29] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpb5nc_2th.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7bc28wra.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpb5nc_2th.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7bc28wra.fastq.gz']\n",
      "--[2019-09-30 16:21:29] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdjyamxlp.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmnwg7gwq.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdjyamxlp.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmnwg7gwq.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:21:30] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7bc28wra.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpmnwg7gwq.fastq.gz--\n",
      "--[2019-09-30 16:21:30] bin/bbmap/bbduk.sh in=GW9_counts/966-61_S61_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphhfb83f_.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:21:30] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphhfb83f_.fastq.gz out=GW9_counts/966-61_S61_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:21:31] bin/bbmap/kmercountexact.sh in=GW9_counts/966-61_S61_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpykmwm9em.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-61_S61_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpykmwm9em.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:21:31] bin/bbmap/bbmerge.sh in=GW9_counts/848-05_S5_L001_R1_001.fastq.gz in2=GW9_counts/848-05_S5_L001_R2_001.fastq.gz out=GW9_counts/848-05_S5_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:22:04] bin/bbmap/bbduk.sh in=GW9_counts/848-05_S5_L001.merged.fastq.gz outm=GW9_counts/848-05_S5_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:22:08] bin/bbmap/bbduk.sh in=GW9_counts/848-05_S5_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/848-05_S5_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:22:09] bin/bbmap/bbmap.sh in=GW9_counts/848-05_S5_L001.containing_barcode.fastq.gz outm=GW9_counts/848-05_S5_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:22:18] bin/bbmap/splitsam.sh GW9_counts/848-05_S5_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ixnrh4x.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbs71nxuu.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp374mg840.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/848-05_S5_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ixnrh4x.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbs71nxuu.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp374mg840.sam']\n",
      "--[2019-09-30 16:22:18] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ixnrh4x.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvatu_l7t.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp2ixnrh4x.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvatu_l7t.fastq.gz']\n",
      "--[2019-09-30 16:22:19] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbs71nxuu.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsln7o205.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpbs71nxuu.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsln7o205.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:22:20] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvatu_l7t.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpsln7o205.fastq.gz--\n",
      "--[2019-09-30 16:22:20] bin/bbmap/bbduk.sh in=GW9_counts/848-05_S5_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpwaq_97gi.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:22:20] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpwaq_97gi.fastq.gz out=GW9_counts/848-05_S5_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:22:21] bin/bbmap/kmercountexact.sh in=GW9_counts/848-05_S5_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpi740eymy.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/848-05_S5_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpi740eymy.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:22:21] bin/bbmap/bbmerge.sh in=GW9_counts/1253-19_S17_L001_R1_001.fastq.gz in2=GW9_counts/1253-19_S17_L001_R2_001.fastq.gz out=GW9_counts/1253-19_S17_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:22:37] bin/bbmap/bbduk.sh in=GW9_counts/1253-19_S17_L001.merged.fastq.gz outm=GW9_counts/1253-19_S17_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:22:39] bin/bbmap/bbduk.sh in=GW9_counts/1253-19_S17_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/1253-19_S17_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:22:39] bin/bbmap/bbmap.sh in=GW9_counts/1253-19_S17_L001.containing_barcode.fastq.gz outm=GW9_counts/1253-19_S17_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:22:45] bin/bbmap/splitsam.sh GW9_counts/1253-19_S17_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpw1ry4y6w.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmppux78flx.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5nlxdrwb.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/1253-19_S17_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpw1ry4y6w.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmppux78flx.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5nlxdrwb.sam']\n",
      "--[2019-09-30 16:22:45] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpw1ry4y6w.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpnff5_3es.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpw1ry4y6w.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpnff5_3es.fastq.gz']\n",
      "--[2019-09-30 16:22:45] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmppux78flx.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9epf21r4.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmppux78flx.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9epf21r4.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:22:46] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpnff5_3es.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9epf21r4.fastq.gz--\n",
      "--[2019-09-30 16:22:46] bin/bbmap/bbduk.sh in=GW9_counts/1253-19_S17_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmptn_3gygm.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:22:46] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmptn_3gygm.fastq.gz out=GW9_counts/1253-19_S17_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:22:47] bin/bbmap/kmercountexact.sh in=GW9_counts/1253-19_S17_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9qvarkx6.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/1253-19_S17_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9qvarkx6.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:22:47] bin/bbmap/bbmerge.sh in=GW9_counts/848-04_S4_L001_R1_001.fastq.gz in2=GW9_counts/848-04_S4_L001_R2_001.fastq.gz out=GW9_counts/848-04_S4_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:23:34] bin/bbmap/bbduk.sh in=GW9_counts/848-04_S4_L001.merged.fastq.gz outm=GW9_counts/848-04_S4_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:23:39] bin/bbmap/bbduk.sh in=GW9_counts/848-04_S4_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/848-04_S4_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:23:40] bin/bbmap/bbmap.sh in=GW9_counts/848-04_S4_L001.containing_barcode.fastq.gz outm=GW9_counts/848-04_S4_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:23:47] bin/bbmap/splitsam.sh GW9_counts/848-04_S4_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq0_blajl.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpiqqs2w9g.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkvnpm4on.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/848-04_S4_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq0_blajl.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpiqqs2w9g.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkvnpm4on.sam']\n",
      "--[2019-09-30 16:23:47] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq0_blajl.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdwi9kb3z.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq0_blajl.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdwi9kb3z.fastq.gz']\n",
      "--[2019-09-30 16:23:48] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpiqqs2w9g.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvw14i24a.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpiqqs2w9g.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvw14i24a.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:23:48] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdwi9kb3z.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvw14i24a.fastq.gz--\n",
      "--[2019-09-30 16:23:48] bin/bbmap/bbduk.sh in=GW9_counts/848-04_S4_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphupv7gyx.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:23:49] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphupv7gyx.fastq.gz out=GW9_counts/848-04_S4_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:23:49] bin/bbmap/kmercountexact.sh in=GW9_counts/848-04_S4_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuse_a42q.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/848-04_S4_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuse_a42q.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:23:50] bin/bbmap/bbmerge.sh in=GW9_counts/1253-18_S16_L001_R1_001.fastq.gz in2=GW9_counts/1253-18_S16_L001_R2_001.fastq.gz out=GW9_counts/1253-18_S16_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:24:02] bin/bbmap/bbduk.sh in=GW9_counts/1253-18_S16_L001.merged.fastq.gz outm=GW9_counts/1253-18_S16_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:03] bin/bbmap/bbduk.sh in=GW9_counts/1253-18_S16_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/1253-18_S16_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:04] bin/bbmap/bbmap.sh in=GW9_counts/1253-18_S16_L001.containing_barcode.fastq.gz outm=GW9_counts/1253-18_S16_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:24:09] bin/bbmap/splitsam.sh GW9_counts/1253-18_S16_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpagzi1z20.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphaccublh.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkvc4f2_a.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/1253-18_S16_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpagzi1z20.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphaccublh.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkvc4f2_a.sam']\n",
      "--[2019-09-30 16:24:09] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpagzi1z20.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmplhkplxrs.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpagzi1z20.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmplhkplxrs.fastq.gz']\n",
      "--[2019-09-30 16:24:09] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphaccublh.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkujh4qlq.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmphaccublh.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkujh4qlq.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:24:10] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmplhkplxrs.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpkujh4qlq.fastq.gz--\n",
      "--[2019-09-30 16:24:10] bin/bbmap/bbduk.sh in=GW9_counts/1253-18_S16_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpk9l7db43.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:10] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpk9l7db43.fastq.gz out=GW9_counts/1253-18_S16_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:24:11] bin/bbmap/kmercountexact.sh in=GW9_counts/1253-18_S16_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn60e4hsd.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/1253-18_S16_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn60e4hsd.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:24:11] bin/bbmap/bbmerge.sh in=GW9_counts/966-59_S59_L001_R1_001.fastq.gz in2=GW9_counts/966-59_S59_L001_R2_001.fastq.gz out=GW9_counts/966-59_S59_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:24:27] bin/bbmap/bbduk.sh in=GW9_counts/966-59_S59_L001.merged.fastq.gz outm=GW9_counts/966-59_S59_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:30] bin/bbmap/bbduk.sh in=GW9_counts/966-59_S59_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/966-59_S59_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:30] bin/bbmap/bbmap.sh in=GW9_counts/966-59_S59_L001.containing_barcode.fastq.gz outm=GW9_counts/966-59_S59_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:24:38] bin/bbmap/splitsam.sh GW9_counts/966-59_S59_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpzm9jdnen.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9sfg11i2.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq95kf7yz.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/966-59_S59_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpzm9jdnen.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9sfg11i2.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpq95kf7yz.sam']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--[2019-09-30 16:24:38] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpzm9jdnen.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgi854bir.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpzm9jdnen.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgi854bir.fastq.gz']\n",
      "--[2019-09-30 16:24:39] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9sfg11i2.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5ojc9khf.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9sfg11i2.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5ojc9khf.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:24:39] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpgi854bir.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp5ojc9khf.fastq.gz--\n",
      "--[2019-09-30 16:24:39] bin/bbmap/bbduk.sh in=GW9_counts/966-59_S59_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpo52zc69q.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:24:40] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpo52zc69q.fastq.gz out=GW9_counts/966-59_S59_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:24:41] bin/bbmap/kmercountexact.sh in=GW9_counts/966-59_S59_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmplg0idylv.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/966-59_S59_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmplg0idylv.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:24:41] bin/bbmap/bbmerge.sh in=GW9_counts/20064_S2_L001_R1_001.fastq.gz in2=GW9_counts/20064_S2_L001_R2_001.fastq.gz out=GW9_counts/20064_S2_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:25:02] bin/bbmap/bbduk.sh in=GW9_counts/20064_S2_L001.merged.fastq.gz outm=GW9_counts/20064_S2_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:05] bin/bbmap/bbduk.sh in=GW9_counts/20064_S2_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/20064_S2_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:05] bin/bbmap/bbmap.sh in=GW9_counts/20064_S2_L001.containing_barcode.fastq.gz outm=GW9_counts/20064_S2_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:25:15] bin/bbmap/splitsam.sh GW9_counts/20064_S2_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjf74x5e5.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdr5f9qir.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpabn7_hpd.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/20064_S2_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjf74x5e5.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdr5f9qir.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpabn7_hpd.sam']\n",
      "--[2019-09-30 16:25:15] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjf74x5e5.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjt_lftx7.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjf74x5e5.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjt_lftx7.fastq.gz']\n",
      "--[2019-09-30 16:25:15] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdr5f9qir.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuwzeee1i.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpdr5f9qir.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuwzeee1i.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:25:16] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjt_lftx7.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpuwzeee1i.fastq.gz--\n",
      "--[2019-09-30 16:25:16] bin/bbmap/bbduk.sh in=GW9_counts/20064_S2_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmps5mjh9qx.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:16] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmps5mjh9qx.fastq.gz out=GW9_counts/20064_S2_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:25:17] bin/bbmap/kmercountexact.sh in=GW9_counts/20064_S2_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn3s_bphp.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/20064_S2_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpn3s_bphp.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:25:17] bin/bbmap/bbmerge.sh in=GW9_counts/20066_S4_L001_R1_001.fastq.gz in2=GW9_counts/20066_S4_L001_R2_001.fastq.gz out=GW9_counts/20066_S4_L001.merged.fastq.gz qtrim=t ow=t--\n",
      "--[2019-09-30 16:25:27] bin/bbmap/bbduk.sh in=GW9_counts/20066_S4_L001.merged.fastq.gz outm=GW9_counts/20066_S4_L001.containing_upstream_flank.fastq.gz literal=TTGCAGCAGCCCAACAGAGG k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:29] bin/bbmap/bbduk.sh in=GW9_counts/20066_S4_L001.containing_upstream_flank.fastq.gz outm=GW9_counts/20066_S4_L001.containing_barcode.fastq.gz literal=AATTGTGGGAAAGAGGGACA k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:30] bin/bbmap/bbmap.sh in=GW9_counts/20066_S4_L001.containing_barcode.fastq.gz outm=GW9_counts/20066_S4_L001.mapped.sam ref=ref/M33262.fasta ow=t--\n",
      "--[2019-09-30 16:25:36] bin/bbmap/splitsam.sh GW9_counts/20066_S4_L001.mapped.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjyedv_l5.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3fizbwws.sam /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmprz9d1wu2.sam--\n",
      "['bin/bbmap/splitsam.sh', 'GW9_counts/20066_S4_L001.mapped.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjyedv_l5.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3fizbwws.sam', '/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmprz9d1wu2.sam']\n",
      "--[2019-09-30 16:25:36] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjyedv_l5.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxtmhro_1.fastq.gz--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpjyedv_l5.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxtmhro_1.fastq.gz']\n",
      "--[2019-09-30 16:25:36] bin/bbmap/reformat.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3fizbwws.sam out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvuj5whup.fastq.gz rcomp=t--\n",
      "['bin/bbmap/reformat.sh', 'in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp3fizbwws.sam', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvuj5whup.fastq.gz', 'rcomp=t']\n",
      "--[2019-09-30 16:25:37] cat /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpxtmhro_1.fastq.gz /var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmpvuj5whup.fastq.gz--\n",
      "--[2019-09-30 16:25:37] bin/bbmap/bbduk.sh in=GW9_counts/20066_S4_L001.oriented.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9z_50a85.fastq.gz literal=TTGCAGCAGCCCAACAGAGG ktrim=l rcomp=f k=20 hdist=1 ow=t--\n",
      "--[2019-09-30 16:25:37] bin/bbmap/bbduk.sh in=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp9z_50a85.fastq.gz out=GW9_counts/20066_S4_L001.barcodes.fastq.gz literal=AATTGTGGGAAAGAGGGACA ktrim=r rcomp=f k=20 hdist=1 minlength=27 maxlength=27 ow=t--\n",
      "--[2019-09-30 16:25:38] bin/bbmap/kmercountexact.sh in=GW9_counts/20066_S4_L001.barcodes.fastq.gz out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7r22grld.txt fastadump=f k=27 rcomp=f--\n",
      "['bin/bbmap/kmercountexact.sh', 'in=GW9_counts/20066_S4_L001.barcodes.fastq.gz', 'out=/var/folders/lz/b8xzxdf952bck9bb57k2d4yr0000gq/T/tmp7r22grld.txt', 'fastadump=f', 'k=27', 'rcomp=f']\n",
      "--[2019-09-30 16:25:38] bin/bbmap/bbmerge.sh in=GW9_counts/848-06_S6_L001_R1_001.fastq.gz in2=GW9_counts/848-06_S6_L001_R2_001.fastq.gz out=GW9_counts/848-06_S6_L001.merged.fastq.gz qtrim=t ow=t--\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-55603ade7137>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m                  \u001b[0;34m'qtrim=t'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                  'ow=t'],\n\u001b[0;32m---> 21\u001b[0;31m                  stderr_file = STATS_PATH + '/' + SAMPLE_NAME + '.merging.stats.txt')\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;31m## 5. Extracting reads that contain the barcode ##\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-e97e256c670a>\u001b[0m in \u001b[0;36mrun_command\u001b[0;34m(cmd_list, stdout_file, stderr_file)\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mstdout_file\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstderr_file\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstderr_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;31m# if both stdout and stderr are specified\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(timeout, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m    287\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpopenargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    288\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 289\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    290\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m             \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout, endtime)\u001b[0m\n\u001b[1;32m   1475\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m                             \u001b[0;32mbreak\u001b[0m  \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1477\u001b[0;31m                         \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1478\u001b[0m                         \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1479\u001b[0m                         \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m   1422\u001b[0m             \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1423\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1424\u001b[0;31m                 \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1425\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1426\u001b[0m                 \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "## 3. Iterate over every sample to process and: ##\n",
    "\n",
    "for key,val in SAMPLES.items():\n",
    "    \n",
    "    # get sample path and sample name\n",
    "    SAMPLE_PATH = val[0]\n",
    "    SAMPLE_NAME = val[1]\n",
    "    \n",
    "    # create directory to hold statistics files\n",
    "    STATS_PATH = create_directory(SAMPLE_PATH + '/stats')\n",
    "\n",
    "    ## 4. Quality trimming and merging R1 and R2 FASTQ sequences ##\n",
    "    \n",
    "    # merge and quality trim\n",
    "    run_command([BBMAP_PATH + 'bbmerge.sh',\n",
    "                 'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '_R1_001.fastq.gz',\n",
    "                 'in2=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '_R2_001.fastq.gz',\n",
    "                 'out=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.merged.fastq.gz',\n",
    "                 'qtrim=t',\n",
    "                 'ow=t'],\n",
    "                 stderr_file = STATS_PATH + '/' + SAMPLE_NAME + '.merging.stats.txt')\n",
    "    \n",
    "    ## 5. Extracting reads that contain the barcode ##\n",
    "    \n",
    "    # extract reads containing barcode\n",
    "    # bbduk command to find reads containing upstream flanking sequence   \n",
    "    run_command([BBMAP_PATH + 'bbduk.sh',\n",
    "                  'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.merged.fastq.gz',\n",
    "                  'outm=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_upstream_flank.fastq.gz',\n",
    "                  'literal=' + UPSTREAM_FLANKING,\n",
    "                  'k=20',\n",
    "                  'hdist=1',\n",
    "                  'ow=t'],\n",
    "                  stderr_file = STATS_PATH + '/' + SAMPLE_NAME + '.containing_upstream_flank.stats.txt')\n",
    "    \n",
    "    # bbduk command to find reads containing downstream flanking sequence \n",
    "    # (among those that contain upstream flanking sequence)\n",
    "    run_command([BBMAP_PATH + 'bbduk.sh',\n",
    "                  'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_upstream_flank.fastq.gz',\n",
    "                  'outm=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_barcode.fastq.gz',\n",
    "                  'literal=' + DOWNSTREAM_FLANKING,\n",
    "                  'k=20',\n",
    "                  'hdist=1',\n",
    "                  'ow=t'],\n",
    "                  stderr_file=STATS_PATH + '/' + SAMPLE_NAME + '.containing_barcode.stats.txt')\n",
    "    \n",
    "    ## 6. Orient reads in same direction ##title\n",
    "    \n",
    "    # orient reads by mapping to reference\n",
    "    # use SIV reference with accession M33262\n",
    "    run_command([BBMAP_PATH + 'bbmap.sh',\n",
    "                'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_barcode.fastq.gz',\n",
    "                'outm=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.mapped.sam',\n",
    "                'ref=' + REF_FASTA,\n",
    "                'ow=t'],\n",
    "                stderr_file=STATS_PATH + '/' + SAMPLE_NAME + '.mapping.stats.txt')\n",
    "    \n",
    "    # split mapped SAM file into plus, minus, and unmapped strand SAM temporary files\n",
    "    PLUS_STRAND_SAM = create_temp_file()\n",
    "    MINUS_STRAND_SAM = create_temp_file()\n",
    "    UNMAPPED_STRAND_SAM = create_temp_file()\n",
    "    \n",
    "    run_command([BBMAP_PATH + 'splitsam.sh',\n",
    "                SAMPLE_PATH + '/' + SAMPLE_NAME + '.mapped.sam',\n",
    "                PLUS_STRAND_SAM[1] + '.sam',\n",
    "                MINUS_STRAND_SAM[1] + '.sam',\n",
    "                UNMAPPED_STRAND_SAM[1] + '.sam'])    \n",
    "    \n",
    "    # reformat plus and minus strand SAM files to FASTQ\n",
    "    PLUS_STRAND_FASTQ = create_temp_file()\n",
    "    MINUS_STRAND_FASTQ = create_temp_file()    \n",
    "\n",
    "    run_command([BBMAP_PATH + 'reformat.sh',\n",
    "                'in=' + PLUS_STRAND_SAM[1] + '.sam',\n",
    "                'out=' + PLUS_STRAND_FASTQ[1] + '.fastq.gz'])\n",
    "                 \n",
    "    run_command([BBMAP_PATH + 'reformat.sh',\n",
    "                'in=' + MINUS_STRAND_SAM[1] + '.sam',\n",
    "                'out=' + MINUS_STRAND_FASTQ[1] + '.fastq.gz',\n",
    "                'rcomp=t'])\n",
    "    \n",
    "    # concatenate oriented plus and minus strand FASTQ\n",
    "    run_command(['cat',\n",
    "                PLUS_STRAND_FASTQ[1] + '.fastq.gz',\n",
    "                MINUS_STRAND_FASTQ[1] + '.fastq.gz'],\n",
    "               stdout_file = SAMPLE_PATH + '/' + SAMPLE_NAME + '.oriented.fastq.gz')\n",
    "    \n",
    "    ## 7. Trim reads to contain only the barcode ##\n",
    "    \n",
    "    # remove upstream flanking sequence from oriented reads\n",
    "    UPSTREAM_FLANK_REMOVED = create_temp_file()\n",
    "    \n",
    "    run_command([BBMAP_PATH + 'bbduk.sh',\n",
    "               'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.oriented.fastq.gz',\n",
    "               'out=' + UPSTREAM_FLANK_REMOVED[1] + '.fastq.gz',\n",
    "               'literal=' + UPSTREAM_FLANKING,\n",
    "               'ktrim=l',\n",
    "               'rcomp=f',\n",
    "               'k=20',\n",
    "               'hdist=1',\n",
    "               'ow=t'],\n",
    "               stderr_file = STATS_PATH + '/' + SAMPLE_NAME + '.upstream.flank.removal.stats.txt')\n",
    "    \n",
    "    # remove downstream flanking sequence from oriented reads\n",
    "    run_command([BBMAP_PATH + 'bbduk.sh',\n",
    "               'in=' + UPSTREAM_FLANK_REMOVED[1] + '.fastq.gz',\n",
    "               'out=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.barcodes.fastq.gz',\n",
    "               'literal=' + DOWNSTREAM_FLANKING,\n",
    "               'ktrim=r',\n",
    "               'rcomp=f',\n",
    "               'k=20',\n",
    "               'hdist=1',\n",
    "               'minlength=27',  \n",
    "               'maxlength=27',\n",
    "               'ow=t'],\n",
    "               stderr_file = STATS_PATH + '/' + SAMPLE_NAME + '.downstream.flank.removal.stats.txt')\n",
    "    \n",
    "    \n",
    "    ## 8. Count number of times each barcode occurs ##\n",
    "    \n",
    "    # count number of identical barcodes in each sample\n",
    "    \n",
    "    # create temporary count file\n",
    "    BARCODE_COUNT_TEMP = create_temp_file()\n",
    "    \n",
    "    run_command([BBMAP_PATH + 'kmercountexact.sh',\n",
    "          'in=' + SAMPLE_PATH + '/' + SAMPLE_NAME + '.barcodes.fastq.gz',\n",
    "          'out=' + BARCODE_COUNT_TEMP[1] + '.txt',\n",
    "          'fastadump=f',\n",
    "          'k=27',\n",
    "          'rcomp=f'])\n",
    "#took last argument out of this, which was 'ow=t'. I couldn't find this in the documentation\n",
    "    \n",
    "    ## 9. Create TSV file with barcode counts for each sequence ##\n",
    "    \n",
    "    # create three column barcode count file with sample name\n",
    "    create_barcode_tsv(BARCODE_COUNT_TEMP[1] + '.txt',\n",
    "                       SAMPLE_PATH + '/' + SAMPLE_NAME + '.Gag_GW9_barcode_counts.txt',\n",
    "                       SAMPLE_NAME)\n",
    "    \n",
    "    # remove temporary files\n",
    "    PLUS_STRAND_SAM[0].close()\n",
    "    PLUS_STRAND_FASTQ[0].close()\n",
    "    MINUS_STRAND_SAM[0].close()\n",
    "    MINUS_STRAND_FASTQ[0].close()\n",
    "    UNMAPPED_STRAND_SAM[0].close()\n",
    "    UPSTREAM_FLANK_REMOVED[0].close()\n",
    "    BARCODE_COUNT_TEMP\n",
    "    \n",
    "    # remove intermediate files\n",
    "    # copies of these already exist in 02-renamed_fastq/\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '_R1.fastq.gz')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '_R2.fastq.gz')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '.merged.fastq.gz')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_upstream_flank.fastq.gz')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '.containing_barcode.fastq.gz')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '.mapped.sam')\n",
    "    #remove_file(SAMPLE_PATH + '/' + SAMPLE_NAME + '.oriented.fastq.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}