log_file_to_pandas.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "dca0ade3e726a953b501b15e8e990130d2b7799f14cfd9f4271676035ebe5511"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/global/cscratch1/sd/ameza9/runs/210623_nightly_suite_single_precision/ocean/global_ocean/QU240/PHC/performance_test/forward/log.ocean.0000.out\n/global/cscratch1/sd/ameza9/runs/210623_nightly_suite_single_precision/ocean/global_ocean/QU240/PHC/performance_test/forward/log.ocean.0002.out\n/global/cscratch1/sd/ameza9/runs/210623_nightly_suite_single_precision/ocean/global_ocean/QU240/PHC/performance_test/forward/log.ocean.0003.out\n/global/cscratch1/sd/ameza9/runs/210623_nightly_suite_single_precision/ocean/global_ocean/QU240/PHC/performance_test/forward/log.ocean.0001.out\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "#gets .out file names and prints out the file path\n",
    "DIR_NAME = os.getcwd()\n",
    "for file in os.listdir(DIR_NAME):\n",
    "    if (file.endswith('.out')):\n",
    "        print(os.path.join(DIR_NAME, file))\n",
    "        # open(os.path.join(DIR_NAME, file))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "INDEX     timer_name                                            total       calls        min            max            avg      pct_tot   pct_par     par_eff\nhere\n1 total time                                           8.18714         1        8.18703        8.18714        8.18707   100.00       0.00       1.00\n2  initialize                                          5.72691         1        5.64663        5.72691        5.68617    69.95      69.95       0.99\n3   io_read                                            4.38475         2        1.47788        2.90507        2.18955    53.56      76.56       1.00\n4    analysis_bootstrap                                0.00008         1        0.00006        0.00008        0.00007     0.00       0.00       0.86\n5     init_read_mixedLayerDepths                       0.00004         1        0.00002        0.00004        0.00002     0.00      47.71       0.63\n3   reset_io_alarms                                    0.00011         2        0.00000        0.00010        0.00005     0.00       0.00       0.94\nhere\n3   diagnostic solve                                   0.02340         1        0.02148        0.02340        0.02241     0.29       0.41       0.96\nhere\n4    equation of state                                 0.00459         3        0.00112        0.00213        0.00148     0.06      19.60       0.97\n3   analysis_init                                      0.00020         1        0.00018        0.00020        0.00019     0.00       0.00       0.96\n4    init_mixedLayerDepths                             0.00014         1        0.00013        0.00014        0.00013     0.00      71.17       0.94\n2  analysis_compute_startup                            0.26951         1        0.18925        0.26951        0.23001     3.29       3.29       0.85\n3   compute_startup_mixedLayerDepths                   0.00038         1        0.00034        0.00038        0.00036     0.00       0.14       0.94\n2  io_shortwave                                        0.00010         4        0.00002        0.00004        0.00002     0.00       0.00       0.91\n2  io_read                                             0.00005         3        0.00001        0.00003        0.00002     0.00       0.00       0.97\n2  reset_io_alarms                                     0.00041        12        0.00000        0.00006        0.00003     0.00       0.00       0.98\n2  land_ice_build_arrays                               0.00000         3        0.00000        0.00000        0.00000     0.00       0.00       0.81\nhere\n2  gm bolus velocity                                   0.09386         3        0.02707        0.03145        0.02889     1.15       1.15       0.92\nhere\n2  time integration                                    1.83905         3        0.60384        0.62440        0.61203    22.46      22.46       1.00\nhere\n3   se timestep                                        1.83891         3        0.60379        0.62435        0.61198    22.46      99.99       1.00\nhere\n4    se prep                                           0.00801         3        0.00243        0.00268        0.00257     0.10       0.44       0.96\nhere\n4    se loop                                           1.60810         6        0.21418        0.32454        0.26728    19.64      87.45       1.00\nhere\n5     se halo diag obd                                 0.01922         6        0.00002        0.00445        0.00191     0.23       1.20       0.59\nhere\n5     se halo diag                                     0.00386         6        0.00048        0.00074        0.00061     0.05       0.24       0.95\nhere\n5     se bcl vel                                       0.24750         6        0.03546        0.04641        0.04120     3.02      15.39       1.00\nhere\n6      se bcl vel tend                                 0.15339         6        0.02214        0.02848        0.02399     1.87      61.98       0.94\n7       ocn_tend_vel                                   0.13852         6        0.01994        0.02599        0.02165     1.69      90.31       0.94\n8        bulk_ws                                       0.00129         6        0.00020        0.00023        0.00021     0.02       0.93       0.98\n8        coriolis                                      0.04603         6        0.00677        0.00786        0.00724     0.56      33.23       0.94\nhere\n8        vel vadv                                      0.00732         6        0.00111        0.00122        0.00116     0.09       5.29       0.95\nhere\n8        pressure grad                                 0.04113         6        0.00578        0.00798        0.00630     0.50      29.69       0.92\nhere\n8        vel hmix                                      0.02848         6        0.00398        0.00621        0.00446     0.35      20.56       0.94\nhere\n9         vel del2                                     0.00468         6        0.00067        0.00106        0.00075     0.06      16.42       0.97\nhere\n9         vel del4                                     0.02379         6        0.00330        0.00514        0.00371     0.29      83.53       0.94\nhere\n8        vel surface stress                            0.01137         6        0.00161        0.00229        0.00179     0.14       8.21       0.94\nhere\n6      bcl iters on linear Coriolis                    0.10874         9        0.01016        0.01345        0.01141     1.33      43.93       0.94\n7       ocn_fuperp                                     0.07390         9        0.00708        0.00851        0.00757     0.90      67.96       0.92\nhere\n7       se halo normalBaroclinicVelocity               0.02926         9        0.00027        0.00451        0.00210     0.36      26.91       0.64\nhere\n6      se halo barotropicForcing                       0.00082         6        0.00003        0.00015        0.00009     0.01       0.33       0.66\nhere\n5     se btr vel                                       0.47914         6        0.07903        0.08301        0.07981     5.85      29.80       1.00\nhere\n6      btr vel se init                                 0.00034         6        0.00004        0.00007        0.00005     0.00       0.07       0.92\nhere\n6      btr se subcycle loop                            0.47035         6        0.07759        0.08154        0.07835     5.75      98.17       1.00\nhere\n7       se halo subcycle                               0.02509       240        0.00003        0.00018        0.00007     0.31       5.33       0.64\nhere\n6      btr se norm                                     0.00023         6        0.00003        0.00006        0.00004     0.00       0.05       0.95\nhere\n6      se halo F and btr vel                           0.00075         6        0.00004        0.00020        0.00009     0.01       0.16       0.75\nhere\n6      btr se ssh verif                                0.00787         6        0.00120        0.00153        0.00127     0.10       1.64       0.97\nhere\n5     se thick tend                                    0.02518         6        0.00368        0.00435        0.00393     0.31       1.57       0.94\nhere\n6      thick vert trans vel top                        0.01343         6        0.00192        0.00239        0.00206     0.16      53.36       0.92\n6      ocn_tend_thick                                  0.01117         6        0.00166        0.00195        0.00177     0.14      44.36       0.95\n7       bulk_thick                                     0.00015         6        0.00002        0.00003        0.00002     0.00       1.31       0.97\nhere\n7       thick hadv                                     0.00884         6        0.00130        0.00155        0.00139     0.11      79.17       0.94\nhere\n7       thick vadv                                     0.00077         6        0.00012        0.00014        0.00013     0.01       6.88       0.99\nhere\n7       thick surface flux                             0.00140         6        0.00021        0.00025        0.00023     0.02      12.50       0.98\nhere\n5     se halo thickness                                0.00367         6        0.00009        0.00094        0.00039     0.04       0.23       0.64\nhere\n5     se tracer tend                                   0.75653         6        0.06471        0.18240        0.11749     9.24      47.04       0.93\n6      ocn_tend_tracer                                 0.75652         6        0.06471        0.18240        0.11749     9.24     100.00       0.93\nhere\n7       tracer adv                                     0.40852         9        0.03305        0.06194        0.04220     4.99      54.00       0.93\nhere\n7       tracer hmix                                    0.29409         9        0.02499        0.04032        0.03031     3.59      38.87       0.93\nhere\n8        tracer redi                                   0.29408         9        0.02499        0.04032        0.03031     3.59      99.99       0.93\n7       surface_tracer_flux                            0.00399         9        0.00036        0.00054        0.00042     0.05       0.53       0.95\n7       surface_tracer_runoff_flux                     0.00457         9        0.00040        0.00063        0.00048     0.06       0.60       0.94\nhere\n7       non-local flux from KPP                        0.02965         9        0.00274        0.00362        0.00321     0.36       3.92       0.97\nhere\n8        KPP input fields                              0.02008         9        0.00208        0.00233        0.00219     0.25      67.70       0.98\nhere\n9         equation of state                            0.01796         9        0.00187        0.00209        0.00196     0.22      89.45       0.98\nhere\n8        non-local flux                                0.00514         9        0.00044        0.00074        0.00054     0.06      17.32       0.94\nhere\n5     se halo tracers                                  0.09112         6        0.00021        0.02731        0.00892     1.11       5.67       0.59\nhere\n5     se loop fini                                     0.08274         6        0.00453        0.02324        0.01303     1.01       5.15       0.94\nhere\n6      diagnostic solve                                0.06241         3        0.01886        0.02157        0.01974     0.76      75.43       0.95\nhere\n7       equation of state                              0.01373         9        0.00113        0.00221        0.00149     0.17      21.99       0.98\nhere\n4    se implicit vert mix                              0.14322         3        0.04702        0.04783        0.04750     1.75       7.79       1.00\nhere\n5     diagnostic solve                                 0.06876         3        0.02104        0.02299        0.02187     0.84      48.01       0.95\nhere\n6      equation of state                               0.01373         9        0.00113        0.00212        0.00148     0.17      19.97       0.97\nhere\n5     vmix imp                                         0.07029         3        0.02199        0.02352        0.02264     0.86      49.08       0.97\nhere\n6      vmix coefs                                      0.04723         3        0.01476        0.01584        0.01530     0.58      67.19       0.97\nhere\n7       cvmix cell loop                                0.04512         3        0.01407        0.01514        0.01459     0.55      95.53       0.97\nhere\n7       tracer redi coef                               0.00044         3        0.00014        0.00015        0.00015     0.01       0.94       0.99\nhere\n6      CVMix avg                                       0.00162         3        0.00050        0.00055        0.00052     0.02       2.30       0.97\nhere\n6      vmix solve momentum                             0.01037         3        0.00307        0.00346        0.00324     0.13      14.75       0.94\nhere\n6      vmix solve tracers                              0.01106         3        0.00347        0.00371        0.00357     0.14      15.73       0.97\nhere\n7       vmix tracers tend imp loop                     0.00932         6        0.00131        0.00174        0.00150     0.11      84.28       0.96\nhere\n5     se vmix halos                                    0.00606         3        0.00075        0.00205        0.00149     0.07       4.23       0.74\nhere\n6      se vmix halos normalVelFld                      0.00462         3        0.00028        0.00157        0.00101     0.06      76.18       0.66\nhere\n4    se fini                                           0.08521         3        0.02639        0.02849        0.02734     1.04       4.63       0.96\nhere\n5     diagnostic solve                                 0.06779         3        0.02080        0.02261        0.02160     0.83      79.56       0.96\nhere\n6      equation of state                               0.01367         9        0.00113        0.00212        0.00147     0.17      20.17       0.97\nhere\n5     se final mpas reconstruct                        0.00512         3        0.00157        0.00172        0.00166     0.06       6.01       0.97\nhere\n5     reconstruct gm vecs                              0.01173         3        0.00383        0.00398        0.00388     0.14      13.77       0.99\n2  analysis_compute                                    0.00124         3        0.00038        0.00042        0.00039     0.02       0.02       0.96\n3   compute_mixedLayerDepths                           0.00105         3        0.00032        0.00036        0.00034     0.01      85.33       0.96\n2  analysis_restart                                    0.00008         3        0.00002        0.00003        0.00003     0.00       0.00       0.94\n3   restart_mixedLayerDepths                           0.00001         3        0.00000        0.00000        0.00000     0.00       6.23       0.89\n2  analysis_write                                      0.08679         3        0.02479        0.03259        0.02795     1.06       1.06       0.97\n3   write_mixedLayerDepths                             0.08629         3        0.02463        0.03243        0.02779     1.05      99.43       0.97\n3   reset_alarm_mixedLayerDepths                       0.00035         3        0.00011        0.00012        0.00011     0.00       0.40       0.98\n2  io_write                                            0.25600         9        0.00001        0.19651        0.02844     3.13       3.13       1.00\n2  analysis_finalize                                   0.00008         1        0.00006        0.00008        0.00007     0.00       0.00       0.86\n3   finalize_mixedLayerDepths                          0.00001         1        0.00001        0.00001        0.00001     0.00      10.15       0.78\n"
     ]
    }
   ],
   "source": [
    "# search = open(\"log.ocean.0000.out\")\n",
    "timer_name_string = \"timer_name\"\n",
    "num_bins = 0\n",
    "import csv \n",
    "\n",
    "copy_flag = False \n",
    "with open(\"log.ocean.0000.out\") as search:\n",
    "    with open(\"test_log.csv\", \"w\") as outfile:\n",
    "        csvwriter = csv.writer(outfile, delimiter=',', quotechar='\\\"')\n",
    "\n",
    "        for line in search:\n",
    "            \n",
    "            if timer_name_string in line:\n",
    "                copy_flag = True \n",
    "\n",
    "                # this is to get a nicer looking database \n",
    "                line = 'LEVEL ' + line \n",
    "            if copy_flag == True and line.strip(): #only edits lines with text on them\n",
    "                \n",
    "                #replaces all white space with comma \n",
    "                comma_sep = re.sub(\"\\s+\", \",\", line.strip()) \n",
    "                comma_sep = list(comma_sep.split(\",\"))\n",
    "                if timer_name_string in line: \n",
    "                    num_bins = len(comma_sep)\n",
    "                elif len(comma_sep) > num_bins:\n",
    "\n",
    "                    #this inserts underscores into timer names (e.g. some word -> some_word)\n",
    "                    #theres probably a better way to do this. \n",
    "                    for i in range((len(comma_sep) - num_bins) + 1, 1, -1):\n",
    "                        comma_sep[i - 1] = comma_sep[i - 1] + \"_\" + comma_sep[i]\n",
    "                        comma_sep.pop(i)\n",
    "\n",
    "                csvwriter.writerow(comma_sep) \n",
    "                print(line)\n",
    "\n",
    "            elif not line.strip(): # when we reach bottom of text file, stop parsing\n",
    "                copy_flag = False\n",
    "                print(line)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                            INDEX    total  calls      min      max      avg  \\\n",
       "timer_name                                                                     \n",
       "total_time                      1  8.18714      1  8.18703  8.18714  8.18707   \n",
       "initialize                      2  5.72691      1  5.64663  5.72691  5.68617   \n",
       "io_read                         3  4.38475      2  1.47788  2.90507  2.18955   \n",
       "analysis_bootstrap              4  0.00008      1  0.00006  0.00008  0.00007   \n",
       "init_read_mixedLayerDepths      5  0.00004      1  0.00002  0.00004  0.00002   \n",
       "\n",
       "                            pct_tot  pct_par  par_eff  \n",
       "timer_name                                             \n",
       "total_time                   100.00     0.00     1.00  \n",
       "initialize                    69.95    69.95     0.99  \n",
       "io_read                       53.56    76.56     1.00  \n",
       "analysis_bootstrap             0.00     0.00     0.86  \n",
       "init_read_mixedLayerDepths     0.00    47.71     0.63  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>INDEX</th>\n      <th>total</th>\n      <th>calls</th>\n      <th>min</th>\n      <th>max</th>\n      <th>avg</th>\n      <th>pct_tot</th>\n      <th>pct_par</th>\n      <th>par_eff</th>\n    </tr>\n    <tr>\n      <th>timer_name</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>total_time</th>\n      <td>1</td>\n      <td>8.18714</td>\n      <td>1</td>\n      <td>8.18703</td>\n      <td>8.18714</td>\n      <td>8.18707</td>\n      <td>100.00</td>\n      <td>0.00</td>\n      <td>1.00</td>\n    </tr>\n    <tr>\n      <th>initialize</th>\n      <td>2</td>\n      <td>5.72691</td>\n      <td>1</td>\n      <td>5.64663</td>\n      <td>5.72691</td>\n      <td>5.68617</td>\n      <td>69.95</td>\n      <td>69.95</td>\n      <td>0.99</td>\n    </tr>\n    <tr>\n      <th>io_read</th>\n      <td>3</td>\n      <td>4.38475</td>\n      <td>2</td>\n      <td>1.47788</td>\n      <td>2.90507</td>\n      <td>2.18955</td>\n      <td>53.56</td>\n      <td>76.56</td>\n      <td>1.00</td>\n    </tr>\n    <tr>\n      <th>analysis_bootstrap</th>\n      <td>4</td>\n      <td>0.00008</td>\n      <td>1</td>\n      <td>0.00006</td>\n      <td>0.00008</td>\n      <td>0.00007</td>\n      <td>0.00</td>\n      <td>0.00</td>\n      <td>0.86</td>\n    </tr>\n    <tr>\n      <th>init_read_mixedLayerDepths</th>\n      <td>5</td>\n      <td>0.00004</td>\n      <td>1</td>\n      <td>0.00002</td>\n      <td>0.00004</td>\n      <td>0.00002</td>\n      <td>0.00</td>\n      <td>47.71</td>\n      <td>0.63</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 80
    }
   ],
   "source": [
    "data = pd.read_csv(\"test_log.csv\")\n",
    "import matplotlib.pyplot as plt\n",
    "#plotting using timer data as main index \n",
    "timer_data = data.set_index(['timer_name'])\n",
    "timer_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-695f142492a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test_log.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(\"test_log.csv\")\n",
    "import matplotlib.pyplot as plt\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}