tested and updated all notebooks

MD2Korg · Jun 26, 2020 · 2cee5e6 · 2cee5e6
1 parent 9d83705
commit 2cee5e6
Show file tree

Hide file tree

Showing 4 changed files with 754 additions and 4,815 deletions.
diff --git a/jupyter_demo/cc_algorithms.ipynb b/jupyter_demo/cc_algorithms.ipynb
@@ -75,7 +75,7 @@
     "%reload_ext autoreload\n",
     "from util.dependencies import *\n",
     "\n",
-    "CC = Kernel(\"/home/md2k/cc_conf/\", study_name=\"default\")"
+    "CC = Kernel(\"/home/jovyan/cc_conf/\", study_name=\"default\")"
    ]
   },
   {
@@ -87,49 +87,13 @@
     "This example utilizes a data generator to protect the privacy of real participants and allows for anyone utilizing this system to explore the data without required institutional review board approvals. This is disabled for this demonstration to not create too much data at once."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "gen_location_datastream(CC, user_id=USER_ID, stream_name=\"GPS--org.md2k.phonesensor--PHONE\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "toc-hr-collapsed": false
-   },
-   "source": [
-    "## Get stream data\n",
-    "Read the demo GPS stream and show some example values.  A typical GPS sample contains values for _latitude, longitude, altitude, speed, bearing, and accuracy_."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+--------------------+\n",
-      "|          timestamp|          localtime|          latitude|         longitude|altitude|   speed|   bearing| accuracy|version|                user|\n",
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+--------------------+\n",
-      "|2019-09-01 22:26:59|2019-09-02 03:26:59|35.157442557926856| -90.0349952849216|      93|2.834021|105.992312|17.288347|      1|00000000-afb8-476...|\n",
-      "|2019-09-01 22:27:59|2019-09-02 03:27:59|  35.1561572521825|-90.03681018833898|      94|3.796146|  8.256657|15.215249|      1|00000000-afb8-476...|\n",
-      "|2019-09-01 22:28:59|2019-09-02 03:28:59| 35.15747745895255|-90.03574140915266|      93| 1.11443|   4.37211|11.172609|      1|00000000-afb8-476...|\n",
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+--------------------+\n",
-      "only showing top 3 rows\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "gps_stream = CC.get_stream(\"GPS--org.md2k.phonesensor--PHONE\")\n",
-    "gps_stream.show(3)"
+    "gps_stream = gen_location_datastream(user_id=USER_ID, stream_name=\"GPS--org.md2k.phonesensor--PHONE\")"
    ]
   },
   {
@@ -147,34 +111,44 @@
    "execution_count": 4,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/spark/python/pyspark/sql/pandas/group_ops.py:76: UserWarning:\n",
+      "\n",
+      "It is preferred to use 'applyInPandas' over this API. This API will be deprecated in the future releases. See SPARK-28264 for more details.\n",
+      "\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+------------------------------------+------------------+------------------+-----------+------------------+\n",
-      "|timestamp          |localtime          |latitude          |longitude         |altitude|speed   |bearing   |accuracy |version|user                                |centroid_longitude|centroid_latitude |centroid_id|centroid_area     |\n",
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+------------------------------------+------------------+------------------+-----------+------------------+\n",
-      "|2019-09-01 22:26:59|2019-09-02 03:26:59|35.157442557926856|-90.0349952849216 |93      |2.834021|105.992312|17.288347|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.0353360730015 |35.157759542552085|0          |267.12246418031026|\n",
-      "|2019-09-01 22:27:59|2019-09-02 03:27:59|35.1561572521825  |-90.03681018833898|94      |3.796146|8.256657  |15.215249|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03602813351293|35.15622962415908 |1          |455.94082767615083|\n",
-      "|2019-09-01 22:28:59|2019-09-02 03:28:59|35.15747745895255 |-90.03574140915266|93      |1.11443 |4.37211   |11.172609|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.0353360730015 |35.157759542552085|0          |267.12246418031026|\n",
-      "|2019-09-01 22:29:59|2019-09-02 03:29:59|35.15708891291051 |-90.03650897768411|92      |3.131751|6.704876  |17.811686|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03642313288314|35.15703951621386 |2          |179.45217175005607|\n",
-      "|2019-09-01 22:30:59|2019-09-02 03:30:59|35.157819954100916|-90.0380609165374 |93      |2.71621 |215.514534|29.340396|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:31:59|2019-09-02 03:31:59|35.15731861756214 |-90.0377806906272 |86      |3.796168|54.177014 |24.2978  |1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:32:59|2019-09-02 03:32:59|35.156717879382036|-90.03594834894683|88      |2.518586|324.965058|26.444573|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03602813351293|35.15622962415908 |1          |455.94082767615083|\n",
-      "|2019-09-01 22:33:59|2019-09-02 03:33:59|35.155986739160966|-90.03891991755704|86      |3.083471|177.534574|25.160381|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03891991755704|35.155986739160966|4          |1.0               |\n",
-      "|2019-09-01 22:34:59|2019-09-02 03:34:59|35.15754765423845 |-90.03771300796573|87      |4.174375|267.407334|14.297924|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:35:59|2019-09-02 03:35:59|35.157786834689006|-90.0347758093867 |88      |3.827271|313.008206|17.894079|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.0353360730015 |35.157759542552085|0          |267.12246418031026|\n",
-      "|2019-09-01 22:36:59|2019-09-02 03:36:59|35.156678602300644|-90.03686257325478|84      |4.088938|142.509126|29.763369|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:37:59|2019-09-02 03:37:59|35.1581569112939  |-90.03727713587398|88      |2.487589|281.067347|22.69205 |1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:38:59|2019-09-02 03:38:59|35.15536408131731 |-90.03720501988431|97      |2.335055|210.778948|22.977393|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03741878512767|35.15540481242479 |5          |97.7751362016774  |\n",
-      "|2019-09-01 22:39:59|2019-09-02 03:39:59|35.15787692049786 |-90.03726121254982|90      |3.085715|60.793675 |24.448123|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:40:59|2019-09-02 03:40:59|35.15771076901891 |-90.03581959760821|85      |3.520521|143.61742 |30.251623|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.0353360730015 |35.157759542552085|0          |267.12246418031026|\n",
-      "|2019-09-01 22:41:59|2019-09-02 03:41:59|35.1576540323178  |-90.03851852378476|84      |1.17867 |207.978631|23.996826|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03851852378476|35.1576540323178  |6          |1.0               |\n",
-      "|2019-09-01 22:42:59|2019-09-02 03:42:59|35.15780605456528 |-90.03819011048405|86      |0.152229|229.816167|13.878084|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:43:59|2019-09-02 03:43:59|35.15857756261247 |-90.0376389856197 |97      |4.014761|50.363152 |26.450883|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.0376389856197 |35.15857756261247 |7          |117.10787696220476|\n",
-      "|2019-09-01 22:44:59|2019-09-02 03:44:59|35.15708793876734 |-90.03775364668604|90      |1.890624|268.467044|21.994646|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03724267947695|35.15747101937749 |3          |768.5279283203273 |\n",
-      "|2019-09-01 22:45:59|2019-09-02 03:45:59|35.15736678958873 |-90.03624488367248|87      |0.290523|187.061816|28.722373|1      |00000000-afb8-476e-9872-6472b4e66b68|-90.03624488367248|35.15736678958873 |8          |50.08025241852694 |\n",
-      "+-------------------+-------------------+------------------+------------------+--------+--------+----------+---------+-------+------------------------------------+------------------+------------------+-----------+------------------+\n",
+      "+-------------------+-------------------+------------------------------------+-------+------------------+------------------+--------+--------+----------+---------+------------------+-----------------+-----------+------------------+\n",
+      "|timestamp          |localtime          |user                                |version|latitude          |longitude         |altitude|speed   |bearing   |accuracy |centroid_longitude|centroid_latitude|centroid_id|centroid_area     |\n",
+      "+-------------------+-------------------+------------------------------------+-------+------------------+------------------+--------+--------+----------+---------+------------------+-----------------+-----------+------------------+\n",
+      "|2019-09-01 11:35:59|2019-09-01 16:35:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.13976879971648 |-89.96575210246465|96      |1.416015|215.352509|15.284655|-89.96575210246465|35.13976879971648|0          |1.0               |\n",
+      "|2019-09-01 11:36:59|2019-09-01 16:36:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.140209588105364|-89.96513291366226|94      |3.710222|169.585097|20.610609|-89.96507298817545|35.14012607629936|1          |84.26621997061004 |\n",
+      "|2019-09-01 11:37:59|2019-09-01 16:37:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.141791339266355|-89.966609875023  |88      |3.904812|217.47797 |18.891424|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:38:59|2019-09-01 16:38:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14154706294343 |-89.96461184774307|95      |0.874029|105.173754|16.342223|-89.96481967637504|35.14128157689931|3          |566.2219615425483 |\n",
+      "|2019-09-01 11:39:59|2019-09-01 16:39:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14257741150938 |-89.96714867390146|92      |3.985796|306.306289|28.219226|-89.96714867390146|35.14257741150938|4          |1.0               |\n",
+      "|2019-09-01 11:40:59|2019-09-01 16:40:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14201852430586 |-89.96562966128454|88      |4.162096|325.480318|16.742511|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:41:59|2019-09-01 16:41:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.142831849083116|-89.96560968616144|85      |1.339617|177.410238|27.932266|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:42:59|2019-09-01 16:42:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14276361994767 |-89.96643888986277|90      |3.744733|73.913189 |12.800099|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:43:59|2019-09-01 16:43:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.140285961117684|-89.96601306864135|84      |0.094406|109.813907|19.100104|-89.96581587108294|35.14037664605037|5          |155.56199287843944|\n",
+      "|2019-09-01 11:44:59|2019-09-01 16:44:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14225953581674 |-89.96404663824158|86      |4.290311|143.548887|22.872055|-89.96404663824158|35.14225953581674|6          |1.0               |\n",
+      "|2019-09-01 11:45:59|2019-09-01 16:45:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.142269895986395|-89.96626997527599|94      |1.64047 |334.443296|15.586599|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:46:59|2019-09-01 16:46:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.140878522857484|-89.9660406708222 |98      |4.179958|345.510366|12.252356|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:47:59|2019-09-01 16:47:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.142089995627884|-89.96685844602943|85      |0.76181 |133.114138|19.148577|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:48:59|2019-09-01 16:48:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14132779055138 |-89.96502620022976|87      |3.728501|57.338208 |24.885406|-89.96481967637504|35.14128157689931|3          |566.2219615425483 |\n",
+      "|2019-09-01 11:49:59|2019-09-01 16:49:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14111914230905 |-89.96674472033256|91      |1.716824|57.154817 |11.263093|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:50:59|2019-09-01 16:50:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14177832777922 |-89.96375011163056|93      |1.434387|10.550654 |28.138054|-89.96375011163056|35.14177832777922|7          |1.0               |\n",
+      "|2019-09-01 11:51:59|2019-09-01 16:51:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.141572425998774|-89.96599970498741|92      |0.202097|226.554321|21.868812|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:52:59|2019-09-01 16:52:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14190664422532 |-89.9661990032071 |96      |3.706949|222.192994|26.625996|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:53:59|2019-09-01 16:53:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14108637621499 |-89.96696422183204|92      |3.690029|227.721474|16.208357|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "|2019-09-01 11:54:59|2019-09-01 16:54:59|00000000-afb8-476e-9872-6472b4e66b68|1      |35.14242465950471 |-89.96558289321497|84      |1.572824|15.292668 |23.387593|-89.9661453213007 |35.14194125447436|2          |1026.732067784292 |\n",
+      "+-------------------+-------------------+------------------------------------+-------+------------------+------------------+--------+--------+----------+---------+------------------+-----------------+-----------+------------------+\n",
       "only showing top 20 rows\n",
       "\n"
      ]
@@ -210,12 +184,12 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c0d25f6ed262416590b56c4e349c2b6a",
+       "model_id": "2133d32582324b37a293ddf4033ff5b7",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "Map(basemap={'url': 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', 'max_zoom': 19, 'attribution': 'Map …"
+       "Map(center=[35.13976879971648, -89.96575210246465], controls=(ZoomControl(options=['position', 'zoom_in_text',…"
       ]
      },
      "metadata": {},
@@ -251,7 +225,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.7.6"
   },
   "toc-autonumbering": true,
   "toc-showmarkdowntxt": true