From 578acf3be6818c648c574ded1e61a51e6634b662 Mon Sep 17 00:00:00 2001 From: amandaha8 Date: Mon, 25 Sep 2023 18:59:25 +0000 Subject: [PATCH 1/4] automating gtfs quality check --- gtfs_quality_check/gtfs_checking.ipynb | 1785 ++++++++++++++++++++++++ 1 file changed, 1785 insertions(+) create mode 100644 gtfs_quality_check/gtfs_checking.ipynb diff --git a/gtfs_quality_check/gtfs_checking.ipynb b/gtfs_quality_check/gtfs_checking.ipynb new file mode 100644 index 000000000..ebb3dd626 --- /dev/null +++ b/gtfs_quality_check/gtfs_checking.ipynb @@ -0,0 +1,1785 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "9a12bb6d-e7f8-4517-ad77-1012b4716575", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "import pandas as pd\n", + "import requests\n", + "from calitp_data_analysis.sql import to_snakecase" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "457f42cc-b42d-49c1-9142-82efb618b17c", + "metadata": {}, + "outputs": [], + "source": [ + "pd.options.display.max_columns = 100\n", + "pd.options.display.float_format = \"{:.2f}\".format\n", + "pd.set_option(\"display.max_rows\", None)\n", + "pd.set_option(\"display.max_colwidth\", None)" + ] + }, + { + "cell_type": "markdown", + "id": "967cdc78-614b-4589-ab6e-ac7f78cfae31", + "metadata": {}, + "source": [ + "## Catastrophic" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a60a3276-57a9-420c-a8a6-21aa5ff5b8a2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] + } + ], + "source": [ + "expired_df = pd.read_excel(\"./feed_info_expired_feeds_2023-09-21T19_13_29.263825Z.xlsx\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4f5b05ee-47a6-4614-ace5-998f192047b9", + "metadata": {}, + "outputs": [], + "source": [ + "expired_df = expired_df.rename(columns = {'Dim Provider Gtfs Data → Schedule Gtfs Dataset Name':'schedule_gtfs_name'})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9b47e637-d755-470f-b585-eb1ee46dc918", + "metadata": {}, + "outputs": [], + "source": [ + "expired_df = to_snakecase(expired_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cd9429b4-5108-4c9e-b86b-36fbe6171212", + "metadata": {}, + "outputs": [], + "source": [ + "no_lacmta = expired_df[~expired_df['dim_gtfs_datasets_→_uri'].str.contains('LACMTA')]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "177b7d83-5bec-498e-b3ca-eeefbdb436fc", + "metadata": {}, + "outputs": [], + "source": [ + "no_lacmta2 = (no_lacmta\n", + " [['dim_provider_gtfs_data_→_service_name',\"dim_county_geography_→_caltrans_district\", \"date\"]]\n", + " .sort_values([\"dim_provider_gtfs_data_→_service_name\", \"date\"],\n", + " ascending = [False, False])\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a1a0be85-784c-483d-ac15-e0a8c3da8e5e", + "metadata": {}, + "outputs": [], + "source": [ + "open_airtable_tickets = to_snakecase(pd.read_csv(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "05c95752-05ca-4147-94f0-3cd1859174d0", + "metadata": {}, + "outputs": [], + "source": [ + "open_airtable_tickets = open_airtable_tickets.fillna('None')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "22440833-8d90-4bcd-8e77-8ea23f88702f", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "open_airtable_tickets = open_airtable_tickets[['gtfs_datasets', 'services',]]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1728d945-d52b-4888-9007-012f857c0ff3", + "metadata": {}, + "outputs": [], + "source": [ + "open_airtable_tickets['airtable_ticket'] = 'yes'" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "bfac56a1-49c7-4b28-bc6e-e7009dc94ad7", + "metadata": {}, + "outputs": [], + "source": [ + "no_lacmta2_summary = (no_lacmta2\n", + " .groupby(['dim_provider_gtfs_data_→_service_name'])\n", + " .agg({'date':'count'})\n", + " .reset_index()\n", + " .rename(columns = {'date':'# of days with expired feed'})\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "c3914805-a115-4029-97da-bc6bd2446706", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dim_provider_gtfs_data_→_service_name# of days with expired feedgtfs_datasetsservicesairtable_ticket
0Morro Bay Transit14Morro Bay ScheduleMorro Bay Transityes
1Plumas Transit Systems14Plumas SchedulePlumas Transit Systemsyes
2Sage Stage Intercity2NaNNaNNaN
3TRACER14Tracy ScheduleTRACERyes
\n", + "
" + ], + "text/plain": [ + " dim_provider_gtfs_data_→_service_name # of days with expired feed \\\n", + "0 Morro Bay Transit 14 \n", + "1 Plumas Transit Systems 14 \n", + "2 Sage Stage Intercity 2 \n", + "3 TRACER 14 \n", + "\n", + " gtfs_datasets services airtable_ticket \n", + "0 Morro Bay Schedule Morro Bay Transit yes \n", + "1 Plumas Schedule Plumas Transit Systems yes \n", + "2 NaN NaN NaN \n", + "3 Tracy Schedule TRACER yes " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.merge(no_lacmta2_summary, open_airtable_tickets, left_on = ['dim_provider_gtfs_data_→_service_name'], right_on = [\"services\"], how = \"left\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "8da60e37-7e1e-4518-8f1a-eaf7e7d41086", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dim_provider_gtfs_data_→_service_namedim_county_geography_→_caltrans_districtdate
8TRACER102023-09-20
18TRACER102023-09-19
28TRACER102023-09-18
38TRACER102023-09-17
48TRACER102023-09-16
58TRACER102023-09-15
68TRACER102023-09-14
78TRACER102023-09-13
88TRACER102023-09-12
98TRACER102023-09-11
108TRACER102023-09-10
118TRACER102023-09-09
128TRACER102023-09-08
139TRACER102023-09-07
129Sage Stage Intercity22023-09-08
140Sage Stage Intercity22023-09-07
9Plumas Transit Systems22023-09-20
19Plumas Transit Systems22023-09-19
29Plumas Transit Systems22023-09-18
39Plumas Transit Systems22023-09-17
49Plumas Transit Systems22023-09-16
59Plumas Transit Systems22023-09-15
69Plumas Transit Systems22023-09-14
79Plumas Transit Systems22023-09-13
89Plumas Transit Systems22023-09-12
99Plumas Transit Systems22023-09-11
109Plumas Transit Systems22023-09-10
119Plumas Transit Systems22023-09-09
130Plumas Transit Systems22023-09-08
141Plumas Transit Systems22023-09-07
6Morro Bay Transit52023-09-20
16Morro Bay Transit52023-09-19
26Morro Bay Transit52023-09-18
36Morro Bay Transit52023-09-17
46Morro Bay Transit52023-09-16
56Morro Bay Transit52023-09-15
66Morro Bay Transit52023-09-14
76Morro Bay Transit52023-09-13
86Morro Bay Transit52023-09-12
96Morro Bay Transit52023-09-11
106Morro Bay Transit52023-09-10
116Morro Bay Transit52023-09-09
126Morro Bay Transit52023-09-08
137Morro Bay Transit52023-09-07
\n", + "
" + ], + "text/plain": [ + " dim_provider_gtfs_data_→_service_name \\\n", + "8 TRACER \n", + "18 TRACER \n", + "28 TRACER \n", + "38 TRACER \n", + "48 TRACER \n", + "58 TRACER \n", + "68 TRACER \n", + "78 TRACER \n", + "88 TRACER \n", + "98 TRACER \n", + "108 TRACER \n", + "118 TRACER \n", + "128 TRACER \n", + "139 TRACER \n", + "129 Sage Stage Intercity \n", + "140 Sage Stage Intercity \n", + "9 Plumas Transit Systems \n", + "19 Plumas Transit Systems \n", + "29 Plumas Transit Systems \n", + "39 Plumas Transit Systems \n", + "49 Plumas Transit Systems \n", + "59 Plumas Transit Systems \n", + "69 Plumas Transit Systems \n", + "79 Plumas Transit Systems \n", + "89 Plumas Transit Systems \n", + "99 Plumas Transit Systems \n", + "109 Plumas Transit Systems \n", + "119 Plumas Transit Systems \n", + "130 Plumas Transit Systems \n", + "141 Plumas Transit Systems \n", + "6 Morro Bay Transit \n", + "16 Morro Bay Transit \n", + "26 Morro Bay Transit \n", + "36 Morro Bay Transit \n", + "46 Morro Bay Transit \n", + "56 Morro Bay Transit \n", + "66 Morro Bay Transit \n", + "76 Morro Bay Transit \n", + "86 Morro Bay Transit \n", + "96 Morro Bay Transit \n", + "106 Morro Bay Transit \n", + "116 Morro Bay Transit \n", + "126 Morro Bay Transit \n", + "137 Morro Bay Transit \n", + "\n", + " dim_county_geography_→_caltrans_district date \n", + "8 10 2023-09-20 \n", + "18 10 2023-09-19 \n", + "28 10 2023-09-18 \n", + "38 10 2023-09-17 \n", + "48 10 2023-09-16 \n", + "58 10 2023-09-15 \n", + "68 10 2023-09-14 \n", + "78 10 2023-09-13 \n", + "88 10 2023-09-12 \n", + "98 10 2023-09-11 \n", + "108 10 2023-09-10 \n", + "118 10 2023-09-09 \n", + "128 10 2023-09-08 \n", + "139 10 2023-09-07 \n", + "129 2 2023-09-08 \n", + "140 2 2023-09-07 \n", + "9 2 2023-09-20 \n", + "19 2 2023-09-19 \n", + "29 2 2023-09-18 \n", + "39 2 2023-09-17 \n", + "49 2 2023-09-16 \n", + "59 2 2023-09-15 \n", + "69 2 2023-09-14 \n", + "79 2 2023-09-13 \n", + "89 2 2023-09-12 \n", + "99 2 2023-09-11 \n", + "109 2 2023-09-10 \n", + "119 2 2023-09-09 \n", + "130 2 2023-09-08 \n", + "141 2 2023-09-07 \n", + "6 5 2023-09-20 \n", + "16 5 2023-09-19 \n", + "26 5 2023-09-18 \n", + "36 5 2023-09-17 \n", + "46 5 2023-09-16 \n", + "56 5 2023-09-15 \n", + "66 5 2023-09-14 \n", + "76 5 2023-09-13 \n", + "86 5 2023-09-12 \n", + "96 5 2023-09-11 \n", + "106 5 2023-09-10 \n", + "116 5 2023-09-09 \n", + "126 5 2023-09-08 \n", + "137 5 2023-09-07 " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "no_lacmta2" + ] + }, + { + "cell_type": "markdown", + "id": "78eecb32-2a91-4ba3-bafc-3ff3ca4a424e", + "metadata": {}, + "source": [ + "## Incompleteness" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "18276af1-e87e-4218-b486-47a1bdd1da6d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] + } + ], + "source": [ + "trip_update_df = pd.read_excel(\"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "9c0dd62b-dc5b-408c-8d4b-2a5ce9b10666", + "metadata": {}, + "outputs": [], + "source": [ + "trip_update_df = to_snakecase(trip_update_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "940c0a74-82eb-40e8-b351-4366b4de6cfd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] + } + ], + "source": [ + "vp_df = to_snakecase(pd.read_excel(\"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "681da5bd-6fda-4e2d-afa6-1817f69c3140", + "metadata": {}, + "outputs": [], + "source": [ + "low_tu = (trip_update_df[trip_update_df['%_of_trips_with_tu_messages'] < 41].sort_values(['%_of_trips_with_tu_messages'])).reset_index(drop = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "f235a5ed-85aa-452d-bc48-7499dab186ce", + "metadata": {}, + "outputs": [], + "source": [ + "low_vp = (vp_df[vp_df['%_of_trips_with_vp_messages'] < 41].sort_values(['%_of_trips_with_vp_messages'])).reset_index(drop = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "d84c1256-827b-4f21-890b-4d1b2d861145", + "metadata": {}, + "outputs": [], + "source": [ + "incomplete = pd.merge(low_tu, low_vp, on = 'name', how = 'outer')\n", + "incomplete = incomplete.sort_values(['name'])" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "f072f171-cbfc-46b7-a752-88ede3bdf30e", + "metadata": {}, + "outputs": [], + "source": [ + "# incomplete.name = incomplete.name.str.replace('Schedule','')" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "4155eabf-6a8c-4dcf-a740-3075fbb785b3", + "metadata": {}, + "outputs": [], + "source": [ + "# Read in 511\n", + "df_511 = to_snakecase(pd.read_excel('./API511.xlsx'))" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "id": "57cfcd92-2580-4b86-8383-1b4e8407e222", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['organisations>'], dtype='object')" + ] + }, + "execution_count": 93, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_511.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "f91d42cb-a1ea-44f8-be1d-dca2624dd00b", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "df_511_2 = df_511[df_511['organisations>'].str.contains(('Name|Monitored'))].reset_index(drop = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "6d08d56a-b8a6-4052-bf75-8925554f8733", + "metadata": {}, + "outputs": [], + "source": [ + "df_511_2['organisations>'] = (df_511_2['organisations>'].str.replace('','')\n", + " .str.replace('','')\n", + " .str.replace('','')\n", + " .str.replace('','')\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "1acd04fb-d50d-4e86-9da3-6af9ec0dfb13", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_158/2396432539.py:1: FutureWarning: this method is deprecated in favour of `Styler.applymap()`\n", + " df_511_2.style.where(lambda val: 'false' in str(val), 'color: red')\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 organisations>
0false
1511 Emergency
2511 Emergency
3false
4511 Flap Sign
5511 Flap Sign
6false
7511 Operations
8511 Emergency
9false
10511 Staff
11511 Staff
12true
13AC TRANSIT
14AC Transit
15false
16Altamont Corridor Express
17ACE
18false
19Angel Island Tiburon Ferry
20Angel Island Ferry
21true
22Bay Area Rapid Transit
23BART
24true
25Caltrain
26Caltrain
27false
28Capitol Corridor Joint Powers Authority
29Capitol Corridor
30false
31City of South San Francisco
32South City Shuttle
33false
34Commute.org Shuttles
35Commute.org
36true
37County Connection
38County Connection
39true
40Dumbarton Express Consortium
41Dumbarton
42true
43Emery Go-Round
44Emery Go-Round
45true
46FAST
47FAST
48false
49Golden Gate Ferry
50GG Ferry
51true
52Golden Gate Transit
53GG Transit
54true
55Livermore Amador Valley Transit Authority
56Wheels
57true
58Marin Transit
59Marin Transit
60false
61Mission Bay TMA
62
63true
64MVgo Mountain View
65
66true
67Petaluma
68Petaluma Transit
69false
70Regional GTFS
71
72false
73Rio Vista Delta Breeze
74Delta Breeze
75true
76SamTrans
77SamTrans
78true
79San Francisco Bay Ferry
80SF Bay Ferry
81false
82San Francisco International Airport
83
84true
85San Francisco Municipal Transportation Agency
86SF Muni
87false
88San Joaquins
89
90true
91Santa Rosa CityBus
92SR CityBus
93true
94SolTrans
95Soltrans
96true
97Sonoma County Transit
98Sonoma County
99true
100Sonoma Marin Area Rail Transit
101SMART
102false
103Treasure Island Ferry
104
105true
106Tri Delta Transit
107Tri Delta Transit
108true
109Union City Transit
110Union City
111false
112Vacaville City Coach
113Vacaville City Coach
114true
115VINE Transit
116Napa VINE
117true
118VTA
119VTA
120true
121WestCat (Western Contra Costa)
122WestCAT
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 102, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_511_2.style.where(lambda val: 'false' in str(val), 'color: red')" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "id": "a2779bdb-cb88-463e-a320-d48fc2e09e74", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "# open_airtable_tickets.sort_values(['gtfs_datasets'])" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "c6d592ce-fb2a-46bb-bf5b-8ce7f4a27dcc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name%_of_trips_with_tu_messages%_of_trips_with_vp_messagesgtfs_datasetsservicesairtable_ticket_merge
17Anaheim Resort Schedule21.1117.73Anaheim Resort ScheduleAnaheim Resort Transportationyesboth
1Bay Area 511 ACE Schedule0.000.00NaNNaNNaNleft_only
2Bay Area 511 Angel Island-Tiburon Ferry Schedule0.000.00NaNNaNNaNleft_only
18Bay Area 511 BART Schedule39.310.00NaNNaNNaNleft_only
4Bay Area 511 Capitol Corridor Schedule0.000.00NaNNaNNaNleft_only
5Bay Area 511 Commute.org Schedule0.000.00NaNNaNNaNleft_only
6Bay Area 511 Golden Gate Ferry Schedule0.000.00NaNNaNNaNleft_only
7Bay Area 511 MVGO Schedule0.000.00NaNNaNNaNleft_only
8Bay Area 511 Mission Bay Schedule0.000.00NaNNaNNaNleft_only
0Bay Area 511 Rio Vista Delta Breeze Schedule0.000.00NaNNaNNaNleft_only
9Bay Area 511 SFO AirTrain Schedule0.000.00NaNNaNNaNleft_only
20Bay Area 511 Santa Rosa CityBus ScheduleNaN17.11NaNNaNNaNleft_only
10Bay Area 511 South San Francisco Shuttle Schedule0.000.00NaNNaNNaNleft_only
11Bay Area 511 Treasure Island Ferry Schedule0.000.00NaNNaNNaNleft_only
15Bay Area 511 Union City Transit Schedule16.109.38NaNNaNNaNleft_only
12Bay Area 511 Vacaville City Coach Schedule0.000.00NaNNaNNaNleft_only
22Bay Area 511 Vine Transit ScheduleNaN37.77NaNNaNNaNleft_only
19Merced GMV ScheduleNaN0.00NaNNaNNaNleft_only
14SLORTA Schedule15.7415.60NaNNaNNaNleft_only
3San Diego Schedule0.00NaNNaNNaNNaNleft_only
21Santa Rosa CityBus ScheduleNaN21.26NaNNaNNaNleft_only
13SunLine Avail Schedule0.960.71NaNNaNNaNleft_only
16Union City GMV Schedule18.3910.68Union City GMV ScheduleUnion City Transityesboth
23Vine GMV ScheduleNaN37.77NaNNaNNaNleft_only
\n", + "
" + ], + "text/plain": [ + " name \\\n", + "17 Anaheim Resort Schedule \n", + "1 Bay Area 511 ACE Schedule \n", + "2 Bay Area 511 Angel Island-Tiburon Ferry Schedule \n", + "18 Bay Area 511 BART Schedule \n", + "4 Bay Area 511 Capitol Corridor Schedule \n", + "5 Bay Area 511 Commute.org Schedule \n", + "6 Bay Area 511 Golden Gate Ferry Schedule \n", + "7 Bay Area 511 MVGO Schedule \n", + "8 Bay Area 511 Mission Bay Schedule \n", + "0 Bay Area 511 Rio Vista Delta Breeze Schedule \n", + "9 Bay Area 511 SFO AirTrain Schedule \n", + "20 Bay Area 511 Santa Rosa CityBus Schedule \n", + "10 Bay Area 511 South San Francisco Shuttle Schedule \n", + "11 Bay Area 511 Treasure Island Ferry Schedule \n", + "15 Bay Area 511 Union City Transit Schedule \n", + "12 Bay Area 511 Vacaville City Coach Schedule \n", + "22 Bay Area 511 Vine Transit Schedule \n", + "19 Merced GMV Schedule \n", + "14 SLORTA Schedule \n", + "3 San Diego Schedule \n", + "21 Santa Rosa CityBus Schedule \n", + "13 SunLine Avail Schedule \n", + "16 Union City GMV Schedule \n", + "23 Vine GMV Schedule \n", + "\n", + " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \\\n", + "17 21.11 17.73 \n", + "1 0.00 0.00 \n", + "2 0.00 0.00 \n", + "18 39.31 0.00 \n", + "4 0.00 0.00 \n", + "5 0.00 0.00 \n", + "6 0.00 0.00 \n", + "7 0.00 0.00 \n", + "8 0.00 0.00 \n", + "0 0.00 0.00 \n", + "9 0.00 0.00 \n", + "20 NaN 17.11 \n", + "10 0.00 0.00 \n", + "11 0.00 0.00 \n", + "15 16.10 9.38 \n", + "12 0.00 0.00 \n", + "22 NaN 37.77 \n", + "19 NaN 0.00 \n", + "14 15.74 15.60 \n", + "3 0.00 NaN \n", + "21 NaN 21.26 \n", + "13 0.96 0.71 \n", + "16 18.39 10.68 \n", + "23 NaN 37.77 \n", + "\n", + " gtfs_datasets services airtable_ticket \\\n", + "17 Anaheim Resort Schedule Anaheim Resort Transportation yes \n", + "1 NaN NaN NaN \n", + "2 NaN NaN NaN \n", + "18 NaN NaN NaN \n", + "4 NaN NaN NaN \n", + "5 NaN NaN NaN \n", + "6 NaN NaN NaN \n", + "7 NaN NaN NaN \n", + "8 NaN NaN NaN \n", + "0 NaN NaN NaN \n", + "9 NaN NaN NaN \n", + "20 NaN NaN NaN \n", + "10 NaN NaN NaN \n", + "11 NaN NaN NaN \n", + "15 NaN NaN NaN \n", + "12 NaN NaN NaN \n", + "22 NaN NaN NaN \n", + "19 NaN NaN NaN \n", + "14 NaN NaN NaN \n", + "3 NaN NaN NaN \n", + "21 NaN NaN NaN \n", + "13 NaN NaN NaN \n", + "16 Union City GMV Schedule Union City Transit yes \n", + "23 NaN NaN NaN \n", + "\n", + " _merge \n", + "17 both \n", + "1 left_only \n", + "2 left_only \n", + "18 left_only \n", + "4 left_only \n", + "5 left_only \n", + "6 left_only \n", + "7 left_only \n", + "8 left_only \n", + "0 left_only \n", + "9 left_only \n", + "20 left_only \n", + "10 left_only \n", + "11 left_only \n", + "15 left_only \n", + "12 left_only \n", + "22 left_only \n", + "19 left_only \n", + "14 left_only \n", + "3 left_only \n", + "21 left_only \n", + "13 left_only \n", + "16 both \n", + "23 left_only " + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " pd.merge(incomplete, open_airtable_tickets, left_on = \"name\", right_on = \"gtfs_datasets\", how = \"left\", indicator = True).sort_values('name')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 1f6744fe188e04af27d28425ba42716fa937ceca Mon Sep 17 00:00:00 2001 From: amandaha8 Date: Mon, 25 Sep 2023 23:04:06 +0000 Subject: [PATCH 2/4] experimenting with 511 --- gtfs_quality_check/gtfs_checking.ipynb | 1477 +++++++++++++++--------- 1 file changed, 930 insertions(+), 547 deletions(-) diff --git a/gtfs_quality_check/gtfs_checking.ipynb b/gtfs_quality_check/gtfs_checking.ipynb index ebb3dd626..eddfa8543 100644 --- a/gtfs_quality_check/gtfs_checking.ipynb +++ b/gtfs_quality_check/gtfs_checking.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "9a12bb6d-e7f8-4517-ad77-1012b4716575", "metadata": {}, "outputs": [], @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "457f42cc-b42d-49c1-9142-82efb618b17c", "metadata": {}, "outputs": [], @@ -37,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "a60a3276-57a9-420c-a8a6-21aa5ff5b8a2", "metadata": {}, "outputs": [ @@ -54,16 +54,6 @@ "expired_df = pd.read_excel(\"./feed_info_expired_feeds_2023-09-21T19_13_29.263825Z.xlsx\")" ] }, - { - "cell_type": "code", - "execution_count": 5, - "id": "4f5b05ee-47a6-4614-ace5-998f192047b9", - "metadata": {}, - "outputs": [], - "source": [ - "expired_df = expired_df.rename(columns = {'Dim Provider Gtfs Data → Schedule Gtfs Dataset Name':'schedule_gtfs_name'})" - ] - }, { "cell_type": "code", "execution_count": 6, @@ -76,51 +66,58 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "cd9429b4-5108-4c9e-b86b-36fbe6171212", "metadata": {}, "outputs": [], "source": [ - "no_lacmta = expired_df[~expired_df['dim_gtfs_datasets_→_uri'].str.contains('LACMTA')]" + "# Don't look at LACMTA stuff\n", + "no_lacmta = expired_df[~expired_df[\"dim_gtfs_datasets_→_uri\"].str.contains(\"LACMTA\")]" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 38, "id": "177b7d83-5bec-498e-b3ca-eeefbdb436fc", "metadata": {}, "outputs": [], "source": [ - "no_lacmta2 = (no_lacmta\n", - " [['dim_provider_gtfs_data_→_service_name',\"dim_county_geography_→_caltrans_district\", \"date\"]]\n", - " .sort_values([\"dim_provider_gtfs_data_→_service_name\", \"date\"],\n", - " ascending = [False, False])\n", - " )" + "no_lacmta2 = no_lacmta[\n", + " [\n", + " \"dim_provider_gtfs_data_→_service_name\",\n", + " \"dim_county_geography_→_caltrans_district\",\n", + " \"date\",\n", + " ]\n", + "].sort_values(\n", + " [\"dim_provider_gtfs_data_→_service_name\", \"date\"], ascending=[False, False]\n", + ")" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 39, "id": "a1a0be85-784c-483d-ac15-e0a8c3da8e5e", "metadata": {}, "outputs": [], "source": [ - "open_airtable_tickets = to_snakecase(pd.read_csv(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\"))" + "open_airtable_tickets = to_snakecase(\n", + " pd.read_csv(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\")\n", + ")" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 40, "id": "05c95752-05ca-4147-94f0-3cd1859174d0", "metadata": {}, "outputs": [], "source": [ - "open_airtable_tickets = open_airtable_tickets.fillna('None')" + "open_airtable_tickets = open_airtable_tickets.fillna(\"None\")" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 41, "id": "22440833-8d90-4bcd-8e77-8ea23f88702f", "metadata": { "scrolled": true, @@ -128,37 +125,47 @@ }, "outputs": [], "source": [ - "open_airtable_tickets = open_airtable_tickets[['gtfs_datasets', 'services',]]" + "open_airtable_tickets = (\n", + " open_airtable_tickets[\n", + " [\n", + " \"description\",\n", + " \"gtfs_datasets\",\n", + " \"services\",\n", + " ]\n", + " ]\n", + " .sort_values([\"gtfs_datasets\", \"services\"])\n", + " .reset_index(drop=True)\n", + ")" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 42, "id": "1728d945-d52b-4888-9007-012f857c0ff3", "metadata": {}, "outputs": [], "source": [ - "open_airtable_tickets['airtable_ticket'] = 'yes'" + "open_airtable_tickets[\"airtable_ticket\"] = \"yes\"" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 43, "id": "bfac56a1-49c7-4b28-bc6e-e7009dc94ad7", "metadata": {}, "outputs": [], "source": [ - "no_lacmta2_summary = (no_lacmta2\n", - " .groupby(['dim_provider_gtfs_data_→_service_name'])\n", - " .agg({'date':'count'})\n", - " .reset_index()\n", - " .rename(columns = {'date':'# of days with expired feed'})\n", - " )" + "no_lacmta2_summary = (\n", + " no_lacmta2.groupby([\"dim_provider_gtfs_data_→_service_name\"])\n", + " .agg({\"date\": \"count\"})\n", + " .reset_index()\n", + " .rename(columns={\"date\": \"# of days with expired feed\"})\n", + ")" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 44, "id": "c3914805-a115-4029-97da-bc6bd2446706", "metadata": {}, "outputs": [ @@ -241,18 +248,24 @@ "3 Tracy Schedule TRACER yes " ] }, - "execution_count": 27, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pd.merge(no_lacmta2_summary, open_airtable_tickets, left_on = ['dim_provider_gtfs_data_→_service_name'], right_on = [\"services\"], how = \"left\")" + "pd.merge(\n", + " no_lacmta2_summary,\n", + " open_airtable_tickets,\n", + " left_on=[\"dim_provider_gtfs_data_→_service_name\"],\n", + " right_on=[\"services\"],\n", + " how=\"left\",\n", + ")" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 45, "id": "8da60e37-7e1e-4518-8f1a-eaf7e7d41086", "metadata": {}, "outputs": [ @@ -645,7 +658,7 @@ "137 5 2023-09-07 " ] }, - "execution_count": 28, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -664,7 +677,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 46, "id": "18276af1-e87e-4218-b486-47a1bdd1da6d", "metadata": {}, "outputs": [ @@ -678,22 +691,16 @@ } ], "source": [ - "trip_update_df = pd.read_excel(\"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\")" + "trip_update_df = to_snakecase(\n", + " pd.read_excel(\n", + " \"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\"\n", + " )\n", + ")" ] }, { "cell_type": "code", - "execution_count": 44, - "id": "9c0dd62b-dc5b-408c-8d4b-2a5ce9b10666", - "metadata": {}, - "outputs": [], - "source": [ - "trip_update_df = to_snakecase(trip_update_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, + "execution_count": 47, "id": "940c0a74-82eb-40e8-b351-4366b4de6cfd", "metadata": {}, "outputs": [ @@ -707,668 +714,1044 @@ } ], "source": [ - "vp_df = to_snakecase(pd.read_excel(\"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\"))" + "vp_df = to_snakecase(\n", + " pd.read_excel(\n", + " \"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\"\n", + " )\n", + ")" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 48, "id": "681da5bd-6fda-4e2d-afa6-1817f69c3140", "metadata": {}, "outputs": [], "source": [ - "low_tu = (trip_update_df[trip_update_df['%_of_trips_with_tu_messages'] < 41].sort_values(['%_of_trips_with_tu_messages'])).reset_index(drop = True)" + "low_tu = (\n", + " trip_update_df[trip_update_df[\"%_of_trips_with_tu_messages\"] < 41].sort_values(\n", + " [\"%_of_trips_with_tu_messages\"]\n", + " )\n", + ").reset_index(drop=True)" ] }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 49, "id": "f235a5ed-85aa-452d-bc48-7499dab186ce", "metadata": {}, "outputs": [], "source": [ - "low_vp = (vp_df[vp_df['%_of_trips_with_vp_messages'] < 41].sort_values(['%_of_trips_with_vp_messages'])).reset_index(drop = True)" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "id": "d84c1256-827b-4f21-890b-4d1b2d861145", - "metadata": {}, - "outputs": [], - "source": [ - "incomplete = pd.merge(low_tu, low_vp, on = 'name', how = 'outer')\n", - "incomplete = incomplete.sort_values(['name'])" + "low_vp = (\n", + " vp_df[vp_df[\"%_of_trips_with_vp_messages\"] < 41].sort_values(\n", + " [\"%_of_trips_with_vp_messages\"]\n", + " )\n", + ").reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 50, - "id": "f072f171-cbfc-46b7-a752-88ede3bdf30e", + "id": "d84c1256-827b-4f21-890b-4d1b2d861145", "metadata": {}, "outputs": [], "source": [ - "# incomplete.name = incomplete.name.str.replace('Schedule','')" + "incomplete = pd.merge(low_tu, low_vp, on=\"name\", how=\"outer\")\n", + "incomplete = incomplete.sort_values([\"name\"]).reset_index(drop=True)" ] }, { "cell_type": "code", - "execution_count": 92, - "id": "4155eabf-6a8c-4dcf-a740-3075fbb785b3", + "execution_count": 51, + "id": "8d9faa5c-959f-4203-9879-67cb32032711", "metadata": {}, "outputs": [], "source": [ - "# Read in 511\n", - "df_511 = to_snakecase(pd.read_excel('./API511.xlsx'))" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "id": "57cfcd92-2580-4b86-8383-1b4e8407e222", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['organisations>'], dtype='object')" - ] - }, - "execution_count": 93, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_511.columns" + "incomplete = incomplete.fillna(\"OK\")" ] }, { "cell_type": "code", - "execution_count": 103, - "id": "f91d42cb-a1ea-44f8-be1d-dca2624dd00b", + "execution_count": 62, + "id": "d62ff2ad-cfe5-4555-9e33-49b44680b6e9", "metadata": { "scrolled": true, "tags": [] }, "outputs": [], "source": [ - "df_511_2 = df_511[df_511['organisations>'].str.contains(('Name|Monitored'))].reset_index(drop = True)" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "id": "6d08d56a-b8a6-4052-bf75-8925554f8733", - "metadata": {}, - "outputs": [], - "source": [ - "df_511_2['organisations>'] = (df_511_2['organisations>'].str.replace('','')\n", - " .str.replace('','')\n", - " .str.replace('','')\n", - " .str.replace('','')\n", - " )" + "# open_airtable_tickets" ] }, { "cell_type": "code", - "execution_count": 102, - "id": "1acd04fb-d50d-4e86-9da3-6af9ec0dfb13", + "execution_count": 53, + "id": "3e5b4267-0778-41f2-a1bd-29c1c783d2bb", "metadata": { "scrolled": true, "tags": [] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_158/2396432539.py:1: FutureWarning: this method is deprecated in favour of `Styler.applymap()`\n", - " df_511_2.style.where(lambda val: 'false' in str(val), 'color: red')\n" - ] - }, { "data": { "text/html": [ - "\n", - "\n", + "
\n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + "
 organisations>
name%_of_trips_with_tu_messages%_of_trips_with_vp_messages
0false
1511 Emergency
2511 Emergency
3false
4511 Flap Sign
5511 Flap Sign
6false
7511 Operations
8511 Emergency
9false
10511 Staff
11511 Staff
12true
13AC TRANSIT
14AC Transit
15false
16Altamont Corridor Express
17ACE
18false
19Angel Island Tiburon Ferry
20Angel Island Ferry
21true
22Bay Area Rapid Transit
23BART
24true
25Caltrain
26Caltrain
27false
28Capitol Corridor Joint Powers Authority
29Capitol Corridor
30false
31City of South San Francisco
32South City Shuttle
33false
34Commute.org Shuttles
35Commute.org0Anaheim Resort Schedule21.1117.73
36true1Bay Area 511 ACE Schedule0.000.00
37County Connection2Bay Area 511 Angel Island-Tiburon Ferry Schedule0.000.00
38County Connection3Bay Area 511 BART Schedule39.310.00
39true4Bay Area 511 Capitol Corridor Schedule0.000.00
40Dumbarton Express Consortium5Bay Area 511 Commute.org Schedule0.000.00
41Dumbarton6Bay Area 511 Golden Gate Ferry Schedule0.000.00
42true7Bay Area 511 MVGO Schedule0.000.00
43Emery Go-Round8Bay Area 511 Mission Bay Schedule0.000.00
44Emery Go-Round9Bay Area 511 Rio Vista Delta Breeze Schedule0.000.00
45true10Bay Area 511 SFO AirTrain Schedule0.000.00
46FAST11Bay Area 511 Santa Rosa CityBus ScheduleOK17.11
47FAST12Bay Area 511 South San Francisco Shuttle Schedule0.000.00
48false13Bay Area 511 Treasure Island Ferry Schedule0.000.00
49Golden Gate Ferry14Bay Area 511 Union City Transit Schedule16.109.38
50GG Ferry15Bay Area 511 Vacaville City Coach Schedule0.000.00
51true16Bay Area 511 Vine Transit ScheduleOK37.77
52Golden Gate Transit17Merced GMV ScheduleOK0.00
53GG Transit18SLORTA Schedule15.7415.60
54true19San Diego Schedule0.00OK
55Livermore Amador Valley Transit Authority20Santa Rosa CityBus ScheduleOK21.26
56Wheels21SunLine Avail Schedule0.960.71
57true22Union City GMV Schedule18.3910.68
58Marin Transit23Vine GMV ScheduleOK37.77
\n", + "" + ], + "text/plain": [ + " name \\\n", + "0 Anaheim Resort Schedule \n", + "1 Bay Area 511 ACE Schedule \n", + "2 Bay Area 511 Angel Island-Tiburon Ferry Schedule \n", + "3 Bay Area 511 BART Schedule \n", + "4 Bay Area 511 Capitol Corridor Schedule \n", + "5 Bay Area 511 Commute.org Schedule \n", + "6 Bay Area 511 Golden Gate Ferry Schedule \n", + "7 Bay Area 511 MVGO Schedule \n", + "8 Bay Area 511 Mission Bay Schedule \n", + "9 Bay Area 511 Rio Vista Delta Breeze Schedule \n", + "10 Bay Area 511 SFO AirTrain Schedule \n", + "11 Bay Area 511 Santa Rosa CityBus Schedule \n", + "12 Bay Area 511 South San Francisco Shuttle Schedule \n", + "13 Bay Area 511 Treasure Island Ferry Schedule \n", + "14 Bay Area 511 Union City Transit Schedule \n", + "15 Bay Area 511 Vacaville City Coach Schedule \n", + "16 Bay Area 511 Vine Transit Schedule \n", + "17 Merced GMV Schedule \n", + "18 SLORTA Schedule \n", + "19 San Diego Schedule \n", + "20 Santa Rosa CityBus Schedule \n", + "21 SunLine Avail Schedule \n", + "22 Union City GMV Schedule \n", + "23 Vine GMV Schedule \n", + "\n", + " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \n", + "0 21.11 17.73 \n", + "1 0.00 0.00 \n", + "2 0.00 0.00 \n", + "3 39.31 0.00 \n", + "4 0.00 0.00 \n", + "5 0.00 0.00 \n", + "6 0.00 0.00 \n", + "7 0.00 0.00 \n", + "8 0.00 0.00 \n", + "9 0.00 0.00 \n", + "10 0.00 0.00 \n", + "11 OK 17.11 \n", + "12 0.00 0.00 \n", + "13 0.00 0.00 \n", + "14 16.10 9.38 \n", + "15 0.00 0.00 \n", + "16 OK 37.77 \n", + "17 OK 0.00 \n", + "18 15.74 15.60 \n", + "19 0.00 OK \n", + "20 OK 21.26 \n", + "21 0.96 0.71 \n", + "22 18.39 10.68 \n", + "23 OK 37.77 " + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "incomplete" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "f072f171-cbfc-46b7-a752-88ede3bdf30e", + "metadata": {}, + "outputs": [], + "source": [ + "# incomplete.name = incomplete.name.str.replace('Schedule','')" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "4155eabf-6a8c-4dcf-a740-3075fbb785b3", + "metadata": {}, + "outputs": [], + "source": [ + "# Read in 511\n", + "df_511 = to_snakecase(pd.read_excel(\"./API511.xlsx\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "57cfcd92-2580-4b86-8383-1b4e8407e222", + "metadata": {}, + "outputs": [], + "source": [ + "df_511 = df_511.rename(columns={df_511.columns[0]: \"new\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "63fa2dfc-fa14-45f4-be7b-a416073dd7b4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['new'], dtype='object')" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_511.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "f91d42cb-a1ea-44f8-be1d-dca2624dd00b", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "df_511_2 = df_511[df_511[\"new\"].str.contains((\"Name|Monitored\"))].reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "id": "6d08d56a-b8a6-4052-bf75-8925554f8733", + "metadata": {}, + "outputs": [], + "source": [ + "df_511_2.new = (\n", + " df_511_2.new.str.replace(\">\", \"\")\n", + " .str.replace(\"ShortName\", \"\")\n", + " .str.replace(\"Name\", \"\")\n", + " .str.replace(\"Monitored\", \"\")\n", + " .str.replace(\">\", \"\")\n", + " .str.replace(\"/\", \"\")\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "89d4a2c6-bec4-48a6-8612-e80c7a26111d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", + " \n", + "
new
59Marin Transit13AC TRANSIT
60false22Bay Area Rapid Transit
61Mission Bay TMA25Caltrain
6237County Connection
63true40Dumbarton Express Consortium
64MVgo Mountain View43Emery Go-Round
6546FAST
66true52Golden Gate Transit
67Petaluma55Livermore Amador Valley Transit Authority
68Petaluma Transit58Marin Transit
69false64MVgo Mountain View
70Regional GTFS67Petaluma
7176SamTrans
72false79San Francisco Bay Ferry
73Rio Vista Delta Breeze85San Francisco Municipal Transportation Agency
74Delta Breeze91Santa Rosa CityBus
75true94SolTrans
76SamTrans97Sonoma County Transit
77SamTrans100Sonoma Marin Area Rail Transit
78true106Tri Delta Transit
79San Francisco Bay Ferry109Union City Transit
80SF Bay Ferry115VINE Transit
81false118VTA
82San Francisco International Airport121WestCat (Western Contra Costa)
\n", + "
" + ], + "text/plain": [ + " new\n", + "13 AC TRANSIT\n", + "22 Bay Area Rapid Transit\n", + "25 Caltrain\n", + "37 County Connection\n", + "40 Dumbarton Express Consortium\n", + "43 Emery Go-Round\n", + "46 FAST\n", + "52 Golden Gate Transit\n", + "55 Livermore Amador Valley Transit Authority\n", + "58 Marin Transit\n", + "64 MVgo Mountain View\n", + "67 Petaluma\n", + "76 SamTrans\n", + "79 San Francisco Bay Ferry\n", + "85 San Francisco Municipal Transportation Agency\n", + "91 Santa Rosa CityBus\n", + "94 SolTrans\n", + "97 Sonoma County Transit\n", + "100 Sonoma Marin Area Rail Transit\n", + "106 Tri Delta Transit\n", + "109 Union City Transit\n", + "115 VINE Transit\n", + "118 VTA\n", + "121 WestCat (Western Contra Costa)" + ] + }, + "execution_count": 103, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_511_2[(df_511_2.new == \"true\").shift(1).fillna(False)]" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "a2779bdb-cb88-463e-a320-d48fc2e09e74", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - "
gtfs_datasetsservicesairtable_ticket
830AC Transit Alerts,Bay Area 511 Regional AlertsAC Transityes
84true1Alhambra ScheduleAlhambra Community Transityes
85San Francisco Municipal Transportation Agency2Anaheim Resort ScheduleAnaheim Resort Transportationyes
86SF Muni3Arcadia ScheduleArcadia Transityes
87false4Artesia ScheduleArtesia Transityes
88San Joaquins5Arvin ScheduleArvin Transityes
896Auburn ScheduleAuburn Transityes
90true7Baldwin Park ScheduleBaldwin Park Expressyes
91Santa Rosa CityBus8Bay Area 511 Regional VehiclePositionsAC Transityes
92SR CityBus9Bay Area 511 Union City Transit VehiclePositionsUnion City Transityes
93true10Bell Gardens ScheduleBell Gardens Trollyyes
94SolTrans11Bellflower Bus ScheduleBellflower Busyes
95Soltrans12Calabasas ScheduleCalabasas Shuttle and Trolleyyes
96true13Capitol Corridor Schedule ,Bay Area 511 ACE Schedule,Amtrak ScheduleCapitol Corridoryes
97Sonoma County Transit14Catalina Flyer Schedule HistoricCatalina Flyeryes
98Sonoma County15Compton ScheduleCompton Renaissance Transit Serviceyes
99true16Cudahy ScheduleCudahy Area Rapid Transityes
100Sonoma Marin Area Rail Transit17DowneyLINK ScheduleDowneyLINKyes
101SMART18El Segundo ScheduleEl Segundo Shuttlesyes
102false19Get Around Town Express ScheduleGet Around Town Expressyes
103Treasure Island Ferry20Glendora ScheduleGlendora Shuttlesyes
10421Glenn ScheduleGlenn Rideyes
105true22Go West ScheduleGo West Shuttleyes
106Tri Delta Transit23Historic Tuolumne ScheduleTuolumne County Transityes
107Tri Delta Transit24Huntington ScheduleHuntington Park Expressyes
108true25Inglewood ScheduleI-Line Shuttleyes
109Union City Transit26La Campana ScheduleLa Campanayes
110Union City27La Puente ScheduleLa Puente Linkyes
111false28Lawndale Beat Vehicle PositionsLawndale Beatyes
112Vacaville City Coach29Lynwood ScheduleLynwood Breezeyes
113Vacaville City Coach30MVGO VehiclePositions Historic,MVGO TripUpdates HistoricMountain View Transportation Management Associationyes
114true31Maywood ScheduleMaywood Express Shuttleyes
115VINE Transit32Merced VehiclePositionsMerced The Busyes
116Napa VINE33Morro Bay ScheduleMorro Bay Transityes
34NoneGTransyes
35NoneHavasu Landing Resort & Casino Ferryyes
36NoneKern Transityes
37NoneSan Clemente Trolleyyes
38Playa Vista SchedulePlaya Vista Shuttleyes
39Plumas SchedulePlumas Transit Systemsyes
40Rosemead ScheduleRosemead Exploreryes
41Sage Stage ScheduleSage Stage Intercityyes
42San Diego Vehicle PositionsSan Diego Metropolitan Transit Systemyes
117true43San Juan Capistrano ScheduleSan Juan Capistrano Free Weekend Trolleyyes
118VTA44Sierra Madre ScheduleGateway Coachyes
119VTA45SunLine Vehicle Positions,SunLine Trip UpdatesSunLine Transityes
120true46Tracy ScheduleTRACERyes
121WestCat (Western Contra Costa)47Union City GMV ScheduleUnion City Transityes
122WestCAT48eTrans Schedule,eTrans VehiclePositions,eTrans TripUpdateseTransyes
\n" + "\n", + "
" ], "text/plain": [ - "" + " gtfs_datasets \\\n", + "0 AC Transit Alerts,Bay Area 511 Regional Alerts \n", + "1 Alhambra Schedule \n", + "2 Anaheim Resort Schedule \n", + "3 Arcadia Schedule \n", + "4 Artesia Schedule \n", + "5 Arvin Schedule \n", + "6 Auburn Schedule \n", + "7 Baldwin Park Schedule \n", + "8 Bay Area 511 Regional VehiclePositions \n", + "9 Bay Area 511 Union City Transit VehiclePositions \n", + "10 Bell Gardens Schedule \n", + "11 Bellflower Bus Schedule \n", + "12 Calabasas Schedule \n", + "13 Capitol Corridor Schedule ,Bay Area 511 ACE Schedule,Amtrak Schedule \n", + "14 Catalina Flyer Schedule Historic \n", + "15 Compton Schedule \n", + "16 Cudahy Schedule \n", + "17 DowneyLINK Schedule \n", + "18 El Segundo Schedule \n", + "19 Get Around Town Express Schedule \n", + "20 Glendora Schedule \n", + "21 Glenn Schedule \n", + "22 Go West Schedule \n", + "23 Historic Tuolumne Schedule \n", + "24 Huntington Schedule \n", + "25 Inglewood Schedule \n", + "26 La Campana Schedule \n", + "27 La Puente Schedule \n", + "28 Lawndale Beat Vehicle Positions \n", + "29 Lynwood Schedule \n", + "30 MVGO VehiclePositions Historic,MVGO TripUpdates Historic \n", + "31 Maywood Schedule \n", + "32 Merced VehiclePositions \n", + "33 Morro Bay Schedule \n", + "34 None \n", + "35 None \n", + "36 None \n", + "37 None \n", + "38 Playa Vista Schedule \n", + "39 Plumas Schedule \n", + "40 Rosemead Schedule \n", + "41 Sage Stage Schedule \n", + "42 San Diego Vehicle Positions \n", + "43 San Juan Capistrano Schedule \n", + "44 Sierra Madre Schedule \n", + "45 SunLine Vehicle Positions,SunLine Trip Updates \n", + "46 Tracy Schedule \n", + "47 Union City GMV Schedule \n", + "48 eTrans Schedule,eTrans VehiclePositions,eTrans TripUpdates \n", + "\n", + " services airtable_ticket \n", + "0 AC Transit yes \n", + "1 Alhambra Community Transit yes \n", + "2 Anaheim Resort Transportation yes \n", + "3 Arcadia Transit yes \n", + "4 Artesia Transit yes \n", + "5 Arvin Transit yes \n", + "6 Auburn Transit yes \n", + "7 Baldwin Park Express yes \n", + "8 AC Transit yes \n", + "9 Union City Transit yes \n", + "10 Bell Gardens Trolly yes \n", + "11 Bellflower Bus yes \n", + "12 Calabasas Shuttle and Trolley yes \n", + "13 Capitol Corridor yes \n", + "14 Catalina Flyer yes \n", + "15 Compton Renaissance Transit Service yes \n", + "16 Cudahy Area Rapid Transit yes \n", + "17 DowneyLINK yes \n", + "18 El Segundo Shuttles yes \n", + "19 Get Around Town Express yes \n", + "20 Glendora Shuttles yes \n", + "21 Glenn Ride yes \n", + "22 Go West Shuttle yes \n", + "23 Tuolumne County Transit yes \n", + "24 Huntington Park Express yes \n", + "25 I-Line Shuttle yes \n", + "26 La Campana yes \n", + "27 La Puente Link yes \n", + "28 Lawndale Beat yes \n", + "29 Lynwood Breeze yes \n", + "30 Mountain View Transportation Management Association yes \n", + "31 Maywood Express Shuttle yes \n", + "32 Merced The Bus yes \n", + "33 Morro Bay Transit yes \n", + "34 GTrans yes \n", + "35 Havasu Landing Resort & Casino Ferry yes \n", + "36 Kern Transit yes \n", + "37 San Clemente Trolley yes \n", + "38 Playa Vista Shuttle yes \n", + "39 Plumas Transit Systems yes \n", + "40 Rosemead Explorer yes \n", + "41 Sage Stage Intercity yes \n", + "42 San Diego Metropolitan Transit System yes \n", + "43 San Juan Capistrano Free Weekend Trolley yes \n", + "44 Gateway Coach yes \n", + "45 SunLine Transit yes \n", + "46 TRACER yes \n", + "47 Union City Transit yes \n", + "48 eTrans yes " ] }, - "execution_count": 102, + "execution_count": 101, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df_511_2.style.where(lambda val: 'false' in str(val), 'color: red')" + "open_airtable_tickets" ] }, { "cell_type": "code", - "execution_count": 104, - "id": "a2779bdb-cb88-463e-a320-d48fc2e09e74", - "metadata": { - "scrolled": true, - "tags": [] - }, + "execution_count": 96, + "id": "c6d592ce-fb2a-46bb-bf5b-8ce7f4a27dcc", + "metadata": {}, "outputs": [], "source": [ - "# open_airtable_tickets.sort_values(['gtfs_datasets'])" + "incomplete2 = pd.merge(\n", + " incomplete,\n", + " open_airtable_tickets,\n", + " left_on=\"name\",\n", + " right_on=\"gtfs_datasets\",\n", + " how=\"left\",\n", + " indicator=True,\n", + ").sort_values(\"name\")" ] }, { "cell_type": "code", - "execution_count": 62, - "id": "c6d592ce-fb2a-46bb-bf5b-8ce7f4a27dcc", - "metadata": {}, + "execution_count": 97, + "id": "af1804f6-e204-4c4d-94ef-cf667417a971", + "metadata": { + "scrolled": true, + "tags": [] + }, "outputs": [ { "data": { @@ -1402,7 +1785,7 @@ " \n", " \n", " \n", - " 17\n", + " 0\n", " Anaheim Resort Schedule\n", " 21.11\n", " 17.73\n", @@ -1432,7 +1815,7 @@ " left_only\n", " \n", " \n", - " 18\n", + " 3\n", " Bay Area 511 BART Schedule\n", " 39.31\n", " 0.00\n", @@ -1492,7 +1875,7 @@ " left_only\n", " \n", " \n", - " 0\n", + " 9\n", " Bay Area 511 Rio Vista Delta Breeze Schedule\n", " 0.00\n", " 0.00\n", @@ -1502,7 +1885,7 @@ " left_only\n", " \n", " \n", - " 9\n", + " 10\n", " Bay Area 511 SFO AirTrain Schedule\n", " 0.00\n", " 0.00\n", @@ -1512,9 +1895,9 @@ " left_only\n", " \n", " \n", - " 20\n", + " 11\n", " Bay Area 511 Santa Rosa CityBus Schedule\n", - " NaN\n", + " OK\n", " 17.11\n", " NaN\n", " NaN\n", @@ -1522,7 +1905,7 @@ " left_only\n", " \n", " \n", - " 10\n", + " 12\n", " Bay Area 511 South San Francisco Shuttle Schedule\n", " 0.00\n", " 0.00\n", @@ -1532,7 +1915,7 @@ " left_only\n", " \n", " \n", - " 11\n", + " 13\n", " Bay Area 511 Treasure Island Ferry Schedule\n", " 0.00\n", " 0.00\n", @@ -1542,7 +1925,7 @@ " left_only\n", " \n", " \n", - " 15\n", + " 14\n", " Bay Area 511 Union City Transit Schedule\n", " 16.10\n", " 9.38\n", @@ -1552,7 +1935,7 @@ " left_only\n", " \n", " \n", - " 12\n", + " 15\n", " Bay Area 511 Vacaville City Coach Schedule\n", " 0.00\n", " 0.00\n", @@ -1562,9 +1945,9 @@ " left_only\n", " \n", " \n", - " 22\n", + " 16\n", " Bay Area 511 Vine Transit Schedule\n", - " NaN\n", + " OK\n", " 37.77\n", " NaN\n", " NaN\n", @@ -1572,9 +1955,9 @@ " left_only\n", " \n", " \n", - " 19\n", + " 17\n", " Merced GMV Schedule\n", - " NaN\n", + " OK\n", " 0.00\n", " NaN\n", " NaN\n", @@ -1582,7 +1965,7 @@ " left_only\n", " \n", " \n", - " 14\n", + " 18\n", " SLORTA Schedule\n", " 15.74\n", " 15.60\n", @@ -1592,19 +1975,19 @@ " left_only\n", " \n", " \n", - " 3\n", + " 19\n", " San Diego Schedule\n", " 0.00\n", - " NaN\n", + " OK\n", " NaN\n", " NaN\n", " NaN\n", " left_only\n", " \n", " \n", - " 21\n", + " 20\n", " Santa Rosa CityBus Schedule\n", - " NaN\n", + " OK\n", " 21.26\n", " NaN\n", " NaN\n", @@ -1612,7 +1995,7 @@ " left_only\n", " \n", " \n", - " 13\n", + " 21\n", " SunLine Avail Schedule\n", " 0.96\n", " 0.71\n", @@ -1622,7 +2005,7 @@ " left_only\n", " \n", " \n", - " 16\n", + " 22\n", " Union City GMV Schedule\n", " 18.39\n", " 10.68\n", @@ -1634,7 +2017,7 @@ " \n", " 23\n", " Vine GMV Schedule\n", - " NaN\n", + " OK\n", " 37.77\n", " NaN\n", " NaN\n", @@ -1647,117 +2030,117 @@ ], "text/plain": [ " name \\\n", - "17 Anaheim Resort Schedule \n", + "0 Anaheim Resort Schedule \n", "1 Bay Area 511 ACE Schedule \n", "2 Bay Area 511 Angel Island-Tiburon Ferry Schedule \n", - "18 Bay Area 511 BART Schedule \n", + "3 Bay Area 511 BART Schedule \n", "4 Bay Area 511 Capitol Corridor Schedule \n", "5 Bay Area 511 Commute.org Schedule \n", "6 Bay Area 511 Golden Gate Ferry Schedule \n", "7 Bay Area 511 MVGO Schedule \n", "8 Bay Area 511 Mission Bay Schedule \n", - "0 Bay Area 511 Rio Vista Delta Breeze Schedule \n", - "9 Bay Area 511 SFO AirTrain Schedule \n", - "20 Bay Area 511 Santa Rosa CityBus Schedule \n", - "10 Bay Area 511 South San Francisco Shuttle Schedule \n", - "11 Bay Area 511 Treasure Island Ferry Schedule \n", - "15 Bay Area 511 Union City Transit Schedule \n", - "12 Bay Area 511 Vacaville City Coach Schedule \n", - "22 Bay Area 511 Vine Transit Schedule \n", - "19 Merced GMV Schedule \n", - "14 SLORTA Schedule \n", - "3 San Diego Schedule \n", - "21 Santa Rosa CityBus Schedule \n", - "13 SunLine Avail Schedule \n", - "16 Union City GMV Schedule \n", + "9 Bay Area 511 Rio Vista Delta Breeze Schedule \n", + "10 Bay Area 511 SFO AirTrain Schedule \n", + "11 Bay Area 511 Santa Rosa CityBus Schedule \n", + "12 Bay Area 511 South San Francisco Shuttle Schedule \n", + "13 Bay Area 511 Treasure Island Ferry Schedule \n", + "14 Bay Area 511 Union City Transit Schedule \n", + "15 Bay Area 511 Vacaville City Coach Schedule \n", + "16 Bay Area 511 Vine Transit Schedule \n", + "17 Merced GMV Schedule \n", + "18 SLORTA Schedule \n", + "19 San Diego Schedule \n", + "20 Santa Rosa CityBus Schedule \n", + "21 SunLine Avail Schedule \n", + "22 Union City GMV Schedule \n", "23 Vine GMV Schedule \n", "\n", - " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \\\n", - "17 21.11 17.73 \n", - "1 0.00 0.00 \n", - "2 0.00 0.00 \n", - "18 39.31 0.00 \n", - "4 0.00 0.00 \n", - "5 0.00 0.00 \n", - "6 0.00 0.00 \n", - "7 0.00 0.00 \n", - "8 0.00 0.00 \n", - "0 0.00 0.00 \n", - "9 0.00 0.00 \n", - "20 NaN 17.11 \n", - "10 0.00 0.00 \n", - "11 0.00 0.00 \n", - "15 16.10 9.38 \n", - "12 0.00 0.00 \n", - "22 NaN 37.77 \n", - "19 NaN 0.00 \n", - "14 15.74 15.60 \n", - "3 0.00 NaN \n", - "21 NaN 21.26 \n", - "13 0.96 0.71 \n", - "16 18.39 10.68 \n", - "23 NaN 37.77 \n", + " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \\\n", + "0 21.11 17.73 \n", + "1 0.00 0.00 \n", + "2 0.00 0.00 \n", + "3 39.31 0.00 \n", + "4 0.00 0.00 \n", + "5 0.00 0.00 \n", + "6 0.00 0.00 \n", + "7 0.00 0.00 \n", + "8 0.00 0.00 \n", + "9 0.00 0.00 \n", + "10 0.00 0.00 \n", + "11 OK 17.11 \n", + "12 0.00 0.00 \n", + "13 0.00 0.00 \n", + "14 16.10 9.38 \n", + "15 0.00 0.00 \n", + "16 OK 37.77 \n", + "17 OK 0.00 \n", + "18 15.74 15.60 \n", + "19 0.00 OK \n", + "20 OK 21.26 \n", + "21 0.96 0.71 \n", + "22 18.39 10.68 \n", + "23 OK 37.77 \n", "\n", " gtfs_datasets services airtable_ticket \\\n", - "17 Anaheim Resort Schedule Anaheim Resort Transportation yes \n", + "0 Anaheim Resort Schedule Anaheim Resort Transportation yes \n", "1 NaN NaN NaN \n", "2 NaN NaN NaN \n", - "18 NaN NaN NaN \n", + "3 NaN NaN NaN \n", "4 NaN NaN NaN \n", "5 NaN NaN NaN \n", "6 NaN NaN NaN \n", "7 NaN NaN NaN \n", "8 NaN NaN NaN \n", - "0 NaN NaN NaN \n", "9 NaN NaN NaN \n", - "20 NaN NaN NaN \n", "10 NaN NaN NaN \n", "11 NaN NaN NaN \n", - "15 NaN NaN NaN \n", "12 NaN NaN NaN \n", - "22 NaN NaN NaN \n", - "19 NaN NaN NaN \n", + "13 NaN NaN NaN \n", "14 NaN NaN NaN \n", - "3 NaN NaN NaN \n", + "15 NaN NaN NaN \n", + "16 NaN NaN NaN \n", + "17 NaN NaN NaN \n", + "18 NaN NaN NaN \n", + "19 NaN NaN NaN \n", + "20 NaN NaN NaN \n", "21 NaN NaN NaN \n", - "13 NaN NaN NaN \n", - "16 Union City GMV Schedule Union City Transit yes \n", + "22 Union City GMV Schedule Union City Transit yes \n", "23 NaN NaN NaN \n", "\n", " _merge \n", - "17 both \n", + "0 both \n", "1 left_only \n", "2 left_only \n", - "18 left_only \n", + "3 left_only \n", "4 left_only \n", "5 left_only \n", "6 left_only \n", "7 left_only \n", "8 left_only \n", - "0 left_only \n", "9 left_only \n", - "20 left_only \n", "10 left_only \n", "11 left_only \n", - "15 left_only \n", "12 left_only \n", - "22 left_only \n", - "19 left_only \n", + "13 left_only \n", "14 left_only \n", - "3 left_only \n", + "15 left_only \n", + "16 left_only \n", + "17 left_only \n", + "18 left_only \n", + "19 left_only \n", + "20 left_only \n", "21 left_only \n", - "13 left_only \n", - "16 both \n", + "22 both \n", "23 left_only " ] }, - "execution_count": 62, + "execution_count": 97, "metadata": {}, "output_type": "execute_result" } ], "source": [ - " pd.merge(incomplete, open_airtable_tickets, left_on = \"name\", right_on = \"gtfs_datasets\", how = \"left\", indicator = True).sort_values('name')" + "incomplete2" ] } ], From d3a02c6ba95c730e9ad2903a355075097f1b4c20 Mon Sep 17 00:00:00 2001 From: amandaha8 Date: Wed, 27 Sep 2023 20:35:11 +0000 Subject: [PATCH 3/4] turned work into function --- ah_starterkit/Untitled.ipynb | 151 ++ gtfs_quality_check/gtfs_checking.ipynb | 2003 +++++------------------ gtfs_quality_check/testing_calitp.ipynb | 205 +++ 3 files changed, 800 insertions(+), 1559 deletions(-) create mode 100644 ah_starterkit/Untitled.ipynb create mode 100644 gtfs_quality_check/testing_calitp.ipynb diff --git a/ah_starterkit/Untitled.ipynb b/ah_starterkit/Untitled.ipynb new file mode 100644 index 000000000..40f9d0af6 --- /dev/null +++ b/ah_starterkit/Untitled.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 12, + "id": "3423feb2-b10a-4573-9c51-d91c1bbc9bf3", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Literal, cast" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "e8758d7d-aaf3-4522-86eb-eb26d2d2596e", + "metadata": {}, + "outputs": [], + "source": [ + "hello_t = Literal['Hello there', 'Hello world']" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "10b142a1-a184-4cb4-9e26-f52aad555c0a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "typing.Literal['Hello there', 'Hello world']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hello_t" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "fcef3360-c76a-4340-a4b9-4c7857ff8a2f", + "metadata": {}, + "outputs": [], + "source": [ + "def verify(word: Literal['there', 'world']) -> hello_t:\n", + " a = cast(hello_t, word)\n", + " return a" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c0a7c587-78b6-4939-a20c-9dcd087642ce", + "metadata": {}, + "outputs": [], + "source": [ + "a = verify('there')" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "88135247-761c-4e98-98ba-d0850c17dfc9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'there'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "822bd804-c832-4bd6-b08a-ac4eaaa2c831", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'world'" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "verify('world')" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "997c2d77-c496-41d3-b6bc-5c61b46782ad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'candy'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "verify('candy')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gtfs_quality_check/gtfs_checking.ipynb b/gtfs_quality_check/gtfs_checking.ipynb index eddfa8543..d8b27c61c 100644 --- a/gtfs_quality_check/gtfs_checking.ipynb +++ b/gtfs_quality_check/gtfs_checking.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "9a12bb6d-e7f8-4517-ad77-1012b4716575", "metadata": {}, "outputs": [], @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "457f42cc-b42d-49c1-9142-82efb618b17c", "metadata": {}, "outputs": [], @@ -37,135 +37,112 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "a60a3276-57a9-420c-a8a6-21aa5ff5b8a2", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", - " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" - ] - } - ], - "source": [ - "expired_df = pd.read_excel(\"./feed_info_expired_feeds_2023-09-21T19_13_29.263825Z.xlsx\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9b47e637-d755-470f-b585-eb1ee46dc918", - "metadata": {}, - "outputs": [], - "source": [ - "expired_df = to_snakecase(expired_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "cd9429b4-5108-4c9e-b86b-36fbe6171212", - "metadata": {}, - "outputs": [], - "source": [ - "# Don't look at LACMTA stuff\n", - "no_lacmta = expired_df[~expired_df[\"dim_gtfs_datasets_→_uri\"].str.contains(\"LACMTA\")]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "177b7d83-5bec-498e-b3ca-eeefbdb436fc", + "execution_count": 8, + "id": "27d3987c-298d-483e-9afc-999b3fece287", "metadata": {}, "outputs": [], "source": [ - "no_lacmta2 = no_lacmta[\n", + "def load_catastrophic_errors(excel_file:str):\n", + " df = pd.read_excel(excel_file)\n", + " df = to_snakecase(df)\n", + " \n", + " # Don't look at LACMTA stuff\n", + " no_lacmta = df[~df[\"dim_gtfs_datasets_→_uri\"].str.contains(\"LACMTA\")]\n", + " \n", + " no_lacmta2 = no_lacmta[\n", " [\n", " \"dim_provider_gtfs_data_→_service_name\",\n", " \"dim_county_geography_→_caltrans_district\",\n", " \"date\",\n", - " ]\n", - "].sort_values(\n", - " [\"dim_provider_gtfs_data_→_service_name\", \"date\"], ascending=[False, False]\n", - ")" + " ]].sort_values(\n", + " [\"dim_provider_gtfs_data_→_service_name\", \"date\"], ascending=[False, False])\n", + " \n", + " return no_lacmta2" ] }, { "cell_type": "code", - "execution_count": 39, - "id": "a1a0be85-784c-483d-ac15-e0a8c3da8e5e", + "execution_count": 9, + "id": "cd9429b4-5108-4c9e-b86b-36fbe6171212", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] + } + ], "source": [ - "open_airtable_tickets = to_snakecase(\n", - " pd.read_csv(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\")\n", - ")" + "cat_df = load_catastrophic_errors(\"./feed_info_expired_feeds_2023-09-21T19_13_29.263825Z.xlsx\")" ] }, { "cell_type": "code", - "execution_count": 40, - "id": "05c95752-05ca-4147-94f0-3cd1859174d0", + "execution_count": 14, + "id": "96ac0b97-70c3-436b-a4c1-7614db24614c", "metadata": {}, "outputs": [], "source": [ - "open_airtable_tickets = open_airtable_tickets.fillna(\"None\")" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "22440833-8d90-4bcd-8e77-8ea23f88702f", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [], - "source": [ - "open_airtable_tickets = (\n", - " open_airtable_tickets[\n", - " [\n", + "def load_airtable(csv_file:str):\n", + " df = to_snakecase(pd.read_csv(csv_file))\n", + " \n", + " df = df.fillna(\"None\")\n", + " \n", + " df = (df[[\n", " \"description\",\n", " \"gtfs_datasets\",\n", - " \"services\",\n", - " ]\n", - " ]\n", + " \"services\"]]\n", " .sort_values([\"gtfs_datasets\", \"services\"])\n", - " .reset_index(drop=True)\n", - ")" + " .reset_index(drop=True))\n", + " \n", + " df[\"airtable_ticket\"] = \"Yes\"\n", + " \n", + " return df" ] }, { "cell_type": "code", - "execution_count": 42, - "id": "1728d945-d52b-4888-9007-012f857c0ff3", + "execution_count": 15, + "id": "0e104d0d-02ec-4e6e-9165-660c3548681b", "metadata": {}, "outputs": [], "source": [ - "open_airtable_tickets[\"airtable_ticket\"] = \"yes\"" + "airtable = load_airtable(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\")" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 16, "id": "bfac56a1-49c7-4b28-bc6e-e7009dc94ad7", "metadata": {}, "outputs": [], "source": [ - "no_lacmta2_summary = (\n", - " no_lacmta2.groupby([\"dim_provider_gtfs_data_→_service_name\"])\n", + "def summarize_cat(catastrophic_data:pd.DataFrame, airtable_data:pd.DataFrame) -> pd.DataFrame:\n", + " \n", + " cat_summary = (\n", + " catastrophic_data.groupby([\"dim_provider_gtfs_data_→_service_name\"])\n", " .agg({\"date\": \"count\"})\n", " .reset_index()\n", - " .rename(columns={\"date\": \"# of days with expired feed\"})\n", - ")" + " .rename(columns={\"date\": \"# of days with expired feed\"}))\n", + " \n", + " display(cat_summary)\n", + " \n", + " m1 = pd.merge(\n", + " cat_summary,\n", + " airtable_data,\n", + " left_on=[\"dim_provider_gtfs_data_→_service_name\"],\n", + " right_on=[\"services\"],\n", + " how=\"left\",)\n", + " \n", + " display(m1)" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 17, "id": "c3914805-a115-4029-97da-bc6bd2446706", "metadata": {}, "outputs": [ @@ -192,9 +169,6 @@ " \n", " dim_provider_gtfs_data_→_service_name\n", " # of days with expired feed\n", - " gtfs_datasets\n", - " services\n", - " airtable_ticket\n", " \n", " \n", " \n", @@ -202,73 +176,37 @@ " 0\n", " Morro Bay Transit\n", " 14\n", - " Morro Bay Schedule\n", - " Morro Bay Transit\n", - " yes\n", " \n", " \n", " 1\n", " Plumas Transit Systems\n", " 14\n", - " Plumas Schedule\n", - " Plumas Transit Systems\n", - " yes\n", " \n", " \n", " 2\n", " Sage Stage Intercity\n", " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", " \n", " \n", " 3\n", " TRACER\n", " 14\n", - " Tracy Schedule\n", - " TRACER\n", - " yes\n", " \n", " \n", "\n", "" ], "text/plain": [ - " dim_provider_gtfs_data_→_service_name # of days with expired feed \\\n", - "0 Morro Bay Transit 14 \n", - "1 Plumas Transit Systems 14 \n", - "2 Sage Stage Intercity 2 \n", - "3 TRACER 14 \n", - "\n", - " gtfs_datasets services airtable_ticket \n", - "0 Morro Bay Schedule Morro Bay Transit yes \n", - "1 Plumas Schedule Plumas Transit Systems yes \n", - "2 NaN NaN NaN \n", - "3 Tracy Schedule TRACER yes " + " dim_provider_gtfs_data_→_service_name # of days with expired feed\n", + "0 Morro Bay Transit 14\n", + "1 Plumas Transit Systems 14\n", + "2 Sage Stage Intercity 2\n", + "3 TRACER 14" ] }, - "execution_count": 44, "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pd.merge(\n", - " no_lacmta2_summary,\n", - " open_airtable_tickets,\n", - " left_on=[\"dim_provider_gtfs_data_→_service_name\"],\n", - " right_on=[\"services\"],\n", - " how=\"left\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "id": "8da60e37-7e1e-4518-8f1a-eaf7e7d41086", - "metadata": {}, - "outputs": [ + "output_type": "display_data" + }, { "data": { "text/html": [ @@ -291,380 +229,80 @@ " \n", " \n", " dim_provider_gtfs_data_→_service_name\n", - " dim_county_geography_→_caltrans_district\n", - " date\n", + " # of days with expired feed\n", + " description\n", + " gtfs_datasets\n", + " services\n", + " airtable_ticket\n", " \n", " \n", " \n", " \n", - " 8\n", - " TRACER\n", - " 10\n", - " 2023-09-20\n", - " \n", - " \n", - " 18\n", - " TRACER\n", - " 10\n", - " 2023-09-19\n", - " \n", - " \n", - " 28\n", - " TRACER\n", - " 10\n", - " 2023-09-18\n", - " \n", - " \n", - " 38\n", - " TRACER\n", - " 10\n", - " 2023-09-17\n", - " \n", - " \n", - " 48\n", - " TRACER\n", - " 10\n", - " 2023-09-16\n", - " \n", - " \n", - " 58\n", - " TRACER\n", - " 10\n", - " 2023-09-15\n", - " \n", - " \n", - " 68\n", - " TRACER\n", - " 10\n", - " 2023-09-14\n", - " \n", - " \n", - " 78\n", - " TRACER\n", - " 10\n", - " 2023-09-13\n", - " \n", - " \n", - " 88\n", - " TRACER\n", - " 10\n", - " 2023-09-12\n", - " \n", - " \n", - " 98\n", - " TRACER\n", - " 10\n", - " 2023-09-11\n", - " \n", - " \n", - " 108\n", - " TRACER\n", - " 10\n", - " 2023-09-10\n", - " \n", - " \n", - " 118\n", - " TRACER\n", - " 10\n", - " 2023-09-09\n", - " \n", - " \n", - " 128\n", - " TRACER\n", - " 10\n", - " 2023-09-08\n", - " \n", - " \n", - " 139\n", - " TRACER\n", - " 10\n", - " 2023-09-07\n", - " \n", - " \n", - " 129\n", - " Sage Stage Intercity\n", - " 2\n", - " 2023-09-08\n", - " \n", - " \n", - " 140\n", - " Sage Stage Intercity\n", - " 2\n", - " 2023-09-07\n", - " \n", - " \n", - " 9\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-20\n", - " \n", - " \n", - " 19\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-19\n", - " \n", - " \n", - " 29\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-18\n", - " \n", - " \n", - " 39\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-17\n", - " \n", - " \n", - " 49\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-16\n", - " \n", - " \n", - " 59\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-15\n", - " \n", - " \n", - " 69\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-14\n", - " \n", - " \n", - " 79\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-13\n", - " \n", - " \n", - " 89\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-12\n", - " \n", - " \n", - " 99\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-11\n", - " \n", - " \n", - " 109\n", - " Plumas Transit Systems\n", - " 2\n", - " 2023-09-10\n", + " 0\n", + " Morro Bay Transit\n", + " 14\n", + " City of Morro Bay: Follow-up about GTFS Schedule\n", + " Morro Bay Schedule\n", + " Morro Bay Transit\n", + " Yes\n", " \n", " \n", - " 119\n", + " 1\n", " Plumas Transit Systems\n", - " 2\n", - " 2023-09-09\n", - " \n", - " \n", - " 130\n", + " 14\n", + " Based on the catastrophic errors dashboard, Plumas has not produced data since September 2. Per Evan, \"Plumas has a feed_info.txt file that says the feed was only valid until September 1 while the calendar says the feed is valid all the way until December 1. A strict interpretation of the GTFS spec says that any service beyond the feed end date is merely advisory, but some trip planners may still chose to display the service up until the end date.\" This is the second ticket opened for Plumas.\n", + " Plumas Schedule\n", " Plumas Transit Systems\n", - " 2\n", - " 2023-09-08\n", + " Yes\n", " \n", " \n", - " 141\n", - " Plumas Transit Systems\n", + " 2\n", + " Sage Stage Intercity\n", " 2\n", - " 2023-09-07\n", - " \n", - " \n", - " 6\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-20\n", - " \n", - " \n", - " 16\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-19\n", - " \n", - " \n", - " 26\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-18\n", - " \n", - " \n", - " 36\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-17\n", - " \n", - " \n", - " 46\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-16\n", - " \n", - " \n", - " 56\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-15\n", - " \n", - " \n", - " 66\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-14\n", - " \n", - " \n", - " 76\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-13\n", - " \n", - " \n", - " 86\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-12\n", - " \n", - " \n", - " 96\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-11\n", - " \n", - " \n", - " 106\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-10\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", - " 116\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-09\n", - " \n", - " \n", - " 126\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-08\n", - " \n", - " \n", - " 137\n", - " Morro Bay Transit\n", - " 5\n", - " 2023-09-07\n", + " 3\n", + " TRACER\n", + " 14\n", + " City of Tracy: GTFS Schedule Feed Expired\n", + " Tracy Schedule\n", + " TRACER\n", + " Yes\n", " \n", " \n", "\n", "" ], "text/plain": [ - " dim_provider_gtfs_data_→_service_name \\\n", - "8 TRACER \n", - "18 TRACER \n", - "28 TRACER \n", - "38 TRACER \n", - "48 TRACER \n", - "58 TRACER \n", - "68 TRACER \n", - "78 TRACER \n", - "88 TRACER \n", - "98 TRACER \n", - "108 TRACER \n", - "118 TRACER \n", - "128 TRACER \n", - "139 TRACER \n", - "129 Sage Stage Intercity \n", - "140 Sage Stage Intercity \n", - "9 Plumas Transit Systems \n", - "19 Plumas Transit Systems \n", - "29 Plumas Transit Systems \n", - "39 Plumas Transit Systems \n", - "49 Plumas Transit Systems \n", - "59 Plumas Transit Systems \n", - "69 Plumas Transit Systems \n", - "79 Plumas Transit Systems \n", - "89 Plumas Transit Systems \n", - "99 Plumas Transit Systems \n", - "109 Plumas Transit Systems \n", - "119 Plumas Transit Systems \n", - "130 Plumas Transit Systems \n", - "141 Plumas Transit Systems \n", - "6 Morro Bay Transit \n", - "16 Morro Bay Transit \n", - "26 Morro Bay Transit \n", - "36 Morro Bay Transit \n", - "46 Morro Bay Transit \n", - "56 Morro Bay Transit \n", - "66 Morro Bay Transit \n", - "76 Morro Bay Transit \n", - "86 Morro Bay Transit \n", - "96 Morro Bay Transit \n", - "106 Morro Bay Transit \n", - "116 Morro Bay Transit \n", - "126 Morro Bay Transit \n", - "137 Morro Bay Transit \n", + " dim_provider_gtfs_data_→_service_name # of days with expired feed \\\n", + "0 Morro Bay Transit 14 \n", + "1 Plumas Transit Systems 14 \n", + "2 Sage Stage Intercity 2 \n", + "3 TRACER 14 \n", + "\n", + " description \\\n", + "0 City of Morro Bay: Follow-up about GTFS Schedule \n", + "1 Based on the catastrophic errors dashboard, Plumas has not produced data since September 2. Per Evan, \"Plumas has a feed_info.txt file that says the feed was only valid until September 1 while the calendar says the feed is valid all the way until December 1. A strict interpretation of the GTFS spec says that any service beyond the feed end date is merely advisory, but some trip planners may still chose to display the service up until the end date.\" This is the second ticket opened for Plumas. \n", + "2 NaN \n", + "3 City of Tracy: GTFS Schedule Feed Expired \n", "\n", - " dim_county_geography_→_caltrans_district date \n", - "8 10 2023-09-20 \n", - "18 10 2023-09-19 \n", - "28 10 2023-09-18 \n", - "38 10 2023-09-17 \n", - "48 10 2023-09-16 \n", - "58 10 2023-09-15 \n", - "68 10 2023-09-14 \n", - "78 10 2023-09-13 \n", - "88 10 2023-09-12 \n", - "98 10 2023-09-11 \n", - "108 10 2023-09-10 \n", - "118 10 2023-09-09 \n", - "128 10 2023-09-08 \n", - "139 10 2023-09-07 \n", - "129 2 2023-09-08 \n", - "140 2 2023-09-07 \n", - "9 2 2023-09-20 \n", - "19 2 2023-09-19 \n", - "29 2 2023-09-18 \n", - "39 2 2023-09-17 \n", - "49 2 2023-09-16 \n", - "59 2 2023-09-15 \n", - "69 2 2023-09-14 \n", - "79 2 2023-09-13 \n", - "89 2 2023-09-12 \n", - "99 2 2023-09-11 \n", - "109 2 2023-09-10 \n", - "119 2 2023-09-09 \n", - "130 2 2023-09-08 \n", - "141 2 2023-09-07 \n", - "6 5 2023-09-20 \n", - "16 5 2023-09-19 \n", - "26 5 2023-09-18 \n", - "36 5 2023-09-17 \n", - "46 5 2023-09-16 \n", - "56 5 2023-09-15 \n", - "66 5 2023-09-14 \n", - "76 5 2023-09-13 \n", - "86 5 2023-09-12 \n", - "96 5 2023-09-11 \n", - "106 5 2023-09-10 \n", - "116 5 2023-09-09 \n", - "126 5 2023-09-08 \n", - "137 5 2023-09-07 " + " gtfs_datasets services airtable_ticket \n", + "0 Morro Bay Schedule Morro Bay Transit Yes \n", + "1 Plumas Schedule Plumas Transit Systems Yes \n", + "2 NaN NaN NaN \n", + "3 Tracy Schedule TRACER Yes " ] }, - "execution_count": 45, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "no_lacmta2" + "summarize_cat(cat_df,airtable)" ] }, { @@ -677,618 +315,119 @@ }, { "cell_type": "code", - "execution_count": 46, - "id": "18276af1-e87e-4218-b486-47a1bdd1da6d", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", - " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" - ] - } - ], - "source": [ - "trip_update_df = to_snakecase(\n", - " pd.read_excel(\n", - " \"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\"\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "940c0a74-82eb-40e8-b351-4366b4de6cfd", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", - " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" - ] - } - ], - "source": [ - "vp_df = to_snakecase(\n", - " pd.read_excel(\n", - " \"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\"\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "681da5bd-6fda-4e2d-afa6-1817f69c3140", + "execution_count": 26, + "id": "b3395d37-99de-492b-94bf-eadef5cf527c", "metadata": {}, "outputs": [], "source": [ - "low_tu = (\n", - " trip_update_df[trip_update_df[\"%_of_trips_with_tu_messages\"] < 41].sort_values(\n", + "def load_tu(tu_excel_file:str):\n", + " df = to_snakecase(pd.read_excel(tu_excel_file))\n", + " df = (\n", + " df[df[\"%_of_trips_with_tu_messages\"] < 41].sort_values(\n", " [\"%_of_trips_with_tu_messages\"]\n", - " )\n", - ").reset_index(drop=True)" + " )).reset_index(drop=True)\n", + " return df" ] }, { "cell_type": "code", - "execution_count": 49, - "id": "f235a5ed-85aa-452d-bc48-7499dab186ce", + "execution_count": 23, + "id": "2cfd0fed-560b-4f85-ad80-0e69ead44061", "metadata": {}, "outputs": [], "source": [ - "low_vp = (\n", - " vp_df[vp_df[\"%_of_trips_with_vp_messages\"] < 41].sort_values(\n", + "def load_vp(vp_excel_file:str):\n", + " df = to_snakecase(pd.read_excel(vp_excel_file))\n", + " \n", + " df = (\n", + " df[df[\"%_of_trips_with_vp_messages\"] < 41].sort_values(\n", " [\"%_of_trips_with_vp_messages\"]\n", - " )\n", - ").reset_index(drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "id": "d84c1256-827b-4f21-890b-4d1b2d861145", - "metadata": {}, - "outputs": [], - "source": [ - "incomplete = pd.merge(low_tu, low_vp, on=\"name\", how=\"outer\")\n", - "incomplete = incomplete.sort_values([\"name\"]).reset_index(drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "id": "8d9faa5c-959f-4203-9879-67cb32032711", - "metadata": {}, - "outputs": [], - "source": [ - "incomplete = incomplete.fillna(\"OK\")" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "id": "d62ff2ad-cfe5-4555-9e33-49b44680b6e9", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [], - "source": [ - "# open_airtable_tickets" + " )).reset_index(drop=True)\n", + " return df" ] }, { "cell_type": "code", - "execution_count": 53, - "id": "3e5b4267-0778-41f2-a1bd-29c1c783d2bb", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
name%_of_trips_with_tu_messages%_of_trips_with_vp_messages
0Anaheim Resort Schedule21.1117.73
1Bay Area 511 ACE Schedule0.000.00
2Bay Area 511 Angel Island-Tiburon Ferry Schedule0.000.00
3Bay Area 511 BART Schedule39.310.00
4Bay Area 511 Capitol Corridor Schedule0.000.00
5Bay Area 511 Commute.org Schedule0.000.00
6Bay Area 511 Golden Gate Ferry Schedule0.000.00
7Bay Area 511 MVGO Schedule0.000.00
8Bay Area 511 Mission Bay Schedule0.000.00
9Bay Area 511 Rio Vista Delta Breeze Schedule0.000.00
10Bay Area 511 SFO AirTrain Schedule0.000.00
11Bay Area 511 Santa Rosa CityBus ScheduleOK17.11
12Bay Area 511 South San Francisco Shuttle Schedule0.000.00
13Bay Area 511 Treasure Island Ferry Schedule0.000.00
14Bay Area 511 Union City Transit Schedule16.109.38
15Bay Area 511 Vacaville City Coach Schedule0.000.00
16Bay Area 511 Vine Transit ScheduleOK37.77
17Merced GMV ScheduleOK0.00
18SLORTA Schedule15.7415.60
19San Diego Schedule0.00OK
20Santa Rosa CityBus ScheduleOK21.26
21SunLine Avail Schedule0.960.71
22Union City GMV Schedule18.3910.68
23Vine GMV ScheduleOK37.77
\n", - "
" - ], - "text/plain": [ - " name \\\n", - "0 Anaheim Resort Schedule \n", - "1 Bay Area 511 ACE Schedule \n", - "2 Bay Area 511 Angel Island-Tiburon Ferry Schedule \n", - "3 Bay Area 511 BART Schedule \n", - "4 Bay Area 511 Capitol Corridor Schedule \n", - "5 Bay Area 511 Commute.org Schedule \n", - "6 Bay Area 511 Golden Gate Ferry Schedule \n", - "7 Bay Area 511 MVGO Schedule \n", - "8 Bay Area 511 Mission Bay Schedule \n", - "9 Bay Area 511 Rio Vista Delta Breeze Schedule \n", - "10 Bay Area 511 SFO AirTrain Schedule \n", - "11 Bay Area 511 Santa Rosa CityBus Schedule \n", - "12 Bay Area 511 South San Francisco Shuttle Schedule \n", - "13 Bay Area 511 Treasure Island Ferry Schedule \n", - "14 Bay Area 511 Union City Transit Schedule \n", - "15 Bay Area 511 Vacaville City Coach Schedule \n", - "16 Bay Area 511 Vine Transit Schedule \n", - "17 Merced GMV Schedule \n", - "18 SLORTA Schedule \n", - "19 San Diego Schedule \n", - "20 Santa Rosa CityBus Schedule \n", - "21 SunLine Avail Schedule \n", - "22 Union City GMV Schedule \n", - "23 Vine GMV Schedule \n", - "\n", - " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \n", - "0 21.11 17.73 \n", - "1 0.00 0.00 \n", - "2 0.00 0.00 \n", - "3 39.31 0.00 \n", - "4 0.00 0.00 \n", - "5 0.00 0.00 \n", - "6 0.00 0.00 \n", - "7 0.00 0.00 \n", - "8 0.00 0.00 \n", - "9 0.00 0.00 \n", - "10 0.00 0.00 \n", - "11 OK 17.11 \n", - "12 0.00 0.00 \n", - "13 0.00 0.00 \n", - "14 16.10 9.38 \n", - "15 0.00 0.00 \n", - "16 OK 37.77 \n", - "17 OK 0.00 \n", - "18 15.74 15.60 \n", - "19 0.00 OK \n", - "20 OK 21.26 \n", - "21 0.96 0.71 \n", - "22 18.39 10.68 \n", - "23 OK 37.77 " - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "incomplete" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "id": "f072f171-cbfc-46b7-a752-88ede3bdf30e", - "metadata": {}, - "outputs": [], - "source": [ - "# incomplete.name = incomplete.name.str.replace('Schedule','')" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "id": "4155eabf-6a8c-4dcf-a740-3075fbb785b3", - "metadata": {}, - "outputs": [], - "source": [ - "# Read in 511\n", - "df_511 = to_snakecase(pd.read_excel(\"./API511.xlsx\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "id": "57cfcd92-2580-4b86-8383-1b4e8407e222", - "metadata": {}, - "outputs": [], - "source": [ - "df_511 = df_511.rename(columns={df_511.columns[0]: \"new\"})" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "63fa2dfc-fa14-45f4-be7b-a416073dd7b4", + "execution_count": 24, + "id": "18276af1-e87e-4218-b486-47a1bdd1da6d", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Index(['new'], dtype='object')" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] } ], "source": [ - "df_511.columns" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "id": "f91d42cb-a1ea-44f8-be1d-dca2624dd00b", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [], - "source": [ - "df_511_2 = df_511[df_511[\"new\"].str.contains((\"Name|Monitored\"))].reset_index(drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "id": "6d08d56a-b8a6-4052-bf75-8925554f8733", - "metadata": {}, - "outputs": [], - "source": [ - "df_511_2.new = (\n", - " df_511_2.new.str.replace(\">\", \"\")\n", - " .str.replace(\"ShortName\", \"\")\n", - " .str.replace(\"Name\", \"\")\n", - " .str.replace(\"Monitored\", \"\")\n", - " .str.replace(\">\", \"\")\n", - " .str.replace(\"/\", \"\")\n", - ")" + "tu_df = load_incomplete_tu( \"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\")" ] }, { "cell_type": "code", - "execution_count": 103, - "id": "89d4a2c6-bec4-48a6-8612-e80c7a26111d", + "execution_count": 25, + "id": "60647b1c-bbdb-4056-ab45-9cc9e6b512e3", "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
new
13AC TRANSIT
22Bay Area Rapid Transit
25Caltrain
37County Connection
40Dumbarton Express Consortium
43Emery Go-Round
46FAST
52Golden Gate Transit
55Livermore Amador Valley Transit Authority
58Marin Transit
64MVgo Mountain View
67Petaluma
76SamTrans
79San Francisco Bay Ferry
85San Francisco Municipal Transportation Agency
91Santa Rosa CityBus
94SolTrans
97Sonoma County Transit
100Sonoma Marin Area Rail Transit
106Tri Delta Transit
109Union City Transit
115VINE Transit
118VTA
121WestCat (Western Contra Costa)
\n", - "
" - ], - "text/plain": [ - " new\n", - "13 AC TRANSIT\n", - "22 Bay Area Rapid Transit\n", - "25 Caltrain\n", - "37 County Connection\n", - "40 Dumbarton Express Consortium\n", - "43 Emery Go-Round\n", - "46 FAST\n", - "52 Golden Gate Transit\n", - "55 Livermore Amador Valley Transit Authority\n", - "58 Marin Transit\n", - "64 MVgo Mountain View\n", - "67 Petaluma\n", - "76 SamTrans\n", - "79 San Francisco Bay Ferry\n", - "85 San Francisco Municipal Transportation Agency\n", - "91 Santa Rosa CityBus\n", - "94 SolTrans\n", - "97 Sonoma County Transit\n", - "100 Sonoma Marin Area Rail Transit\n", - "106 Tri Delta Transit\n", - "109 Union City Transit\n", - "115 VINE Transit\n", - "118 VTA\n", - "121 WestCat (Western Contra Costa)" - ] - }, - "execution_count": 103, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] } ], "source": [ - "df_511_2[(df_511_2.new == \"true\").shift(1).fillna(False)]" + "vp_df = load_incomplete_tu(\"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\")" ] }, { "cell_type": "code", - "execution_count": 101, - "id": "a2779bdb-cb88-463e-a320-d48fc2e09e74", + "execution_count": 44, + "id": "0d2c4751-a485-4c6d-9154-d58b9348699c", + "metadata": {}, + "outputs": [], + "source": [ + "def api_511(excel_file:str)-> pd.DataFrame:\n", + " # Read in 511\n", + " df = to_snakecase(pd.read_excel(excel_file))\n", + " df = df.rename(columns={df.columns[0]: \"new\"})\n", + " \n", + " # Only keep rows that have the string name or monitored\n", + " df = df[df[\"new\"].str.contains((\"Name|Monitored\"))].reset_index(drop=True)\n", + " \n", + " # Get rid of random characters\n", + " df.new = (\n", + " df.new.str.replace(\">\", \"\")\n", + " .str.replace(\"ShortName\", \"\")\n", + " .str.replace(\"Name\", \"\")\n", + " .str.replace(\"Monitored\", \"\")\n", + " .str.replace(\"<\", \"\")\n", + " .str.replace(\"/\", \"\"))\n", + " \n", + " # display(df)\n", + " print('These are Bay Area feeds that keep track of RT')\n", + " display(df[(df.new == \"true\").shift(1).fillna(False)])" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "a073faad-7223-4218-b767-6a32153d8ca4", "metadata": { - "scrolled": true, "tags": [] }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is only Bay Area feeds that keep track of RT\n" + ] + }, { "data": { "text/html": [ @@ -1310,449 +449,194 @@ " \n", " \n", " \n", - " gtfs_datasets\n", - " services\n", - " airtable_ticket\n", + " new\n", " \n", " \n", " \n", " \n", - " 0\n", - " AC Transit Alerts,Bay Area 511 Regional Alerts\n", - " AC Transit\n", - " yes\n", - " \n", - " \n", - " 1\n", - " Alhambra Schedule\n", - " Alhambra Community Transit\n", - " yes\n", - " \n", - " \n", - " 2\n", - " Anaheim Resort Schedule\n", - " Anaheim Resort Transportation\n", - " yes\n", - " \n", - " \n", - " 3\n", - " Arcadia Schedule\n", - " Arcadia Transit\n", - " yes\n", - " \n", - " \n", - " 4\n", - " Artesia Schedule\n", - " Artesia Transit\n", - " yes\n", - " \n", - " \n", - " 5\n", - " Arvin Schedule\n", - " Arvin Transit\n", - " yes\n", - " \n", - " \n", - " 6\n", - " Auburn Schedule\n", - " Auburn Transit\n", - " yes\n", - " \n", - " \n", - " 7\n", - " Baldwin Park Schedule\n", - " Baldwin Park Express\n", - " yes\n", - " \n", - " \n", - " 8\n", - " Bay Area 511 Regional VehiclePositions\n", - " AC Transit\n", - " yes\n", - " \n", - " \n", - " 9\n", - " Bay Area 511 Union City Transit VehiclePositions\n", - " Union City Transit\n", - " yes\n", - " \n", - " \n", - " 10\n", - " Bell Gardens Schedule\n", - " Bell Gardens Trolly\n", - " yes\n", - " \n", - " \n", - " 11\n", - " Bellflower Bus Schedule\n", - " Bellflower Bus\n", - " yes\n", - " \n", - " \n", - " 12\n", - " Calabasas Schedule\n", - " Calabasas Shuttle and Trolley\n", - " yes\n", - " \n", - " \n", " 13\n", - " Capitol Corridor Schedule ,Bay Area 511 ACE Schedule,Amtrak Schedule\n", - " Capitol Corridor\n", - " yes\n", - " \n", - " \n", - " 14\n", - " Catalina Flyer Schedule Historic\n", - " Catalina Flyer\n", - " yes\n", - " \n", - " \n", - " 15\n", - " Compton Schedule\n", - " Compton Renaissance Transit Service\n", - " yes\n", - " \n", - " \n", - " 16\n", - " Cudahy Schedule\n", - " Cudahy Area Rapid Transit\n", - " yes\n", - " \n", - " \n", - " 17\n", - " DowneyLINK Schedule\n", - " DowneyLINK\n", - " yes\n", - " \n", - " \n", - " 18\n", - " El Segundo Schedule\n", - " El Segundo Shuttles\n", - " yes\n", - " \n", - " \n", - " 19\n", - " Get Around Town Express Schedule\n", - " Get Around Town Express\n", - " yes\n", - " \n", - " \n", - " 20\n", - " Glendora Schedule\n", - " Glendora Shuttles\n", - " yes\n", - " \n", - " \n", - " 21\n", - " Glenn Schedule\n", - " Glenn Ride\n", - " yes\n", + " AC TRANSIT\n", " \n", " \n", " 22\n", - " Go West Schedule\n", - " Go West Shuttle\n", - " yes\n", - " \n", - " \n", - " 23\n", - " Historic Tuolumne Schedule\n", - " Tuolumne County Transit\n", - " yes\n", - " \n", - " \n", - " 24\n", - " Huntington Schedule\n", - " Huntington Park Express\n", - " yes\n", + " Bay Area Rapid Transit\n", " \n", " \n", " 25\n", - " Inglewood Schedule\n", - " I-Line Shuttle\n", - " yes\n", - " \n", - " \n", - " 26\n", - " La Campana Schedule\n", - " La Campana\n", - " yes\n", - " \n", - " \n", - " 27\n", - " La Puente Schedule\n", - " La Puente Link\n", - " yes\n", + " Caltrain\n", " \n", " \n", - " 28\n", - " Lawndale Beat Vehicle Positions\n", - " Lawndale Beat\n", - " yes\n", + " 37\n", + " County Connection\n", " \n", " \n", - " 29\n", - " Lynwood Schedule\n", - " Lynwood Breeze\n", - " yes\n", + " 40\n", + " Dumbarton Express Consortium\n", " \n", " \n", - " 30\n", - " MVGO VehiclePositions Historic,MVGO TripUpdates Historic\n", - " Mountain View Transportation Management Association\n", - " yes\n", + " 43\n", + " Emery Go-Round\n", " \n", " \n", - " 31\n", - " Maywood Schedule\n", - " Maywood Express Shuttle\n", - " yes\n", + " 46\n", + " FAST\n", " \n", " \n", - " 32\n", - " Merced VehiclePositions\n", - " Merced The Bus\n", - " yes\n", + " 52\n", + " Golden Gate Transit\n", " \n", " \n", - " 33\n", - " Morro Bay Schedule\n", - " Morro Bay Transit\n", - " yes\n", + " 55\n", + " Livermore Amador Valley Transit Authority\n", " \n", " \n", - " 34\n", - " None\n", - " GTrans\n", - " yes\n", + " 58\n", + " Marin Transit\n", " \n", " \n", - " 35\n", - " None\n", - " Havasu Landing Resort & Casino Ferry\n", - " yes\n", + " 64\n", + " MVgo Mountain View\n", " \n", " \n", - " 36\n", - " None\n", - " Kern Transit\n", - " yes\n", + " 67\n", + " Petaluma\n", " \n", " \n", - " 37\n", - " None\n", - " San Clemente Trolley\n", - " yes\n", + " 76\n", + " SamTrans\n", " \n", " \n", - " 38\n", - " Playa Vista Schedule\n", - " Playa Vista Shuttle\n", - " yes\n", + " 79\n", + " San Francisco Bay Ferry\n", " \n", " \n", - " 39\n", - " Plumas Schedule\n", - " Plumas Transit Systems\n", - " yes\n", + " 85\n", + " San Francisco Municipal Transportation Agency\n", " \n", " \n", - " 40\n", - " Rosemead Schedule\n", - " Rosemead Explorer\n", - " yes\n", + " 91\n", + " Santa Rosa CityBus\n", " \n", " \n", - " 41\n", - " Sage Stage Schedule\n", - " Sage Stage Intercity\n", - " yes\n", + " 94\n", + " SolTrans\n", " \n", " \n", - " 42\n", - " San Diego Vehicle Positions\n", - " San Diego Metropolitan Transit System\n", - " yes\n", + " 97\n", + " Sonoma County Transit\n", " \n", " \n", - " 43\n", - " San Juan Capistrano Schedule\n", - " San Juan Capistrano Free Weekend Trolley\n", - " yes\n", + " 100\n", + " Sonoma Marin Area Rail Transit\n", " \n", " \n", - " 44\n", - " Sierra Madre Schedule\n", - " Gateway Coach\n", - " yes\n", + " 106\n", + " Tri Delta Transit\n", " \n", " \n", - " 45\n", - " SunLine Vehicle Positions,SunLine Trip Updates\n", - " SunLine Transit\n", - " yes\n", + " 109\n", + " Union City Transit\n", " \n", " \n", - " 46\n", - " Tracy Schedule\n", - " TRACER\n", - " yes\n", + " 115\n", + " VINE Transit\n", " \n", " \n", - " 47\n", - " Union City GMV Schedule\n", - " Union City Transit\n", - " yes\n", + " 118\n", + " VTA\n", " \n", " \n", - " 48\n", - " eTrans Schedule,eTrans VehiclePositions,eTrans TripUpdates\n", - " eTrans\n", - " yes\n", + " 121\n", + " WestCat (Western Contra Costa)\n", " \n", " \n", "\n", "" ], "text/plain": [ - " gtfs_datasets \\\n", - "0 AC Transit Alerts,Bay Area 511 Regional Alerts \n", - "1 Alhambra Schedule \n", - "2 Anaheim Resort Schedule \n", - "3 Arcadia Schedule \n", - "4 Artesia Schedule \n", - "5 Arvin Schedule \n", - "6 Auburn Schedule \n", - "7 Baldwin Park Schedule \n", - "8 Bay Area 511 Regional VehiclePositions \n", - "9 Bay Area 511 Union City Transit VehiclePositions \n", - "10 Bell Gardens Schedule \n", - "11 Bellflower Bus Schedule \n", - "12 Calabasas Schedule \n", - "13 Capitol Corridor Schedule ,Bay Area 511 ACE Schedule,Amtrak Schedule \n", - "14 Catalina Flyer Schedule Historic \n", - "15 Compton Schedule \n", - "16 Cudahy Schedule \n", - "17 DowneyLINK Schedule \n", - "18 El Segundo Schedule \n", - "19 Get Around Town Express Schedule \n", - "20 Glendora Schedule \n", - "21 Glenn Schedule \n", - "22 Go West Schedule \n", - "23 Historic Tuolumne Schedule \n", - "24 Huntington Schedule \n", - "25 Inglewood Schedule \n", - "26 La Campana Schedule \n", - "27 La Puente Schedule \n", - "28 Lawndale Beat Vehicle Positions \n", - "29 Lynwood Schedule \n", - "30 MVGO VehiclePositions Historic,MVGO TripUpdates Historic \n", - "31 Maywood Schedule \n", - "32 Merced VehiclePositions \n", - "33 Morro Bay Schedule \n", - "34 None \n", - "35 None \n", - "36 None \n", - "37 None \n", - "38 Playa Vista Schedule \n", - "39 Plumas Schedule \n", - "40 Rosemead Schedule \n", - "41 Sage Stage Schedule \n", - "42 San Diego Vehicle Positions \n", - "43 San Juan Capistrano Schedule \n", - "44 Sierra Madre Schedule \n", - "45 SunLine Vehicle Positions,SunLine Trip Updates \n", - "46 Tracy Schedule \n", - "47 Union City GMV Schedule \n", - "48 eTrans Schedule,eTrans VehiclePositions,eTrans TripUpdates \n", - "\n", - " services airtable_ticket \n", - "0 AC Transit yes \n", - "1 Alhambra Community Transit yes \n", - "2 Anaheim Resort Transportation yes \n", - "3 Arcadia Transit yes \n", - "4 Artesia Transit yes \n", - "5 Arvin Transit yes \n", - "6 Auburn Transit yes \n", - "7 Baldwin Park Express yes \n", - "8 AC Transit yes \n", - "9 Union City Transit yes \n", - "10 Bell Gardens Trolly yes \n", - "11 Bellflower Bus yes \n", - "12 Calabasas Shuttle and Trolley yes \n", - "13 Capitol Corridor yes \n", - "14 Catalina Flyer yes \n", - "15 Compton Renaissance Transit Service yes \n", - "16 Cudahy Area Rapid Transit yes \n", - "17 DowneyLINK yes \n", - "18 El Segundo Shuttles yes \n", - "19 Get Around Town Express yes \n", - "20 Glendora Shuttles yes \n", - "21 Glenn Ride yes \n", - "22 Go West Shuttle yes \n", - "23 Tuolumne County Transit yes \n", - "24 Huntington Park Express yes \n", - "25 I-Line Shuttle yes \n", - "26 La Campana yes \n", - "27 La Puente Link yes \n", - "28 Lawndale Beat yes \n", - "29 Lynwood Breeze yes \n", - "30 Mountain View Transportation Management Association yes \n", - "31 Maywood Express Shuttle yes \n", - "32 Merced The Bus yes \n", - "33 Morro Bay Transit yes \n", - "34 GTrans yes \n", - "35 Havasu Landing Resort & Casino Ferry yes \n", - "36 Kern Transit yes \n", - "37 San Clemente Trolley yes \n", - "38 Playa Vista Shuttle yes \n", - "39 Plumas Transit Systems yes \n", - "40 Rosemead Explorer yes \n", - "41 Sage Stage Intercity yes \n", - "42 San Diego Metropolitan Transit System yes \n", - "43 San Juan Capistrano Free Weekend Trolley yes \n", - "44 Gateway Coach yes \n", - "45 SunLine Transit yes \n", - "46 TRACER yes \n", - "47 Union City Transit yes \n", - "48 eTrans yes " + " new\n", + "13 AC TRANSIT\n", + "22 Bay Area Rapid Transit\n", + "25 Caltrain\n", + "37 County Connection\n", + "40 Dumbarton Express Consortium\n", + "43 Emery Go-Round\n", + "46 FAST\n", + "52 Golden Gate Transit\n", + "55 Livermore Amador Valley Transit Authority\n", + "58 Marin Transit\n", + "64 MVgo Mountain View\n", + "67 Petaluma\n", + "76 SamTrans\n", + "79 San Francisco Bay Ferry\n", + "85 San Francisco Municipal Transportation Agency\n", + "91 Santa Rosa CityBus\n", + "94 SolTrans\n", + "97 Sonoma County Transit\n", + "100 Sonoma Marin Area Rail Transit\n", + "106 Tri Delta Transit\n", + "109 Union City Transit\n", + "115 VINE Transit\n", + "118 VTA\n", + "121 WestCat (Western Contra Costa)" ] }, - "execution_count": 101, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "open_airtable_tickets" + "api_511(\"API511.xlsx\")" ] }, { "cell_type": "code", - "execution_count": 96, - "id": "c6d592ce-fb2a-46bb-bf5b-8ce7f4a27dcc", + "execution_count": 63, + "id": "681da5bd-6fda-4e2d-afa6-1817f69c3140", "metadata": {}, "outputs": [], "source": [ - "incomplete2 = pd.merge(\n", + "def incomplete(tu_excel_file:str, vp_excel_file:str, airtable: pd.DataFrame):\n", + " tu_df = load_tu(tu_excel_file)\n", + " vp_df = load_vp(vp_excel_file)\n", + " \n", + " incomplete = pd.merge(tu_df, vp_df, on=\"name\", how=\"outer\")\n", + " incomplete = incomplete.sort_values([\"name\"]).reset_index(drop=True)\n", + " \n", + " incomplete = incomplete.fillna(\"OK\")\n", + " \n", + " #incomplete.name = incomplete.name.str.replace('Schedule','')\n", + " incomplete2 = (pd.merge(\n", " incomplete,\n", - " open_airtable_tickets,\n", + " airtable,\n", " left_on=\"name\",\n", " right_on=\"gtfs_datasets\",\n", - " how=\"left\",\n", - " indicator=True,\n", - ").sort_values(\"name\")" + " how=\"left\")\n", + " .sort_values(\"name\")\n", + " .fillna('NA')\n", + " )\n", + " display(incomplete2)\n", + " " ] }, { "cell_type": "code", - "execution_count": 97, - "id": "af1804f6-e204-4c4d-94ef-cf667417a971", + "execution_count": 64, + "id": "f235a5ed-85aa-452d-bc48-7499dab186ce", "metadata": { - "scrolled": true, "tags": [] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n", + "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", + " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" + ] + }, { "data": { "text/html": [ @@ -1777,10 +661,10 @@ " name\n", " %_of_trips_with_tu_messages\n", " %_of_trips_with_vp_messages\n", + " description\n", " gtfs_datasets\n", " services\n", " airtable_ticket\n", - " _merge\n", " \n", " \n", " \n", @@ -1789,240 +673,240 @@ " Anaheim Resort Schedule\n", " 21.11\n", " 17.73\n", + " Anaheim Transportation Network: GTFS-Schedule issue: thousands of duplicate trips on Toy Story Line\n", " Anaheim Resort Schedule\n", " Anaheim Resort Transportation\n", - " yes\n", - " both\n", + " Yes\n", " \n", " \n", " 1\n", " Bay Area 511 ACE Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 2\n", " Bay Area 511 Angel Island-Tiburon Ferry Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 3\n", " Bay Area 511 BART Schedule\n", " 39.31\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 4\n", " Bay Area 511 Capitol Corridor Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 5\n", " Bay Area 511 Commute.org Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 6\n", " Bay Area 511 Golden Gate Ferry Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 7\n", " Bay Area 511 MVGO Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 8\n", " Bay Area 511 Mission Bay Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 9\n", " Bay Area 511 Rio Vista Delta Breeze Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 10\n", " Bay Area 511 SFO AirTrain Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 11\n", " Bay Area 511 Santa Rosa CityBus Schedule\n", " OK\n", " 17.11\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 12\n", " Bay Area 511 South San Francisco Shuttle Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 13\n", " Bay Area 511 Treasure Island Ferry Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 14\n", " Bay Area 511 Union City Transit Schedule\n", " 16.10\n", " 9.38\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 15\n", " Bay Area 511 Vacaville City Coach Schedule\n", " 0.00\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 16\n", " Bay Area 511 Vine Transit Schedule\n", " OK\n", " 37.77\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 17\n", " Merced GMV Schedule\n", " OK\n", " 0.00\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 18\n", " SLORTA Schedule\n", " 15.74\n", " 15.60\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 19\n", " San Diego Schedule\n", " 0.00\n", " OK\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 20\n", " Santa Rosa CityBus Schedule\n", " OK\n", " 21.26\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 21\n", " SunLine Avail Schedule\n", " 0.96\n", " 0.71\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", " 22\n", " Union City GMV Schedule\n", " 18.39\n", " 10.68\n", + " Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages.\n", " Union City GMV Schedule\n", " Union City Transit\n", - " yes\n", - " both\n", + " Yes\n", " \n", " \n", " 23\n", " Vine GMV Schedule\n", " OK\n", " 37.77\n", - " NaN\n", - " NaN\n", - " NaN\n", - " left_only\n", + " NA\n", + " NA\n", + " NA\n", + " NA\n", " \n", " \n", "\n", @@ -2081,66 +965,67 @@ "22 18.39 10.68 \n", "23 OK 37.77 \n", "\n", - " gtfs_datasets services airtable_ticket \\\n", - "0 Anaheim Resort Schedule Anaheim Resort Transportation yes \n", - "1 NaN NaN NaN \n", - "2 NaN NaN NaN \n", - "3 NaN NaN NaN \n", - "4 NaN NaN NaN \n", - "5 NaN NaN NaN \n", - "6 NaN NaN NaN \n", - "7 NaN NaN NaN \n", - "8 NaN NaN NaN \n", - "9 NaN NaN NaN \n", - "10 NaN NaN NaN \n", - "11 NaN NaN NaN \n", - "12 NaN NaN NaN \n", - "13 NaN NaN NaN \n", - "14 NaN NaN NaN \n", - "15 NaN NaN NaN \n", - "16 NaN NaN NaN \n", - "17 NaN NaN NaN \n", - "18 NaN NaN NaN \n", - "19 NaN NaN NaN \n", - "20 NaN NaN NaN \n", - "21 NaN NaN NaN \n", - "22 Union City GMV Schedule Union City Transit yes \n", - "23 NaN NaN NaN \n", + " description \\\n", + "0 Anaheim Transportation Network: GTFS-Schedule issue: thousands of duplicate trips on Toy Story Line \n", + "1 NA \n", + "2 NA \n", + "3 NA \n", + "4 NA \n", + "5 NA \n", + "6 NA \n", + "7 NA \n", + "8 NA \n", + "9 NA \n", + "10 NA \n", + "11 NA \n", + "12 NA \n", + "13 NA \n", + "14 NA \n", + "15 NA \n", + "16 NA \n", + "17 NA \n", + "18 NA \n", + "19 NA \n", + "20 NA \n", + "21 NA \n", + "22 Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages. \n", + "23 NA \n", "\n", - " _merge \n", - "0 both \n", - "1 left_only \n", - "2 left_only \n", - "3 left_only \n", - "4 left_only \n", - "5 left_only \n", - "6 left_only \n", - "7 left_only \n", - "8 left_only \n", - "9 left_only \n", - "10 left_only \n", - "11 left_only \n", - "12 left_only \n", - "13 left_only \n", - "14 left_only \n", - "15 left_only \n", - "16 left_only \n", - "17 left_only \n", - "18 left_only \n", - "19 left_only \n", - "20 left_only \n", - "21 left_only \n", - "22 both \n", - "23 left_only " + " gtfs_datasets services airtable_ticket \n", + "0 Anaheim Resort Schedule Anaheim Resort Transportation Yes \n", + "1 NA NA NA \n", + "2 NA NA NA \n", + "3 NA NA NA \n", + "4 NA NA NA \n", + "5 NA NA NA \n", + "6 NA NA NA \n", + "7 NA NA NA \n", + "8 NA NA NA \n", + "9 NA NA NA \n", + "10 NA NA NA \n", + "11 NA NA NA \n", + "12 NA NA NA \n", + "13 NA NA NA \n", + "14 NA NA NA \n", + "15 NA NA NA \n", + "16 NA NA NA \n", + "17 NA NA NA \n", + "18 NA NA NA \n", + "19 NA NA NA \n", + "20 NA NA NA \n", + "21 NA NA NA \n", + "22 Union City GMV Schedule Union City Transit Yes \n", + "23 NA NA NA " ] }, - "execution_count": 97, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "incomplete2" + "incomplete(\"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\",\n", + " \"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\",\n", + " airtable)" ] } ], diff --git a/gtfs_quality_check/testing_calitp.ipynb b/gtfs_quality_check/testing_calitp.ipynb new file mode 100644 index 000000000..5835e9423 --- /dev/null +++ b/gtfs_quality_check/testing_calitp.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "5a856df9-daa1-417c-a2d6-98e33e096dae", + "metadata": {}, + "outputs": [], + "source": [ + "import geopandas as gpd\n", + "import pandas as pd\n", + "\n", + "# Hint: if this doesn't import: refer to docs for correctly import\n", + "# cd into _shared_utils folder, run the make setup_env command\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f755b4d7-aa6c-4b18-932e-70c903902668", + "metadata": {}, + "outputs": [], + "source": [ + "my_gdf = pd.read_csv('gs://calitp-analytics-data/data-analyses/csuyat_folder/stops_caltrain_merced.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e64a3d94-3f98-4c77-8a25-f173c6bc95ee", + "metadata": {}, + "outputs": [], + "source": [ + "my_gdf2 = my_gdf.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f3087913-f716-486c-8ab4-54c95b185caa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0feed_keystop_idstop_latstop_lonstop_name
0025c6505166c01099b2f6f2de173e20b922nd_street37.756972-122.39249222nd Street
1125c6505166c01099b2f6f2de173e20b9253774037.438491-122.156405Stanford Caltrain Station
2225c6505166c01099b2f6f2de173e20b9253774437.438425-122.156482Stanford Caltrain Station
3325c6505166c01099b2f6f2de173e20b97001137.776390-122.394992San Francisco Caltrain Station
4425c6505166c01099b2f6f2de173e20b97001237.776348-122.394935San Francisco Caltrain Station
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 feed_key stop_id stop_lat \\\n", + "0 0 25c6505166c01099b2f6f2de173e20b9 22nd_street 37.756972 \n", + "1 1 25c6505166c01099b2f6f2de173e20b9 2537740 37.438491 \n", + "2 2 25c6505166c01099b2f6f2de173e20b9 2537744 37.438425 \n", + "3 3 25c6505166c01099b2f6f2de173e20b9 70011 37.776390 \n", + "4 4 25c6505166c01099b2f6f2de173e20b9 70012 37.776348 \n", + "\n", + " stop_lon stop_name \n", + "0 -122.392492 22nd Street \n", + "1 -122.156405 Stanford Caltrain Station \n", + "2 -122.156482 Stanford Caltrain Station \n", + "3 -122.394992 San Francisco Caltrain Station \n", + "4 -122.394935 San Francisco Caltrain Station " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "my_gdf2" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "48ac310d-9376-4222-b559-b837a5d368b0", + "metadata": {}, + "outputs": [], + "source": [ + "from calitp_data_analysis import geography_utils" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "7f64ae8f-c204-40b0-a6b9-a57c3c350314", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "stops_ptg = geography_utils.create_point_geometry(\n", + " my_gdf2,\n", + " 'stop_lon',\n", + " 'stop_lat',\n", + " crs = 'EPSG:2229'\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e6c13e4f-1058-416b-b4f1-46ffe1150484", + "metadata": {}, + "outputs": [], + "source": [ + "# stops_ptg.explore()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 90f3f03bd09b336d28ac4a54a1a06ca9cd4d3252 Mon Sep 17 00:00:00 2001 From: amandaha8 Date: Fri, 29 Sep 2023 15:53:04 +0000 Subject: [PATCH 4/4] added new cols, simplified scripts --- gtfs_quality_check/gtfs_checking.ipynb | 1051 +++++++++++++++++------ gtfs_quality_check/testing_calitp.ipynb | 205 ----- 2 files changed, 783 insertions(+), 473 deletions(-) delete mode 100644 gtfs_quality_check/testing_calitp.ipynb diff --git a/gtfs_quality_check/gtfs_checking.ipynb b/gtfs_quality_check/gtfs_checking.ipynb index d8b27c61c..ec7d379f0 100644 --- a/gtfs_quality_check/gtfs_checking.ipynb +++ b/gtfs_quality_check/gtfs_checking.ipynb @@ -7,10 +7,7 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "\n", "import pandas as pd\n", - "import requests\n", "from calitp_data_analysis.sql import to_snakecase" ] }, @@ -42,21 +39,31 @@ "metadata": {}, "outputs": [], "source": [ - "def load_catastrophic_errors(excel_file:str):\n", + "def load_catastrophic_errors(excel_file: str):\n", " df = pd.read_excel(excel_file)\n", " df = to_snakecase(df)\n", - " \n", + "\n", " # Don't look at LACMTA stuff\n", " no_lacmta = df[~df[\"dim_gtfs_datasets_→_uri\"].str.contains(\"LACMTA\")]\n", - " \n", + "\n", " no_lacmta2 = no_lacmta[\n", - " [\n", - " \"dim_provider_gtfs_data_→_service_name\",\n", - " \"dim_county_geography_→_caltrans_district\",\n", - " \"date\",\n", - " ]].sort_values(\n", - " [\"dim_provider_gtfs_data_→_service_name\", \"date\"], ascending=[False, False])\n", - " \n", + " [\n", + " \"dim_provider_gtfs_data_→_service_name\",\n", + " \"dim_county_geography_→_caltrans_district\",\n", + " \"dim_gtfs_datasets_→_uri\",\n", + " \"date\",\n", + " ]\n", + " ].sort_values(\n", + " [\"dim_provider_gtfs_data_→_service_name\", \"date\"], ascending=[False, False]\n", + " )\n", + "\n", + " no_lacmta2 = no_lacmta2.rename(\n", + " columns={\n", + " \"dim_provider_gtfs_data_→_service_name\": \"service_name\",\n", + " \"dim_county_geography_→_caltrans_district\": \"district\",\n", + " \"dim_gtfs_datasets_→_uri\": \"uri\",\n", + " }\n", + " )\n", " return no_lacmta2" ] }, @@ -76,73 +83,565 @@ } ], "source": [ - "cat_df = load_catastrophic_errors(\"./feed_info_expired_feeds_2023-09-21T19_13_29.263825Z.xlsx\")" + "cat_df = load_catastrophic_errors(\n", + " \"./feed_info_expired_feeds_2023-09-29T15_22_09.307845Z.xlsx\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9fcc40ac-d121-4c1c-bf21-e1d317934f96", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
service_namedistricturidate
8TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-28
18TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-27
28TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-26
38TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-25
48TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-24
58TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-23
68TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-22
78TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-21
88TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-20
98TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-19
108TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-18
118TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-17
128TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-16
138TRACER10https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip2023-09-15
9Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-28
19Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-27
29Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-26
39Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-25
49Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-24
59Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-23
69Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-22
79Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-21
89Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-20
99Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-19
109Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-18
119Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-17
129Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-16
139Plumas Transit Systems2https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip2023-09-15
6Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-28
16Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-27
26Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-26
36Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-25
46Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-24
56Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-23
66Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-22
76Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-21
86Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-20
96Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-19
106Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-18
116Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-17
126Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-16
136Morro Bay Transit5https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip2023-09-15
\n", + "
" + ], + "text/plain": [ + " service_name district \\\n", + "8 TRACER 10 \n", + "18 TRACER 10 \n", + "28 TRACER 10 \n", + "38 TRACER 10 \n", + "48 TRACER 10 \n", + "58 TRACER 10 \n", + "68 TRACER 10 \n", + "78 TRACER 10 \n", + "88 TRACER 10 \n", + "98 TRACER 10 \n", + "108 TRACER 10 \n", + "118 TRACER 10 \n", + "128 TRACER 10 \n", + "138 TRACER 10 \n", + "9 Plumas Transit Systems 2 \n", + "19 Plumas Transit Systems 2 \n", + "29 Plumas Transit Systems 2 \n", + "39 Plumas Transit Systems 2 \n", + "49 Plumas Transit Systems 2 \n", + "59 Plumas Transit Systems 2 \n", + "69 Plumas Transit Systems 2 \n", + "79 Plumas Transit Systems 2 \n", + "89 Plumas Transit Systems 2 \n", + "99 Plumas Transit Systems 2 \n", + "109 Plumas Transit Systems 2 \n", + "119 Plumas Transit Systems 2 \n", + "129 Plumas Transit Systems 2 \n", + "139 Plumas Transit Systems 2 \n", + "6 Morro Bay Transit 5 \n", + "16 Morro Bay Transit 5 \n", + "26 Morro Bay Transit 5 \n", + "36 Morro Bay Transit 5 \n", + "46 Morro Bay Transit 5 \n", + "56 Morro Bay Transit 5 \n", + "66 Morro Bay Transit 5 \n", + "76 Morro Bay Transit 5 \n", + "86 Morro Bay Transit 5 \n", + "96 Morro Bay Transit 5 \n", + "106 Morro Bay Transit 5 \n", + "116 Morro Bay Transit 5 \n", + "126 Morro Bay Transit 5 \n", + "136 Morro Bay Transit 5 \n", + "\n", + " uri \\\n", + "8 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "18 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "28 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "38 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "48 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "58 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "68 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "78 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "88 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "98 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "108 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "118 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "128 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "138 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "9 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "19 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "29 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "39 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "49 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "59 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "69 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "79 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "89 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "99 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "109 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "119 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "129 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "139 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "6 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "16 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "26 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "36 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "46 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "56 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "66 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "76 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "86 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "96 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "106 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "116 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "126 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "136 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "\n", + " date \n", + "8 2023-09-28 \n", + "18 2023-09-27 \n", + "28 2023-09-26 \n", + "38 2023-09-25 \n", + "48 2023-09-24 \n", + "58 2023-09-23 \n", + "68 2023-09-22 \n", + "78 2023-09-21 \n", + "88 2023-09-20 \n", + "98 2023-09-19 \n", + "108 2023-09-18 \n", + "118 2023-09-17 \n", + "128 2023-09-16 \n", + "138 2023-09-15 \n", + "9 2023-09-28 \n", + "19 2023-09-27 \n", + "29 2023-09-26 \n", + "39 2023-09-25 \n", + "49 2023-09-24 \n", + "59 2023-09-23 \n", + "69 2023-09-22 \n", + "79 2023-09-21 \n", + "89 2023-09-20 \n", + "99 2023-09-19 \n", + "109 2023-09-18 \n", + "119 2023-09-17 \n", + "129 2023-09-16 \n", + "139 2023-09-15 \n", + "6 2023-09-28 \n", + "16 2023-09-27 \n", + "26 2023-09-26 \n", + "36 2023-09-25 \n", + "46 2023-09-24 \n", + "56 2023-09-23 \n", + "66 2023-09-22 \n", + "76 2023-09-21 \n", + "86 2023-09-20 \n", + "96 2023-09-19 \n", + "106 2023-09-18 \n", + "116 2023-09-17 \n", + "126 2023-09-16 \n", + "136 2023-09-15 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat_df" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 37, "id": "96ac0b97-70c3-436b-a4c1-7614db24614c", "metadata": {}, "outputs": [], "source": [ - "def load_airtable(csv_file:str):\n", + "def load_airtable(csv_file: str):\n", " df = to_snakecase(pd.read_csv(csv_file))\n", - " \n", + "\n", " df = df.fillna(\"None\")\n", - " \n", - " df = (df[[\n", - " \"description\",\n", - " \"gtfs_datasets\",\n", - " \"services\"]]\n", - " .sort_values([\"gtfs_datasets\", \"services\"])\n", - " .reset_index(drop=True))\n", - " \n", + "\n", + " df = (\n", + " df[\n", + " [\n", + " \"gtfs_datasets\",\n", + " \"services\",\n", + " \"issue_type\",\n", + " \"description\",\n", + " ]\n", + " ]\n", + " .sort_values([\"gtfs_datasets\", \"services\"])\n", + " .reset_index(drop=True)\n", + " )\n", + "\n", " df[\"airtable_ticket\"] = \"Yes\"\n", - " \n", + "\n", " return df" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 38, "id": "0e104d0d-02ec-4e6e-9165-660c3548681b", "metadata": {}, "outputs": [], "source": [ - "airtable = load_airtable(\"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\")" + "airtable = load_airtable(\n", + " \"./Transit Data Quality Issues-Open Issues by Distrcit (1).csv\"\n", + ")" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 39, "id": "bfac56a1-49c7-4b28-bc6e-e7009dc94ad7", "metadata": {}, "outputs": [], "source": [ - "def summarize_cat(catastrophic_data:pd.DataFrame, airtable_data:pd.DataFrame) -> pd.DataFrame:\n", - " \n", + "def summarize_cat(\n", + " catastrophic_data: pd.DataFrame, airtable_data: pd.DataFrame\n", + ") -> pd.DataFrame:\n", + "\n", " cat_summary = (\n", - " catastrophic_data.groupby([\"dim_provider_gtfs_data_→_service_name\"])\n", - " .agg({\"date\": \"count\"})\n", - " .reset_index()\n", - " .rename(columns={\"date\": \"# of days with expired feed\"}))\n", - " \n", + " catastrophic_data.groupby([\"service_name\", \"uri\"])\n", + " .agg({\"date\": \"count\"})\n", + " .reset_index()\n", + " .rename(columns={\"date\": \"# of days with expired feed\"})\n", + " )\n", + "\n", " display(cat_summary)\n", - " \n", + "\n", " m1 = pd.merge(\n", - " cat_summary,\n", - " airtable_data,\n", - " left_on=[\"dim_provider_gtfs_data_→_service_name\"],\n", - " right_on=[\"services\"],\n", - " how=\"left\",)\n", - " \n", + " cat_summary,\n", + " airtable_data,\n", + " left_on=[\"service_name\"],\n", + " right_on=[\"services\"],\n", + " how=\"left\",\n", + " )\n", + "\n", " display(m1)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 40, "id": "c3914805-a115-4029-97da-bc6bd2446706", "metadata": {}, "outputs": [ @@ -167,7 +666,8 @@ " \n", " \n", " \n", - " dim_provider_gtfs_data_→_service_name\n", + " service_name\n", + " uri\n", " # of days with expired feed\n", " \n", " \n", @@ -175,21 +675,19 @@ " \n", " 0\n", " Morro Bay Transit\n", + " https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip\n", " 14\n", " \n", " \n", " 1\n", " Plumas Transit Systems\n", + " https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip\n", " 14\n", " \n", " \n", " 2\n", - " Sage Stage Intercity\n", - " 2\n", - " \n", - " \n", - " 3\n", " TRACER\n", + " https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip\n", " 14\n", " \n", " \n", @@ -197,11 +695,20 @@ "" ], "text/plain": [ - " dim_provider_gtfs_data_→_service_name # of days with expired feed\n", - "0 Morro Bay Transit 14\n", - "1 Plumas Transit Systems 14\n", - "2 Sage Stage Intercity 2\n", - "3 TRACER 14" + " service_name \\\n", + "0 Morro Bay Transit \n", + "1 Plumas Transit Systems \n", + "2 TRACER \n", + "\n", + " uri \\\n", + "0 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "1 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "2 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "\n", + " # of days with expired feed \n", + "0 14 \n", + "1 14 \n", + "2 14 " ] }, "metadata": {}, @@ -228,11 +735,13 @@ " \n", " \n", " \n", - " dim_provider_gtfs_data_→_service_name\n", + " service_name\n", + " uri\n", " # of days with expired feed\n", - " description\n", " gtfs_datasets\n", " services\n", + " issue_type\n", + " description\n", " airtable_ticket\n", " \n", " \n", @@ -240,37 +749,34 @@ " \n", " 0\n", " Morro Bay Transit\n", + " https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip\n", " 14\n", - " City of Morro Bay: Follow-up about GTFS Schedule\n", " Morro Bay Schedule\n", " Morro Bay Transit\n", + " Expired Schedule Feed\n", + " City of Morro Bay: Follow-up about GTFS Schedule\n", " Yes\n", " \n", " \n", " 1\n", " Plumas Transit Systems\n", + " https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip\n", " 14\n", - " Based on the catastrophic errors dashboard, Plumas has not produced data since September 2. Per Evan, \"Plumas has a feed_info.txt file that says the feed was only valid until September 1 while the calendar says the feed is valid all the way until December 1. A strict interpretation of the GTFS spec says that any service beyond the feed end date is merely advisory, but some trip planners may still chose to display the service up until the end date.\" This is the second ticket opened for Plumas.\n", " Plumas Schedule\n", " Plumas Transit Systems\n", + " Expired Schedule Feed\n", + " Based on the catastrophic errors dashboard, Plumas has not produced data since September 2. Per Evan, \"Plumas has a feed_info.txt file that says the feed was only valid until September 1 while the calendar says the feed is valid all the way until December 1. A strict interpretation of the GTFS spec says that any service beyond the feed end date is merely advisory, but some trip planners may still chose to display the service up until the end date.\" This is the second ticket opened for Plumas.\n", " Yes\n", " \n", " \n", " 2\n", - " Sage Stage Intercity\n", - " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " \n", - " \n", - " 3\n", " TRACER\n", + " https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip\n", " 14\n", - " City of Tracy: GTFS Schedule Feed Expired\n", " Tracy Schedule\n", " TRACER\n", + " Expired Schedule Feed\n", + " City of Tracy: GTFS Schedule Feed Expired\n", " Yes\n", " \n", " \n", @@ -278,23 +784,35 @@ "" ], "text/plain": [ - " dim_provider_gtfs_data_→_service_name # of days with expired feed \\\n", - "0 Morro Bay Transit 14 \n", - "1 Plumas Transit Systems 14 \n", - "2 Sage Stage Intercity 2 \n", - "3 TRACER 14 \n", + " service_name \\\n", + "0 Morro Bay Transit \n", + "1 Plumas Transit Systems \n", + "2 TRACER \n", + "\n", + " uri \\\n", + "0 https://mjcaction.com/MJC_GTFS_Public/morrobay_google_transit.zip \n", + "1 https://data.trilliumtransit.com/gtfs/plumas-ca-us/plumas-ca-us.zip \n", + "2 https://data.trilliumtransit.com/gtfs/tracy-ca-us/tracy-ca-us.zip \n", + "\n", + " # of days with expired feed gtfs_datasets services \\\n", + "0 14 Morro Bay Schedule Morro Bay Transit \n", + "1 14 Plumas Schedule Plumas Transit Systems \n", + "2 14 Tracy Schedule TRACER \n", + "\n", + " issue_type \\\n", + "0 Expired Schedule Feed \n", + "1 Expired Schedule Feed \n", + "2 Expired Schedule Feed \n", "\n", " description \\\n", "0 City of Morro Bay: Follow-up about GTFS Schedule \n", "1 Based on the catastrophic errors dashboard, Plumas has not produced data since September 2. Per Evan, \"Plumas has a feed_info.txt file that says the feed was only valid until September 1 while the calendar says the feed is valid all the way until December 1. A strict interpretation of the GTFS spec says that any service beyond the feed end date is merely advisory, but some trip planners may still chose to display the service up until the end date.\" This is the second ticket opened for Plumas. \n", - "2 NaN \n", - "3 City of Tracy: GTFS Schedule Feed Expired \n", + "2 City of Tracy: GTFS Schedule Feed Expired \n", "\n", - " gtfs_datasets services airtable_ticket \n", - "0 Morro Bay Schedule Morro Bay Transit Yes \n", - "1 Plumas Schedule Plumas Transit Systems Yes \n", - "2 NaN NaN NaN \n", - "3 Tracy Schedule TRACER Yes " + " airtable_ticket \n", + "0 Yes \n", + "1 Yes \n", + "2 Yes " ] }, "metadata": {}, @@ -302,7 +820,7 @@ } ], "source": [ - "summarize_cat(cat_df,airtable)" + "summarize_cat(cat_df, airtable)" ] }, { @@ -315,107 +833,52 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 41, "id": "b3395d37-99de-492b-94bf-eadef5cf527c", "metadata": {}, "outputs": [], "source": [ - "def load_tu(tu_excel_file:str):\n", - " df = to_snakecase(pd.read_excel(tu_excel_file))\n", - " df = (\n", - " df[df[\"%_of_trips_with_tu_messages\"] < 41].sort_values(\n", - " [\"%_of_trips_with_tu_messages\"]\n", - " )).reset_index(drop=True)\n", - " return df" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "2cfd0fed-560b-4f85-ad80-0e69ead44061", - "metadata": {}, - "outputs": [], - "source": [ - "def load_vp(vp_excel_file:str):\n", - " df = to_snakecase(pd.read_excel(vp_excel_file))\n", - " \n", - " df = (\n", - " df[df[\"%_of_trips_with_vp_messages\"] < 41].sort_values(\n", - " [\"%_of_trips_with_vp_messages\"]\n", - " )).reset_index(drop=True)\n", + "def load_tu_or_vp(excel_file: str, column_to_filter: str) -> pd.DataFrame:\n", + " df = to_snakecase(pd.read_excel(excel_file))\n", + " df = (df[df[column_to_filter] < 41].sort_values([column_to_filter])).reset_index(\n", + " drop=True\n", + " )\n", " return df" ] }, { "cell_type": "code", - "execution_count": 24, - "id": "18276af1-e87e-4218-b486-47a1bdd1da6d", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", - " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" - ] - } - ], - "source": [ - "tu_df = load_incomplete_tu( \"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\")" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "60647b1c-bbdb-4056-ab45-9cc9e6b512e3", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.9/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n", - " warn(\"Workbook contains no default style, apply openpyxl's default\")\n" - ] - } - ], - "source": [ - "vp_df = load_incomplete_tu(\"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\")" - ] - }, - { - "cell_type": "code", - "execution_count": 44, + "execution_count": 42, "id": "0d2c4751-a485-4c6d-9154-d58b9348699c", "metadata": {}, "outputs": [], "source": [ - "def api_511(excel_file:str)-> pd.DataFrame:\n", + "def api_511(excel_file: str) -> pd.DataFrame:\n", " # Read in 511\n", " df = to_snakecase(pd.read_excel(excel_file))\n", " df = df.rename(columns={df.columns[0]: \"new\"})\n", - " \n", + "\n", " # Only keep rows that have the string name or monitored\n", " df = df[df[\"new\"].str.contains((\"Name|Monitored\"))].reset_index(drop=True)\n", - " \n", + "\n", " # Get rid of random characters\n", " df.new = (\n", - " df.new.str.replace(\">\", \"\")\n", - " .str.replace(\"ShortName\", \"\")\n", - " .str.replace(\"Name\", \"\")\n", - " .str.replace(\"Monitored\", \"\")\n", - " .str.replace(\"<\", \"\")\n", - " .str.replace(\"/\", \"\"))\n", - " \n", + " df.new.str.replace(\">\", \"\")\n", + " .str.replace(\"ShortName\", \"\")\n", + " .str.replace(\"Name\", \"\")\n", + " .str.replace(\"Monitored\", \"\")\n", + " .str.replace(\"<\", \"\")\n", + " .str.replace(\"/\", \"\")\n", + " )\n", + "\n", " # display(df)\n", - " print('These are Bay Area feeds that keep track of RT')\n", + " print(\"These are Bay Area feeds that keep track of RT\")\n", " display(df[(df.new == \"true\").shift(1).fillna(False)])" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 43, "id": "a073faad-7223-4218-b767-6a32153d8ca4", "metadata": { "tags": [] @@ -425,7 +888,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "This is only Bay Area feeds that keep track of RT\n" + "These are Bay Area feeds that keep track of RT\n" ] }, { @@ -591,37 +1054,44 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 55, "id": "681da5bd-6fda-4e2d-afa6-1817f69c3140", "metadata": {}, "outputs": [], "source": [ - "def incomplete(tu_excel_file:str, vp_excel_file:str, airtable: pd.DataFrame):\n", - " tu_df = load_tu(tu_excel_file)\n", - " vp_df = load_vp(vp_excel_file)\n", - " \n", + "def incomplete(tu_excel_file: str, vp_excel_file: str, airtable: pd.DataFrame):\n", + " tu_df = load_tu_or_vp(tu_excel_file, \"%_of_trips_with_tu_messages\")\n", + " vp_df = load_tu_or_vp(vp_excel_file, \"%_of_trips_with_vp_messages\")\n", + "\n", " incomplete = pd.merge(tu_df, vp_df, on=\"name\", how=\"outer\")\n", " incomplete = incomplete.sort_values([\"name\"]).reset_index(drop=True)\n", - " \n", + "\n", " incomplete = incomplete.fillna(\"OK\")\n", - " \n", - " #incomplete.name = incomplete.name.str.replace('Schedule','')\n", - " incomplete2 = (pd.merge(\n", - " incomplete,\n", - " airtable,\n", - " left_on=\"name\",\n", - " right_on=\"gtfs_datasets\",\n", - " how=\"left\")\n", - " .sort_values(\"name\")\n", - " .fillna('NA')\n", + "\n", + " # incomplete.name = incomplete.name.str.replace('Schedule','')\n", + " incomplete2 = (\n", + " pd.merge(\n", + " incomplete, airtable, left_on=\"name\", right_on=\"gtfs_datasets\", how=\"left\"\n", + " )\n", + " .sort_values(\"name\")\n", + " .fillna(\"NA\")\n", " )\n", - " display(incomplete2)\n", - " " + " display(incomplete2)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "c2f06ddb-490e-4805-bcf1-97d78b6ad340", + "metadata": {}, + "outputs": [], + "source": [ + "# airtable.loc[airtable.issue_type.str.contains('Completeness')]" ] }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 57, "id": "f235a5ed-85aa-452d-bc48-7499dab186ce", "metadata": { "tags": [] @@ -661,9 +1131,10 @@ " name\n", " %_of_trips_with_tu_messages\n", " %_of_trips_with_vp_messages\n", - " description\n", " gtfs_datasets\n", " services\n", + " issue_type\n", + " description\n", " airtable_ticket\n", " \n", " \n", @@ -671,11 +1142,12 @@ " \n", " 0\n", " Anaheim Resort Schedule\n", - " 21.11\n", - " 17.73\n", - " Anaheim Transportation Network: GTFS-Schedule issue: thousands of duplicate trips on Toy Story Line\n", + " 20.80\n", + " 17.65\n", " Anaheim Resort Schedule\n", " Anaheim Resort Transportation\n", + " Service Accuracy\n", + " Anaheim Transportation Network: GTFS-Schedule issue: thousands of duplicate trips on Toy Story Line\n", " Yes\n", " \n", " \n", @@ -687,6 +1159,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 2\n", @@ -697,16 +1170,18 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 3\n", " Bay Area 511 BART Schedule\n", - " 39.31\n", + " OK\n", " 0.00\n", " NA\n", " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 4\n", @@ -717,6 +1192,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 5\n", @@ -727,6 +1203,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 6\n", @@ -737,6 +1214,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 7\n", @@ -747,6 +1225,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 8\n", @@ -757,6 +1236,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 9\n", @@ -767,6 +1247,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 10\n", @@ -777,12 +1258,14 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 11\n", " Bay Area 511 Santa Rosa CityBus Schedule\n", " OK\n", - " 17.11\n", + " 4.90\n", + " NA\n", " NA\n", " NA\n", " NA\n", @@ -797,6 +1280,7 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 13\n", @@ -807,12 +1291,14 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 14\n", " Bay Area 511 Union City Transit Schedule\n", - " 16.10\n", - " 9.38\n", + " 16.93\n", + " 11.24\n", + " NA\n", " NA\n", " NA\n", " NA\n", @@ -827,12 +1313,14 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", " 16\n", - " Bay Area 511 Vine Transit Schedule\n", + " Merced GMV Schedule\n", " OK\n", - " 37.77\n", + " 0.00\n", + " NA\n", " NA\n", " NA\n", " NA\n", @@ -840,26 +1328,17 @@ " \n", " \n", " 17\n", - " Merced GMV Schedule\n", + " PresidiGo Schedule\n", " OK\n", - " 0.00\n", + " 39.78\n", " NA\n", " NA\n", " NA\n", " NA\n", - " \n", - " \n", - " 18\n", - " SLORTA Schedule\n", - " 15.74\n", - " 15.60\n", - " NA\n", - " NA\n", - " NA\n", " NA\n", " \n", " \n", - " 19\n", + " 18\n", " San Diego Schedule\n", " 0.00\n", " OK\n", @@ -867,47 +1346,41 @@ " NA\n", " NA\n", " NA\n", + " NA\n", " \n", " \n", - " 20\n", + " 19\n", " Santa Rosa CityBus Schedule\n", " OK\n", - " 21.26\n", + " 4.90\n", + " NA\n", " NA\n", " NA\n", " NA\n", " NA\n", " \n", " \n", - " 21\n", + " 20\n", " SunLine Avail Schedule\n", " 0.96\n", - " 0.71\n", + " 0.75\n", + " NA\n", " NA\n", " NA\n", " NA\n", " NA\n", " \n", " \n", - " 22\n", + " 21\n", " Union City GMV Schedule\n", - " 18.39\n", - " 10.68\n", - " Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages.\n", + " 20.14\n", + " 13.35\n", " Union City GMV Schedule\n", " Union City Transit\n", + " GTFS Realtime Completeness Problem\n", + " Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages.\n", " Yes\n", " \n", - " \n", - " 23\n", - " Vine GMV Schedule\n", - " OK\n", - " 37.77\n", - " NA\n", - " NA\n", - " NA\n", - " NA\n", - " \n", " \n", "\n", "" @@ -930,20 +1403,18 @@ "13 Bay Area 511 Treasure Island Ferry Schedule \n", "14 Bay Area 511 Union City Transit Schedule \n", "15 Bay Area 511 Vacaville City Coach Schedule \n", - "16 Bay Area 511 Vine Transit Schedule \n", - "17 Merced GMV Schedule \n", - "18 SLORTA Schedule \n", - "19 San Diego Schedule \n", - "20 Santa Rosa CityBus Schedule \n", - "21 SunLine Avail Schedule \n", - "22 Union City GMV Schedule \n", - "23 Vine GMV Schedule \n", + "16 Merced GMV Schedule \n", + "17 PresidiGo Schedule \n", + "18 San Diego Schedule \n", + "19 Santa Rosa CityBus Schedule \n", + "20 SunLine Avail Schedule \n", + "21 Union City GMV Schedule \n", "\n", " %_of_trips_with_tu_messages %_of_trips_with_vp_messages \\\n", - "0 21.11 17.73 \n", + "0 20.80 17.65 \n", "1 0.00 0.00 \n", "2 0.00 0.00 \n", - "3 39.31 0.00 \n", + "3 OK 0.00 \n", "4 0.00 0.00 \n", "5 0.00 0.00 \n", "6 0.00 0.00 \n", @@ -951,19 +1422,65 @@ "8 0.00 0.00 \n", "9 0.00 0.00 \n", "10 0.00 0.00 \n", - "11 OK 17.11 \n", + "11 OK 4.90 \n", "12 0.00 0.00 \n", "13 0.00 0.00 \n", - "14 16.10 9.38 \n", + "14 16.93 11.24 \n", "15 0.00 0.00 \n", - "16 OK 37.77 \n", - "17 OK 0.00 \n", - "18 15.74 15.60 \n", - "19 0.00 OK \n", - "20 OK 21.26 \n", - "21 0.96 0.71 \n", - "22 18.39 10.68 \n", - "23 OK 37.77 \n", + "16 OK 0.00 \n", + "17 OK 39.78 \n", + "18 0.00 OK \n", + "19 OK 4.90 \n", + "20 0.96 0.75 \n", + "21 20.14 13.35 \n", + "\n", + " gtfs_datasets services \\\n", + "0 Anaheim Resort Schedule Anaheim Resort Transportation \n", + "1 NA NA \n", + "2 NA NA \n", + "3 NA NA \n", + "4 NA NA \n", + "5 NA NA \n", + "6 NA NA \n", + "7 NA NA \n", + "8 NA NA \n", + "9 NA NA \n", + "10 NA NA \n", + "11 NA NA \n", + "12 NA NA \n", + "13 NA NA \n", + "14 NA NA \n", + "15 NA NA \n", + "16 NA NA \n", + "17 NA NA \n", + "18 NA NA \n", + "19 NA NA \n", + "20 NA NA \n", + "21 Union City GMV Schedule Union City Transit \n", + "\n", + " issue_type \\\n", + "0 Service Accuracy \n", + "1 NA \n", + "2 NA \n", + "3 NA \n", + "4 NA \n", + "5 NA \n", + "6 NA \n", + "7 NA \n", + "8 NA \n", + "9 NA \n", + "10 NA \n", + "11 NA \n", + "12 NA \n", + "13 NA \n", + "14 NA \n", + "15 NA \n", + "16 NA \n", + "17 NA \n", + "18 NA \n", + "19 NA \n", + "20 NA \n", + "21 GTFS Realtime Completeness Problem \n", "\n", " description \\\n", "0 Anaheim Transportation Network: GTFS-Schedule issue: thousands of duplicate trips on Toy Story Line \n", @@ -987,35 +1504,31 @@ "18 NA \n", "19 NA \n", "20 NA \n", - "21 NA \n", - "22 Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages. \n", - "23 NA \n", + "21 Per the Incompleteness dashboard, only 36.5% of trips have TU messages and 28.2% of trips have VP messages. \n", "\n", - " gtfs_datasets services airtable_ticket \n", - "0 Anaheim Resort Schedule Anaheim Resort Transportation Yes \n", - "1 NA NA NA \n", - "2 NA NA NA \n", - "3 NA NA NA \n", - "4 NA NA NA \n", - "5 NA NA NA \n", - "6 NA NA NA \n", - "7 NA NA NA \n", - "8 NA NA NA \n", - "9 NA NA NA \n", - "10 NA NA NA \n", - "11 NA NA NA \n", - "12 NA NA NA \n", - "13 NA NA NA \n", - "14 NA NA NA \n", - "15 NA NA NA \n", - "16 NA NA NA \n", - "17 NA NA NA \n", - "18 NA NA NA \n", - "19 NA NA NA \n", - "20 NA NA NA \n", - "21 NA NA NA \n", - "22 Union City GMV Schedule Union City Transit Yes \n", - "23 NA NA NA " + " airtable_ticket \n", + "0 Yes \n", + "1 NA \n", + "2 NA \n", + "3 NA \n", + "4 NA \n", + "5 NA \n", + "6 NA \n", + "7 NA \n", + "8 NA \n", + "9 NA \n", + "10 NA \n", + "11 NA \n", + "12 NA \n", + "13 NA \n", + "14 NA \n", + "15 NA \n", + "16 NA \n", + "17 NA \n", + "18 NA \n", + "19 NA \n", + "20 NA \n", + "21 Yes " ] }, "metadata": {}, @@ -1023,9 +1536,11 @@ } ], "source": [ - "incomplete(\"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-25T18_11_34.892466Z.xlsx\",\n", - " \"gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-25T18_11_37.967895Z.xlsx\",\n", - " airtable)" + "incomplete(\n", + " \"./gtfs_rt_trip_updates_completeness__last_14_days__2023-09-29T15_22_26.671152Z.xlsx\",\n", + " \"./gtfs_rt_vehicle_positions_completeness__last_14_days__2023-09-29T15_22_29.375218Z.xlsx\",\n", + " airtable,\n", + ")" ] } ], diff --git a/gtfs_quality_check/testing_calitp.ipynb b/gtfs_quality_check/testing_calitp.ipynb deleted file mode 100644 index 5835e9423..000000000 --- a/gtfs_quality_check/testing_calitp.ipynb +++ /dev/null @@ -1,205 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "id": "5a856df9-daa1-417c-a2d6-98e33e096dae", - "metadata": {}, - "outputs": [], - "source": [ - "import geopandas as gpd\n", - "import pandas as pd\n", - "\n", - "# Hint: if this doesn't import: refer to docs for correctly import\n", - "# cd into _shared_utils folder, run the make setup_env command\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "f755b4d7-aa6c-4b18-932e-70c903902668", - "metadata": {}, - "outputs": [], - "source": [ - "my_gdf = pd.read_csv('gs://calitp-analytics-data/data-analyses/csuyat_folder/stops_caltrain_merced.csv')" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "e64a3d94-3f98-4c77-8a25-f173c6bc95ee", - "metadata": {}, - "outputs": [], - "source": [ - "my_gdf2 = my_gdf.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "f3087913-f716-486c-8ab4-54c95b185caa", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Unnamed: 0feed_keystop_idstop_latstop_lonstop_name
0025c6505166c01099b2f6f2de173e20b922nd_street37.756972-122.39249222nd Street
1125c6505166c01099b2f6f2de173e20b9253774037.438491-122.156405Stanford Caltrain Station
2225c6505166c01099b2f6f2de173e20b9253774437.438425-122.156482Stanford Caltrain Station
3325c6505166c01099b2f6f2de173e20b97001137.776390-122.394992San Francisco Caltrain Station
4425c6505166c01099b2f6f2de173e20b97001237.776348-122.394935San Francisco Caltrain Station
\n", - "
" - ], - "text/plain": [ - " Unnamed: 0 feed_key stop_id stop_lat \\\n", - "0 0 25c6505166c01099b2f6f2de173e20b9 22nd_street 37.756972 \n", - "1 1 25c6505166c01099b2f6f2de173e20b9 2537740 37.438491 \n", - "2 2 25c6505166c01099b2f6f2de173e20b9 2537744 37.438425 \n", - "3 3 25c6505166c01099b2f6f2de173e20b9 70011 37.776390 \n", - "4 4 25c6505166c01099b2f6f2de173e20b9 70012 37.776348 \n", - "\n", - " stop_lon stop_name \n", - "0 -122.392492 22nd Street \n", - "1 -122.156405 Stanford Caltrain Station \n", - "2 -122.156482 Stanford Caltrain Station \n", - "3 -122.394992 San Francisco Caltrain Station \n", - "4 -122.394935 San Francisco Caltrain Station " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "my_gdf2" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "48ac310d-9376-4222-b559-b837a5d368b0", - "metadata": {}, - "outputs": [], - "source": [ - "from calitp_data_analysis import geography_utils" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "7f64ae8f-c204-40b0-a6b9-a57c3c350314", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "stops_ptg = geography_utils.create_point_geometry(\n", - " my_gdf2,\n", - " 'stop_lon',\n", - " 'stop_lat',\n", - " crs = 'EPSG:2229'\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "e6c13e4f-1058-416b-b4f1-46ffe1150484", - "metadata": {}, - "outputs": [], - "source": [ - "# stops_ptg.explore()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}