add compacting (#25)

* add compacting * solve flake8 issues * fix pagination size and solve state bug * clean up compaction * update max line length * increase 120
tsdataclinic · Oct 31, 2024 · c60cc1f · c60cc1f
1 parent 263647b
commit c60cc1f
Show file tree

Hide file tree

Showing 20 changed files with 440 additions and 506 deletions.
diff --git a/.flake8 b/.flake8
diff --git a/examples/compact_demo.py b/examples/compact_demo.py
diff --git a/examples/read_demo.py b/examples/read_demo.py
@@ -4,13 +4,13 @@
 
 from src.normalize.parquet_utils import read_data
 
-begin = dt.datetime(2024, 8, 29)
-end = dt.datetime(2024, 8, 30)
+begin = dt.datetime(2024, 10, 10)
+end = dt.datetime(2024, 10, 17)
 
-uri = "s3://dataclinic-gtfs-rt/gtfs_norm/test/mta-subway-ace/trip-updates"
+uri = "s3://dataclinic-gtfs-rt/norm/mdb-1630/trip-updates"
 # uri = "s3://dataclinic-gtfs-rt/gtfs_norm/test/mta-subway-ace/vehicles"
 # uri = "s3://dataclinic-gtfs-rt/gtfs_norm/test/mta-subway-alerts/"
-s3_fs = s3fs.S3FileSystem(key="foo", secret="bar")
+s3_fs = s3fs.S3FileSystem()
 table = read_data(s3_fs, uri, begin, end)
 df = table.to_pandas()
 print(df)

diff --git a/examples/test.py b/examples/test.py
diff --git a/examples/write_bus.py b/examples/write_bus.py
diff --git a/examples/write_demo.py b/examples/write_demo.py
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120