Use count of lowlevel table to get number of items, formatting

submission_offsets.py → add_submission_offsets.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 from flask.cli import FlaskGroup
 import click
 from collections import defaultdict
@@ -9,19 +11,22 @@
 
 cli = FlaskGroup(add_default_commands=False, create_app=webserver.create_app_flaskgroup)
 
+
 @cli.command(name='add-offsets')
 @click.option("--limit", "-l", default=10000)
 def add_offsets(limit):
     """Update lowlevel submission offsets with a specified limit."""
     incremental_add_offset(limit)
 
+
 def incremental_add_offset(limit):
     with db.engine.connect() as connection:
 
         # Find number of items in table
         size_query = text("""
-            SELECT MAX(id) AS size
+            SELECT count(*) AS size
               FROM lowlevel
+             WHERE submission_offset IS NULL
         """)
         size_result = connection.execute(size_query)
         table_size = size_result.fetchone()["size"]
@@ -53,13 +58,13 @@ def incremental_add_offset(limit):
         print("Starting batch insertions...")
         print("============================")
         while True:
-            batch_result = connection.execute(batch_query, { "limit": limit })
+            batch_result = connection.execute(batch_query, {"limit": limit})
             if not batch_result.rowcount:
-                print("\nSubmission offset exists for all items. Exiting...")
+                print("Submission offset exists for all items. Exiting...")
                 break
 
             batch_count += 1
-            print("\nUpdating batch {}:".format(batch_count))
+            print("Updating batch {}:".format(batch_count))
             with connection.begin() as transaction:
                 for id, gid in batch_result.fetchall():
                     if gid in max_offsets:
@@ -75,14 +80,10 @@ def incremental_add_offset(limit):
                            SET submission_offset = :offset
                          WHERE id = :id
                     """)
-                    connection.execute(query, { "id": id, "offset": offset })
+                    connection.execute(query, {"id": id, "offset": offset})
                     item_count += 1
-                    print("\r   Inserted {}/{} items...".format(item_count, table_size)),
+            print(" Batch done, inserted {}/{} items...".format(item_count, table_size)),
         print("")
 
-
         print("============================")
         print("Batch insertions finished.")
-
-
-

manage.py

@@ -19,7 +19,7 @@
 import db.user
 import webserver
 
-import submission_offsets
+import add_submission_offsets
 
 ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'admin', 'sql')
 
@@ -243,7 +243,7 @@ def toggle_site_status():
 
 # Please keep additional sets of commands down there
 cli.add_command(db.dump_manage.cli, name="dump")
-cli.add_command(submission_offsets.cli, name="update-offsets")
+cli.add_command(add_submission_offsets.cli, name="update-offsets")
 
 if __name__ == '__main__':
     cli()