Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
sqlalchemy upgrade version adjustments
Browse files Browse the repository at this point in the history
atalyaalon committed Jul 25, 2023
1 parent 5a25ad6 commit 90f2675
Showing 8 changed files with 45 additions and 39 deletions.
4 changes: 2 additions & 2 deletions alembic/versions/312c9eb92e40_add_cbs_locations_table.py
Original file line number Diff line number Diff line change
@@ -34,7 +34,7 @@ def upgrade():
sa.PrimaryKeyConstraint('id')
)
conn = op.get_bind()
conn.execute("""INSERT INTO cbs_locations
conn.execute(sa.text("""INSERT INTO cbs_locations
(SELECT ROW_NUMBER() OVER (ORDER BY road1) as id, LOCATIONS.*
FROM
(SELECT DISTINCT road1,
@@ -52,7 +52,7 @@ def upgrade():
WHERE (provider_code=1
OR provider_code=3)
AND (longitude is not null
AND latitude is not null)) LOCATIONS)""")
AND latitude is not null)) LOCATIONS)"""))
# ### end Alembic commands ###


9 changes: 5 additions & 4 deletions alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py
Original file line number Diff line number Diff line change
@@ -12,20 +12,21 @@
depends_on = None

from alembic import op
import sqlalchemy as sa


def upgrade():
### commands auto generated by Alembic - please adjust! ###
conn = op.get_bind()
conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')
conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')
conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));'))
conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));'))

### end Alembic commands ###


def downgrade():
### commands auto generated by Alembic - please adjust! ###
conn = op.get_bind()
conn.execute('DROP INDEX geom_gix;')
conn.execute('DROP INDEX discussions_gix;')
conn.execute(sa.text('DROP INDEX geom_gix;'))
conn.execute(sa.text('DROP INDEX discussions_gix;'))
### end Alembic commands ###
Original file line number Diff line number Diff line change
@@ -12,31 +12,32 @@
depends_on = None

from alembic import op
import sqlalchemy as sa


def upgrade():
### commands auto generated by Alembic - please adjust! ###
conn = op.get_bind()
conn.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
conn.execute('CREATE EXTENSION IF NOT EXISTS postgis_topology;')
conn.execute("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);")
conn.execute('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')
conn.execute('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);')
conn.execute("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);")
conn.execute('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')
conn.execute('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);')
conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis;'))
conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis_topology;'))
conn.execute(sa.text("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);"))
conn.execute(sa.text('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);'))
conn.execute(sa.text('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);'))
conn.execute(sa.text("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);"))
conn.execute(sa.text('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);'))
conn.execute(sa.text('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);'))

### end Alembic commands ###


def downgrade():
### commands auto generated by Alembic - please adjust! ###
conn = op.get_bind()
conn.execute('DROP INDEX idx_markers_geom;')
conn.execute(sa.text('DROP INDEX idx_markers_geom;'))
op.drop_column('markers', 'geom')
conn.execute('DROP INDEX idx_discussions_geom;')
conn.execute(sa.text('DROP INDEX idx_discussions_geom;'))
op.drop_column('discussions', 'geom')
conn.execute('DROP EXTENSION postgis_topology;')
conn.execute('DROP EXTENSION postgis;')
conn.execute('DROP SCHEMA IF EXISTS topology CASCADE;')
conn.execute(sa.text('DROP EXTENSION postgis_topology;'))
conn.execute(sa.text('DROP EXTENSION postgis;'))
conn.execute(sa.text('DROP SCHEMA IF EXISTS topology CASCADE;'))
### end Alembic commands ###
Original file line number Diff line number Diff line change
@@ -34,6 +34,6 @@ def downgrade():
op.create_index('provider_and_id_idx_involved', 'involved', ['provider_and_id'], unique=False)
op.create_index('provider_and_id_idx_vehicles', 'vehicles', ['provider_and_id'], unique=False)
conn = op.get_bind()
conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')
conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')
conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));'))
conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));'))
# ### end Alembic commands ###
6 changes: 3 additions & 3 deletions alembic/versions/7f629b4c8891_add_news_flash_fields.py
Original file line number Diff line number Diff line change
@@ -19,9 +19,9 @@
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
conn = op.get_bind()
conn.execute("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');")
conn.execute("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;")
conn.execute("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;")
conn.execute(sa.text("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');"))
conn.execute(sa.text("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;"))
conn.execute(sa.text("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;"))
op.add_column('news_flash', sa.Column('district_hebrew', sa.Text(), nullable=True))
op.add_column('news_flash', sa.Column('non_urban_intersection_hebrew', sa.Text(), nullable=True))
op.add_column('news_flash', sa.Column('region_hebrew', sa.Text(), nullable=True))
11 changes: 7 additions & 4 deletions anyway/parsers/infographics_data_cache_updater.py
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@

from datetime import datetime
from sqlalchemy import not_
import sqlalchemy as sa
from anyway.models import (
Base,
InfographicsDataCache,
@@ -214,13 +215,15 @@ def copy_temp_into_cache(table: Dict[str, Base]):
db.session.commit()
start = datetime.now()
with db.get_engine().begin() as conn:
conn.execute("lock table infographics_data_cache in exclusive mode")
conn.execute(sa.text("lock table infographics_data_cache in exclusive mode"))
logging.debug(f"in transaction, after lock")
conn.execute(f"delete from {table[CACHE].__tablename__}")
conn.execute(sa.text(f"delete from {table[CACHE].__tablename__}"))
logging.debug(f"in transaction, after delete")
conn.execute(
f"insert into {table[CACHE].__tablename__} "
f"SELECT * from {table[TEMP].__tablename__}"
sa.text(
f"insert into {table[CACHE].__tablename__} "
f"SELECT * from {table[TEMP].__tablename__}"
)
)
logging.debug(f"in transaction, after insert into")
logging.info(f"cache unavailable time: {str(datetime.now() - start)}")
20 changes: 10 additions & 10 deletions anyway/parsers/registered.py
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
from datetime import datetime
from anyway.models import RegisteredVehicle, DeprecatedCity
from anyway.utilities import time_delta, CsvReader, ImporterUI, truncate_tables, decode_hebrew
from anyway.app_and_db import db
from anyway.app_and_db import db, app


COLUMN_CITY_NAME_ENG = 0
@@ -71,8 +71,8 @@ def import_file(self, inputfile):
else:
self.header_row(row)
row_count += 1

db.session.bulk_insert_mappings(RegisteredVehicle, inserts)
with app.app_context():
db.session.bulk_insert_mappings(RegisteredVehicle, inserts)
return total

@staticmethod
@@ -131,11 +131,11 @@ def main(specific_folder, delete_all, path):
started = datetime.now()
for fname in dir_files:
total += importer.import_file(fname)

db.session.commit()
db.engine.execute(
"UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format(
RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__
with app.app_context():
db.session.commit()
db.get_engine().execute(
"UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format(
RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__
)
)
)
logging.info("Total: {0} items in {1}".format(total, time_delta(started)))
logging.info("Total: {0} items in {1}".format(total, time_delta(started)))
3 changes: 2 additions & 1 deletion anyway/utilities.py
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@
from urllib.parse import urlparse
from sqlalchemy import func, or_
from sqlalchemy.sql import select
import sqlalchemy as sa

import phonenumbers
from dateutil.relativedelta import relativedelta
@@ -196,7 +197,7 @@ def truncate_tables(db, tables):
def delete_all_rows_from_table(conn, table):
table_name = table.__tablename__
logging.info("Deleting all rows from table " + table_name)
conn.execute("DELETE FROM " + table_name)
conn.execute(sa.text("DELETE FROM " + table_name))


def split_query_to_chunks_by_column(base_select, column_to_chunk_by, chunk_size, conn):

0 comments on commit 90f2675

Please sign in to comment.