routes and models

Pipfile

@@ -20,6 +20,8 @@ psycopg2-binary = "*"
 uvicorn = "*"
 email_validator = "*"
 sentry-sdk = "*"
+geoalchemy2 = "*"
+boto3 = "*"
 
 [requires]
 python_version = "3.7"

README.md

@@ -1,20 +1,18 @@
-# fastapi-gino-arq-uvicorn
-High-performance Async REST API, in Python. FastAPI + GINO + Arq + Uvicorn (powered by Redis & PostgreSQL).
+# GFW Data API
+High-performance Async REST API, in Python. FastAPI + GINO + Uvicorn (powered by PostgreSQL).
 
 ## Get Started
 ### Run Locally
-_NOTE: You must have PostgreSQL & Redis running locally._
+_NOTE: You must have PostgreSQL running locally._
 
 1. Clone this Repository. `git clone https://github.com/leosussan/fastapi-gino-arq-uvicorn.git`
 2. Run `pipenv install --dev` from root. (Run `pip install pipenv` first, if necessary.)
-3. Make a copy of `.dist.env`, rename to `.env`. Fill in PostgreSQL, Redis connection vars.
+3. Make a copy of `.dist.env`, rename to `.env`. Fill in PostgreSQL connection vars.
 4. Generate DB Migrations: `alembic revision --autogenerate`. It will be applied when the application starts. You can trigger manually with `alembic upgrade head`.
 5. Run:
     - FastAPI Application:
         * _For Active Development (w/ auto-reload):_ Run locally with `pipenv run uvicorn app.main:app --reload `
         * _For Debugging (compatible w/ debuggers, no auto-reload):_ Configure debugger to run `python app/main.py`.
-    - Background Task Worker:
-        * _For Active Development:_ Run with `pipenv run arq app.worker.Worker --watch ./`
 
 ### Run Locally with Docker-Compose
 1. Clone this Repository. `git clone https://github.com/leosussan/fastapi-gino-arq-uvicorn.git`
@@ -30,22 +28,17 @@ _NOTE: You must have PostgreSQL & Redis running locally._
 * Store complex db queries in `/app/models/orm/queries`
 * Store complex tasks in `app/tasks`.
 * Add / edit globals to `/.env`, expose & import them from `/app/settings/globals.py`
-    * Use any coroutine as a background function: store a reference in the `ARQ_BACKGROUND_FUNCTIONS` env.
-    * Set `SENTRY_DSN` in your environment to enable Sentry.
 * Define code to run before launch (migrations, setup, etc) in `/app/settings/prestart.sh`
 
 ## Features
 ### Core Dependencies
 * **FastAPI:** touts performance on-par with NodeJS & Go + automatic Swagger + ReDoc generation. 
 * **GINO:** built on SQLAlchemy core. Lightweight, simple, asynchronous ORM for PostgreSQL.
-* **Arq:** Asyncio + Redis = fast, resource-light job queuing & RPC.
 * **Uvicorn:** Lightning-fast, asynchronous ASGI server.
 * **Optimized Dockerfile:** Optimized Dockerfile for ASGI applications, from https://github.com/tiangolo/uvicorn-gunicorn-docker.
 
 #### Additional Dependencies
 * **Pydantic:** Core to FastAPI. Define how data should be in pure, canonical python; validate it with pydantic. 
 * **Alembic:** Handles database migrations. Compatible with GINO.
 * **SQLAlchemy_Utils:** Provides essential handles & datatypes. Compatible with GINO.
-* **Sentry:** Open-source, cloud-hosted error + event monitoring.
 * **PostgreSQL:** Robust, fully-featured, scalable, open-source.
-* **Redis:** Fast, simple, broker for the Arq task queue.

alembic.ini

@@ -35,6 +35,9 @@ script_location = app/models/orm/migrations
 # are written from script.py.mako
 # output_encoding = utf-8
 
+[alembic:exclude]
+tables = spatial_ref_sys
+
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

app/application.py

@@ -1,13 +1,8 @@
 from fastapi import FastAPI
 from gino.ext.starlette import Gino
-from sentry_sdk import init as initialize_sentry
-from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 from sqlalchemy.schema import MetaData
 
-from .settings.globals import DATABASE_CONFIG, SENTRY_DSN
-
-if SENTRY_DSN not in (None, ""):
-    initialize_sentry(dsn=SENTRY_DSN, integrations=[SqlalchemyIntegration()])
+from .settings.globals import DATABASE_CONFIG
 
 app: FastAPI = FastAPI()
 db: MetaData = Gino(app, dsn=DATABASE_CONFIG.url)

app/main.py

@@ -2,21 +2,15 @@
 
 sys.path.extend(["./"])
 
-from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
-
 from app.application import app
-from app.routes.users import router as user_router
-from app.settings.globals import SENTRY_DSN
+from app.routes import datasets, features, fields, geostore, query, sources, versions
 
 
-ROUTERS = (user_router,)
+ROUTERS = (datasets.router, versions.router, sources.router, fields.router, query.router, features.router, geostore.router)
 
 for r in ROUTERS:
     app.include_router(r)
 
-if SENTRY_DSN not in (None, "", " "):
-    app.add_middleware(SentryAsgiMiddleware)
-
 
 if __name__ == "__main__":
     import uvicorn

app/models/orm/asset.py

@@ -0,0 +1,12 @@
+from .base import Base, db
+
+
+class Asset(Base):
+    __tablename__ = 'assets'
+    dataset = db.Column(db.String, primary_key=True)
+    version = db.Column(db.String, primary_key=True)
+    asset_type = db.Column(db.String, primary_key=True)
+    asset_uri = db.Column(db.String)
+    metadata = db.Column(db.JSONB)
+
+    fk = db.ForeignKeyConstraint(["dataset", "version"], ["versions.dataset", "versions.version"], name="fk")

app/models/orm/base.py

@@ -2,10 +2,11 @@
 
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy_utils import EmailType, generic_repr
+from geoalchemy2 import Geometry
 
 from ...application import db
 
-db.JSONB, db.UUID, db.EmailType = (JSONB, UUID, EmailType)
+db.JSONB, db.UUID, db.EmailType, db.Geometry = (JSONB, UUID, EmailType, Geometry)
 
 
 @generic_repr
@@ -14,4 +15,4 @@ class Base(db.Model):
     created_on = db.Column(db.DateTime, default=datetime.utcnow, server_default=db.func.now())
     updated_on = db.Column(
         db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, server_default=db.func.now())
-    id = db.Column(db.BigInteger, primary_key=True, autoincrement=True)
+

app/models/orm/dataset.py

@@ -0,0 +1,8 @@
+from .base import Base, db
+
+
+class Dataset(Base):
+    __tablename__ = 'datasets'
+    dataset = db.Column(db.String, primary_key=True)
+    metadata = db.Column(db.JSONB)
+

app/models/orm/geostore.py

@@ -0,0 +1,12 @@
+from .base import Base, db
+
+
+class Geostore(Base):
+    __tablename__ = 'geostore'
+
+    gfw_geostore_id = db.Column(db.UUID, primary_key=True)
+    gfw_geojson = db.Column(db.String, nullable=False),
+    gfw_area__ha = db.Column(db.Numeric, nullable=False)
+    gfw_bbox = db.Column(db.Geometry("Polygon", 4326), nullable=False) # TODO check if this is the correct type
+
+    _geostore_gfw_geostore_id_idx = db.Index("geostore_gfw_geostore_id_idx", "gfw_geostore_id", postgresql_using='hash')

app/models/orm/migrations/env.py

@@ -4,9 +4,13 @@
 
 ######################## --- MODELS FOR MIGRATIONS --- ########################
 from app.application import db
-from app.models.orm.user import User
 
 # To include a model in migrations, add a line here.
+from app.models.orm.dataset import Dataset
+from app.models.orm.version import Version
+from app.models.orm.asset import Asset
+from app.models.orm.geostore import Geostore
+
 
 ###############################################################################
 
@@ -23,6 +27,24 @@
 target_metadata = db
 
 
+def exclude_tables_from_config(config_):
+    tables = None
+    tables_ = config_.get("tables", None)
+    if tables_ is not None:
+        tables = tables_.split(",")
+    return tables
+
+
+exclude_tables = exclude_tables_from_config(config.get_section('alembic:exclude'))
+
+
+def include_object(object, name, type_, reflected, compare_to):
+    if type_ == "table" and name in exclude_tables:
+        return False
+    else:
+        return True
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -36,7 +58,7 @@ def run_migrations_offline():
 
     """
     context.configure(
-        url=ALEMBIC_CONFIG.url.__to_string__(hide_password=False), target_metadata=target_metadata, literal_binds=True
+        url=ALEMBIC_CONFIG.url.__to_string__(hide_password=False), target_metadata=target_metadata, literal_binds=True, include_object=include_object
     )
 
     with context.begin_transaction():
@@ -57,7 +79,7 @@ def run_migrations_online():
     )
     with connectable.connect() as connection:
         context.configure(
-            connection=connection, target_metadata=target_metadata
+            connection=connection, target_metadata=target_metadata, include_object=include_object
         )
 
         with context.begin_transaction():

app/models/orm/migrations/versions/86ae41de358d_.py

@@ -0,0 +1,108 @@
+"""empty message
+
+Revision ID: 86ae41de358d
+Revises: 
+Create Date: 2020-04-14 21:58:38.173605
+
+"""
+import json
+import os
+
+import boto3
+import sqlalchemy as sa
+import geoalchemy2
+
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e47ec2fc3c51'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+client = boto3.client("secretsmanager")
+response = client.get_secret_value(SecretId=os.environ["SECRET_NAME"])
+secrets = json.loads(response["SecretString"])
+
+USERNAME = secrets["username"]
+PASSWORD = secrets["password"]
+DBNAME = secrets["dbname"]
+
+
+def upgrade():
+
+    #### Create read only user
+    op.execute(f"""
+                DO
+                $do$
+                BEGIN
+                   IF NOT EXISTS (
+                      SELECT                       -- SELECT list can stay empty for this
+                      FROM   pg_catalog.pg_roles
+                      WHERE  rolname = '{USERNAME}') THEN
+                      CREATE ROLE {USERNAME} LOGIN PASSWORD '{PASSWORD}';
+                   END IF;
+                END
+                $do$;
+                """)
+    op.execute(f"GRANT CONNECT ON DATABASE {DBNAME} TO {USERNAME};")
+    op.execute(f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO {USERNAME};")
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('datasets',
+    sa.Column('created_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('updated_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('dataset', sa.String(), nullable=False),
+    sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.PrimaryKeyConstraint('dataset')
+    )
+    op.create_table('geostore',
+    sa.Column('created_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('updated_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('gfw_geostore_id', postgresql.UUID(), nullable=False),
+    sa.Column('gfw_area__ha', sa.Numeric(), nullable=False),
+    sa.Column('gfw_bbox', geoalchemy2.types.Geometry(geometry_type='POLYGON', srid=4326), nullable=False),
+    sa.PrimaryKeyConstraint('gfw_geostore_id')
+    )
+    op.create_index('geostore_gfw_geostore_id_idx', 'geostore', ['gfw_geostore_id'], unique=False, postgresql_using='hash')
+    op.create_table('versions',
+    sa.Column('created_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('updated_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('dataset', sa.String(), nullable=False),
+    sa.Column('version', sa.String(), nullable=False),
+    sa.Column('is_latest', sa.Boolean(), nullable=True),
+    sa.Column('source_type', sa.String(), nullable=False),
+    sa.Column('has_vector_tile_cache', sa.Boolean(), nullable=True),
+    sa.Column('has_raster_tile_cache', sa.Boolean(), nullable=True),
+    sa.Column('has_geostore', sa.Boolean(), nullable=True),
+    sa.Column('has_feature_info', sa.Boolean(), nullable=True),
+    sa.Column('has_10_40000_tiles', sa.Boolean(), nullable=True),
+    sa.Column('has_90_27008_tiles', sa.Boolean(), nullable=True),
+    sa.Column('has_90_9876_tiles', sa.Boolean(), nullable=True),
+    sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.ForeignKeyConstraint(['dataset'], ['datasets.dataset'], name='fk'),
+    sa.PrimaryKeyConstraint('dataset', 'version')
+    )
+    op.create_table('assets',
+    sa.Column('created_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('updated_on', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
+    sa.Column('dataset', sa.String(), nullable=False),
+    sa.Column('version', sa.String(), nullable=False),
+    sa.Column('asset_type', sa.String(), nullable=False),
+    sa.Column('asset_uri', sa.String(), nullable=True),
+    sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.ForeignKeyConstraint(['dataset', 'version'], ['versions.dataset', 'versions.version'], name='fk'),
+    sa.PrimaryKeyConstraint('dataset', 'version', 'asset_type')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('assets')
+    op.drop_table('versions')
+    op.drop_index('geostore_gfw_geostore_id_idx', table_name='geostore')
+    op.drop_table('geostore')
+    op.drop_table('datasets')
+    # ### end Alembic commands ###

app/models/orm/version.py

@@ -0,0 +1,19 @@
+from .base import Base, db
+
+
+class Version(Base):
+    __tablename__ = 'versions'
+    dataset = db.Column(db.String, primary_key=True)
+    version = db.Column(db.String, primary_key=True)
+    is_latest = db.Column(db.Boolean, default=False)
+    source_type = db.Column(db.String, nullable=False)
+    has_vector_tile_cache = db.Column(db.Boolean, default=False)
+    has_raster_tile_cache = db.Column(db.Boolean, default=False)
+    has_geostore = db.Column(db.Boolean, default=False)
+    has_feature_info = db.Column(db.Boolean, default=False)
+    has_10_40000_tiles = db.Column(db.Boolean, default=False)
+    has_90_27008_tiles = db.Column(db.Boolean, default=False)
+    has_90_9876_tiles = db.Column(db.Boolean, default=False)
+    metadata = db.Column(db.JSONB)
+
+    fk = db.ForeignKeyConstraint(["dataset"], ["datasets.dataset"], name="fk")

app/routes/__init__.py

@@ -0,0 +1,18 @@
+from fastapi import Path
+
+
+VERSION_REGEX = r"^v\d{1,8}\.?\d{1,3}\.?\d{1,3}$|^latest$"
+
+
+async def dataset_dependency(dataset: str = Path(..., title="Dataset")):
+    return dataset
+
+
+async def version_dependency(
+    version: str = Path(..., title="Dataset version", regex=VERSION_REGEX)
+):
+
+    # if version == "latest":
+    #     version = ...
+
+    return version