diff --git a/.gitignore b/.gitignore index e6fa1141..157f14c5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ pypgstac/dist *.pyc *.egg-info *.eggs -venv \ No newline at end of file +venv +.direnv diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5a12b5..f90881ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,32 @@ # Changelog +## [v0.5.0] +Version 0.5.0 is a major refactor of how data is stored. It is recommended to start a new database from scratch and to move data over rather than to use the inbuilt migration which will be very slow for larger amounts of data. + +### Fixed + +### Changed + - The partition layout has been changed from being hardcoded to a partition to week to using nested partitions. The first level is by collection, for each collection, there is an attribute partition_trunc which can be set to NULL (no temporal partitions), month, or year. + + - CQL1 and Query Code have been refactored to translate to CQL2 to reduce duplicated code in query parsing. + + - Unused functions have been stripped from the project. + + - Pypgstac has been changed to use Fire rather than Typo. + + - Pypgstac has been changed to use Psycopg3 rather than Asyncpg to enable easier use as both sync and async. + + - Indexing has been reworked to eliminate indexes that from logs were not being used. The global json index on properties has been removed. Indexes on individual properties can be added either globally or per collection using the new queryables table. + + - Triggers for maintaining partitions have been updated to reduce lock contention and to reflect the new data layout. + + - The data pager which optimizes "order by datetime" searches has been updated to get time periods from the new partition layout and partition metadata. + + - Tests have been updated to reflect the many changes. + +### Added + + - On ingest, the content in an item is compared to the metadata available at the collection level and duplicate information is stripped out (this is primarily data in the item_assets property). Logic is added in to merge this data back in on data usage. + ## [v0.4.5] ### Fixed diff --git a/Dockerfile b/Dockerfile index c0740884..8e62eabe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,8 +33,9 @@ RUN \ python3-setuptools \ && pip3 install -U pip setuptools packaging \ && pip3 install -U psycopg2-binary \ + && pip3 install -U psycopg[binary] \ && pip3 install -U migra[pg] \ - && pip3 install poetry==1.1.12 \ + && pip3 install poetry==1.1.13 \ && apt-get remove -y apt-transport-https \ && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.dev b/Dockerfile.dev index 64334d46..811e533a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,7 +12,9 @@ ENV \ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* -RUN pip install poetry==1.1.7 +RUN pip install --upgrade pip && \ + pip install --upgrade poetry==1.1.13 && \ + pip install --upgrade psycopg[binary] RUN mkdir -p /opt/src/pypgstac diff --git a/README.md b/README.md index e36e2c33..7c115e44 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,53 @@ STAC Client that uses PGStac available in [STAC-FastAPI](https://github.com/stac PGStac requires **Postgresql>=13** and **PostGIS>=3**. Best performance will be had using PostGIS>=3.1. ### PGStac Settings -PGStac installs everything into the pgstac schema in the database. You will need to make sure that this schema is set up in the search_path for the database. +PGStac installs everything into the pgstac schema in the database. This schema must be in the search_path in the postgresql session while using pgstac. + +#### PGStac Users +The pgstac_admin role is the owner of all the objects within pgstac and should be used when running things such as migrations. + +The pgstac_ingest role has read/write priviliges on all tables and should be used for data ingest or if using the transactions extension with stac-fastapi-pgstac. + +The pgstac_read role has read only access to the items and collections, but will still be able to write to the logging tables. + +You can use the roles either directly and adding a password to them or by granting them to a role you are already using. + +To use directly: +```sql +ALTER ROLE pgstac_read LOGIN PASSWORD ''; +``` + +To grant pgstac permissions to a current postgresql user: +```sql +GRANT pgstac_read TO ; +``` + +#### PGStac Search Path +The search_path can be set at the database level or role level or by setting within the current session. The search_path is already set if you are directly using one of the pgstac users. If you are not logging in directly as one of the pgstac users, you will need to set the search_path by adding it to the search_path of the user you are using: +```sql +ALTER ROLE SET SEARCH_PATH TO pgstac, public; +``` +setting the search_path on the database: +```sql +ALTER DATABASE set search_path to pgstac, public; +``` + +In psycopg the search_path can be set by passing it as a configuration when creating your connection: +```python +kwargs={ + "options": "-c search_path=pgstac,public" +} +``` + +#### PGStac Settings Variables There are additional variables that control the settings used for calculating and displaying context (total row count) for a search, as well as a variable to set the filter language (cql-json or cql-json2). The context is "off" by default, and the default filter language is set to "cql2-json". Variables can be set either by passing them in via the connection options using your connection library, setting them in the pgstac_settings table or by setting them on the Role that is used to log in to the database. +Turning "context" on can be **very** expensive on larger databases. Much of what PGStac does is to optimize the search of items sorted by time where only fewer than 10,000 records are returned at a time. It does this by searching for the data in chunks and is able to "short circuit" and return as soon as it has the number of records requested. Calculating the context (the total count for a query) requires a scan of all records that match the query parameters and can take a very long time. Settting "context" to auto will use database statistics to estimate the number of rows much more quickly, but for some queries, the estimate may be quite a bit off. + Example for updating the pgstac_settings table with a new value: ```sql INSERT INTO pgstac_settings (name, value) @@ -41,7 +81,7 @@ ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.va ``` Alternatively, update the role: -``` +```sql ALTER ROLE SET SEARCH_PATH to pgstac, public; ALTER ROLE SET pgstac.context TO <'on','off','auto'>; ALTER ROLE SET pgstac.context_estimated_count TO ''; @@ -49,6 +89,28 @@ ALTER ROLE SET pgstac.context_estimated_cost TO ' SET pgstac.context_stats_ttl TO '>'; ``` +#### PGStac Partitioning +By default PGStac partitions data by collection (note: this is a change starting with version 0.5.0). Each collection can further be partitioned by either year or month. **Partitioning must be set up prior to loading any data!** Partitioning can be configured by setting the partition_trunc flag on a collection in the database. +```sql +UPDATE collections set partition_trunc='month' WHERE id=''; +``` + +In general, you should aim to keep each partition less than a few hundred thousand rows. Further partitioning (ie setting everything to 'month' when not needed to keep the partitions below a few hundred thousand rows) can be detrimental. + +#### PGStac Indexes / Queryables +By default, PGStac includes indexes on the id, datetime, collection, geometry, and the eo:cloud_cover property. Further indexing can be added for additional properties globally or only on particular collections by modifications to the queryables table. + +Currently indexing is the only place the queryables table is used, but in future versions, it will be extended to provide a queryables backend api. + +To add a new global index across all partitions: +```sql +INSERT INTO pgstac.queryables (name, property_wrapper, property_index_type) +VALUES (, , ); +``` +Property wrapper should be one of to_int, to_float, to_tstz, or to_text. The index type should almost always be 'BTREE', but can be any PostgreSQL index type valid for the data type. + +**More indexes is note necessarily better.** You should only index the primary fields that are actively being used to search. Adding too many indexes can be very detrimental to performance and ingest speed. If your primary use case is delivering items sorted by datetime and you do not use the context extension, you likely will not need any further indexes. + ## PyPGStac PGStac includes a Python utility for bulk data loading and managing migrations. diff --git a/pgstac.sql b/pgstac.sql index f12bb8da..a02f376a 100644 --- a/pgstac.sql +++ b/pgstac.sql @@ -1,12 +1,14 @@ BEGIN; \i sql/001_core.sql \i sql/001a_jsonutils.sql -\i sql/001b_cursorutils.sql \i sql/001s_stacutils.sql \i sql/002_collections.sql +\i sql/002a_queryables.sql +\i sql/002b_cql.sql \i sql/003_items.sql \i sql/004_search.sql \i sql/005_tileutils.sql \i sql/006_tilesearch.sql +\i sql/998_permissions.sql \i sql/999_version.sql COMMIT; diff --git a/pypgstac/poetry.lock b/pypgstac/poetry.lock index a295c838..c8a96e13 100644 --- a/pypgstac/poetry.lock +++ b/pypgstac/poetry.lock @@ -1,19 +1,3 @@ -[[package]] -name = "asyncpg" -version = "0.25.0" -description = "An asyncio PostgreSQL driver" -category = "main" -optional = false -python-versions = ">=3.6.0" - -[package.dependencies] -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} - -[package.extras] -dev = ["Cython (>=0.29.24,<0.30.0)", "pytest (>=6.0)", "Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] -docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)"] -test = ["pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"] - [[package]] name = "atomicwrites" version = "1.4.0" @@ -24,52 +8,59 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.2.0" +version = "21.4.0" description = "Classes Without Boilerplate" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] + +[[package]] +name = "backports.zoneinfo" +version = "0.2.1" +description = "Backport of the standard library zoneinfo module" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +tzdata = ["tzdata"] [[package]] name = "black" -version = "21.12b0" +version = "22.3.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.6.2" [package.dependencies] -click = ">=7.1.2" +click = ">=8.0.0" mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0,<1" +pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = ">=0.2.6,<2.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} -typing-extensions = [ - {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}, - {version = "!=3.10.0.1", markers = "python_version >= \"3.10\""}, -] +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -python2 = ["typed-ast (>=1.4.3)"] uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "click" -version = "8.0.3" +version = "8.1.2" description = "Composable command line interface toolkit" -category = "main" +category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -79,10 +70,22 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "fire" +version = "0.4.0" +description = "A library for automatically generating command line interfaces." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" +termcolor = "*" + [[package]] name = "flake8" version = "3.9.2" @@ -99,9 +102,9 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "importlib-metadata" -version = "4.10.0" +version = "4.11.3" description = "Read metadata from Python packages" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" @@ -110,9 +113,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "mccabe" @@ -158,7 +161,7 @@ python-versions = "*" [[package]] name = "orjson" -version = "3.6.5" +version = "3.6.7" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" category = "main" optional = false @@ -185,16 +188,27 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [[package]] name = "platformdirs" -version = "2.4.0" +version = "2.5.1" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +[[package]] +name = "plpygis" +version = "0.2.0" +description = "PostGIS Python tools" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +shapely_support = ["Shapely (>=1.5.0)"] + [[package]] name = "pluggy" version = "0.13.1" @@ -209,6 +223,35 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] +[[package]] +name = "psycopg" +version = "3.0.11" +description = "PostgreSQL database adapter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +"backports.zoneinfo" = {version = ">=0.2.0", markers = "python_version < \"3.9\""} +typing-extensions = {version = ">=3.10", markers = "python_version < \"3.8\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.0.11)"] +c = ["psycopg-c (==3.0.11)"] +dev = ["black (>=22.3.0)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=0.920,!=0.930,!=0.931)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +docs = ["Sphinx (>=4.2)", "furo (==2021.11.23)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)", "dnspython (>=2.1)", "shapely (>=1.7)"] +pool = ["psycopg-pool"] +test = ["mypy (>=0.920,!=0.930,!=0.931)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-asyncio (>=0.16,<0.17)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.10)"] + +[[package]] +name = "psycopg-pool" +version = "3.1.1" +description = "Connection Pool for Psycopg" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "py" version = "1.11.0" @@ -225,6 +268,21 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pydantic" +version = "1.9.0" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pyflakes" version = "2.3.1" @@ -235,7 +293,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pyparsing" -version = "3.0.6" +version = "3.0.7" description = "Python parsing module" category = "dev" optional = false @@ -303,6 +361,25 @@ s3 = ["boto3"] test = ["boto3", "google-cloud-storage", "azure-storage-blob", "azure-common", "azure-core", "requests", "mock", "moto[server] (==1.3.14)", "pathlib2", "responses", "boto3", "paramiko", "parameterizedtestcase", "pytest", "pytest-rerunfailures"] webhdfs = ["requests"] +[[package]] +name = "tenacity" +version = "8.0.1" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + +[[package]] +name = "termcolor" +version = "1.1.0" +description = "ANSII Color formatting for output in terminal." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "toml" version = "0.10.2" @@ -313,11 +390,11 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomli" -version = "1.2.3" +version = "2.0.1" description = "A lil' TOML parser" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [[package]] name = "typed-ast" @@ -327,23 +404,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "typer" -version = "0.4.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -click = ">=7.1.1,<9.0.0" - -[package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)"] -doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=5.4.0,<6.0.0)", "markdown-include (>=0.5.1,<0.6.0)"] -test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "coverage (>=5.2,<6.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.910)", "black (>=19.10b0,<20.0b0)", "isort (>=5.0.6,<6.0.0)"] - [[package]] name = "types-orjson" version = "0.1.1" @@ -354,12 +414,20 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.0.1" +version = "4.1.1" description = "Backported and Experimental Type Hints for Python 3.6+" category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "tzdata" +version = "2022.1" +description = "Provider of IANA time zone data" +category = "main" +optional = false +python-versions = ">=2" + [[package]] name = "wcwidth" version = "0.2.5" @@ -370,77 +438,91 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.6.0" +version = "3.8.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" +category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = ">=3.7" -content-hash = "feb826401446690f1627416374507be755c3747bf9f3af83f79dc1b4ed2d47c9" +content-hash = "b8f8aadcfbac840f6ade94cfb16bf02e7a36bb5e54c4471a77e918e24013c6f9" [metadata.files] -asyncpg = [ - {file = "asyncpg-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3"}, - {file = "asyncpg-0.25.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a"}, - {file = "asyncpg-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a70783f6ffa34cc7dd2de20a873181414a34fd35a4a208a1f1a7f9f695e4ec4"}, - {file = "asyncpg-0.25.0-cp310-cp310-win32.whl", hash = "sha256:43cde84e996a3afe75f325a68300093425c2f47d340c0fc8912765cf24a1c095"}, - {file = "asyncpg-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:56d88d7ef4341412cd9c68efba323a4519c916979ba91b95d4c08799d2ff0c09"}, - {file = "asyncpg-0.25.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a84d30e6f850bac0876990bcd207362778e2208df0bee8be8da9f1558255e634"}, - {file = "asyncpg-0.25.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:beaecc52ad39614f6ca2e48c3ca15d56e24a2c15cbfdcb764a4320cc45f02fd5"}, - {file = "asyncpg-0.25.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:6f8f5fc975246eda83da8031a14004b9197f510c41511018e7b1bedde6968e92"}, - {file = "asyncpg-0.25.0-cp36-cp36m-win32.whl", hash = "sha256:ddb4c3263a8d63dcde3d2c4ac1c25206bfeb31fa83bd70fd539e10f87739dee4"}, - {file = "asyncpg-0.25.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bf6dc9b55b9113f39eaa2057337ce3f9ef7de99a053b8a16360395ce588925cd"}, - {file = "asyncpg-0.25.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:acb311722352152936e58a8ee3c5b8e791b24e84cd7d777c414ff05b3530ca68"}, - {file = "asyncpg-0.25.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a61fb196ce4dae2f2fa26eb20a778db21bbee484d2e798cb3cc988de13bdd1b"}, - {file = "asyncpg-0.25.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2633331cbc8429030b4f20f712f8d0fbba57fa8555ee9b2f45f981b81328b256"}, - {file = "asyncpg-0.25.0-cp37-cp37m-win32.whl", hash = "sha256:863d36eba4a7caa853fd7d83fad5fd5306f050cc2fe6e54fbe10cdb30420e5e9"}, - {file = "asyncpg-0.25.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fe471ccd915b739ca65e2e4dbd92a11b44a5b37f2e38f70827a1c147dafe0fa8"}, - {file = "asyncpg-0.25.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:72a1e12ea0cf7c1e02794b697e3ca967b2360eaa2ce5d4bfdd8604ec2d6b774b"}, - {file = "asyncpg-0.25.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4327f691b1bdb222df27841938b3e04c14068166b3a97491bec2cb982f49f03e"}, - {file = "asyncpg-0.25.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:739bbd7f89a2b2f6bc44cb8bf967dab12c5bc714fcbe96e68d512be45ecdf962"}, - {file = "asyncpg-0.25.0-cp38-cp38-win32.whl", hash = "sha256:18d49e2d93a7139a2fdbd113e320cc47075049997268a61bfbe0dde680c55471"}, - {file = "asyncpg-0.25.0-cp38-cp38-win_amd64.whl", hash = "sha256:191fe6341385b7fdea7dbdcf47fd6db3fd198827dcc1f2b228476d13c05a03c6"}, - {file = "asyncpg-0.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fab7f1b2c29e187dd8781fce896249500cf055b63471ad66332e537e9b5f7e"}, - {file = "asyncpg-0.25.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a738f1b2876f30d710d3dc1e7858160a0afe1603ba16bf5f391f5316eb0ed855"}, - {file = "asyncpg-0.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4105f57ad1e8fbc8b1e535d8fcefa6ce6c71081228f08680c6dea24384ff0e"}, - {file = "asyncpg-0.25.0-cp39-cp39-win32.whl", hash = "sha256:f55918ded7b85723a5eaeb34e86e7b9280d4474be67df853ab5a7fa0cc7c6bf2"}, - {file = "asyncpg-0.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:649e2966d98cc48d0646d9a4e29abecd8b59d38d55c256d5c857f6b27b7407ac"}, - {file = "asyncpg-0.25.0.tar.gz", hash = "sha256:63f8e6a69733b285497c2855464a34de657f2cccd25aeaeeb5071872e9382540"}, -] atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] +"backports.zoneinfo" = [ + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win32.whl", hash = "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win32.whl", hash = "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-win32.whl", hash = "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"}, + {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"}, ] black = [ - {file = "black-21.12b0-py3-none-any.whl", hash = "sha256:a615e69ae185e08fdd73e4715e260e2479c861b5740057fde6e8b4e3b7dd589f"}, - {file = "black-21.12b0.tar.gz", hash = "sha256:77b80f693a569e2e527958459634f18df9b0ba2625ba4e0c2d5da5be42e6f2b3"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] click = [ - {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, - {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"}, + {file = "click-8.1.2-py3-none-any.whl", hash = "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e"}, + {file = "click-8.1.2.tar.gz", hash = "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +fire = [ + {file = "fire-0.4.0.tar.gz", hash = "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"}, +] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.10.0-py3-none-any.whl", hash = "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4"}, - {file = "importlib_metadata-4.10.0.tar.gz", hash = "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6"}, + {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, + {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, @@ -480,30 +562,38 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] orjson = [ - {file = "orjson-3.6.5-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:6c444edc073eb69cf85b28851a7a957807a41ce9bb3a9c14eefa8b33030cf050"}, - {file = "orjson-3.6.5-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:432c6da3d8d4630739f5303dcc45e8029d357b7ff8e70b7239be7bd047df6b19"}, - {file = "orjson-3.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:0fa32319072fadf0732d2c1746152f868a1b0f83c8cce2cad4996f5f3ca4e979"}, - {file = "orjson-3.6.5-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:0d65cc67f2e358712e33bc53810022ef5181c2378a7603249cd0898aa6cd28d4"}, - {file = "orjson-3.6.5-cp310-none-win_amd64.whl", hash = "sha256:fa8e3d0f0466b7d771a8f067bd8961bc17ca6ea4c89a91cd34d6648e6b1d1e47"}, - {file = "orjson-3.6.5-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:470596fbe300a7350fd7bbcf94d2647156401ab6465decb672a00e201af1813a"}, - {file = "orjson-3.6.5-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d2680d9edc98171b0c59e52c1ed964619be5cb9661289c0dd2e667773fa87f15"}, - {file = "orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:001962a334e1ab2162d2f695f2770d2383c7ffd2805cec6dbb63ea2ad96bf0ad"}, - {file = "orjson-3.6.5-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:522c088679c69e0dd2c72f43cd26a9e73df4ccf9ed725ac73c151bbe816fe51a"}, - {file = "orjson-3.6.5-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:d2b871a745a64f72631b633271577c99da628a9b63e10bd5c9c20706e19fe282"}, - {file = "orjson-3.6.5-cp37-none-win_amd64.whl", hash = "sha256:51ab01fed3b3e21561f21386a2f86a0415338541938883b6ca095001a3014a3e"}, - {file = "orjson-3.6.5-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:fc7e62edbc7ece95779a034d9e206d7ba9e2b638cc548fd3a82dc5225f656625"}, - {file = "orjson-3.6.5-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0720d60db3fa25956011a573274a269eb37de98070f3bc186582af1222a2d084"}, - {file = "orjson-3.6.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169a8876aed7a5bff413c53257ef1fa1d9b68c855eb05d658c4e73ed8dff508"}, - {file = "orjson-3.6.5-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:331f9a3bdba30a6913ad1d149df08e4837581e3ce92bf614277d84efccaf796f"}, - {file = "orjson-3.6.5-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:ece5dfe346b91b442590a41af7afe61df0af369195fed13a1b29b96b1ba82905"}, - {file = "orjson-3.6.5-cp38-none-win_amd64.whl", hash = "sha256:6a5e9eb031b44b7a429c705ca48820371d25b9467c9323b6ae7a712daf15fbef"}, - {file = "orjson-3.6.5-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:206237fa5e45164a678b12acc02aac7c5b50272f7f31116e1e08f8bcaf654f93"}, - {file = "orjson-3.6.5-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d5aceeb226b060d11ccb5a84a4cfd760f8024289e3810ec446ef2993a85dbaca"}, - {file = "orjson-3.6.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80dba3dbc0563c49719e8cc7d1568a5cf738accfcd1aa6ca5e8222b57436e75e"}, - {file = "orjson-3.6.5-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:443f39bc5e7966880142430ce091e502aea068b38cb9db5f1ffdcfee682bc2d4"}, - {file = "orjson-3.6.5-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:a06f2dd88323a480ac1b14d5829fb6cdd9b0d72d505fabbfbd394da2e2e07f6f"}, - {file = "orjson-3.6.5-cp39-none-win_amd64.whl", hash = "sha256:82cb42dbd45a3856dbad0a22b54deb5e90b2567cdc2b8ea6708e0c4fe2e12be3"}, - {file = "orjson-3.6.5.tar.gz", hash = "sha256:eb3a7d92d783c89df26951ef3e5aca9d96c9c6f2284c752aa3382c736f950597"}, + {file = "orjson-3.6.7-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:93188a9d6eb566419ad48befa202dfe7cd7a161756444b99c4ec77faea9352a4"}, + {file = "orjson-3.6.7-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:82515226ecb77689a029061552b5df1802b75d861780c401e96ca6bc8495f775"}, + {file = "orjson-3.6.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3af57ffab7848aaec6ba6b9e9b41331250b57bf696f9d502bacdc71a0ebab0ba"}, + {file = "orjson-3.6.7-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:a7297504d1142e7efa236ffc53f056d73934a993a08646dbcee89fc4308a8fcf"}, + {file = "orjson-3.6.7-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:5a50cde0dbbde255ce751fd1bca39d00ecd878ba0903c0480961b31984f2fab7"}, + {file = "orjson-3.6.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d21f9a2d1c30e58070f93988db4cad154b9009fafbde238b52c1c760e3607fbe"}, + {file = "orjson-3.6.7-cp310-none-win_amd64.whl", hash = "sha256:e152464c4606b49398afd911777decebcf9749cc8810c5b4199039e1afb0991e"}, + {file = "orjson-3.6.7-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:0a65f3c403f38b0117c6dd8e76e85a7bd51fcd92f06c5598dfeddbc44697d3e5"}, + {file = "orjson-3.6.7-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:6c47cfca18e41f7f37b08ff3e7abf5ada2d0f27b5ade934f05be5fc5bb956e9d"}, + {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:63185af814c243fad7a72441e5f98120c9ecddf2675befa486d669fb65539e9b"}, + {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2da6fde42182b80b40df2e6ab855c55090ebfa3fcc21c182b7ad1762b61d55c"}, + {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:48c5831ec388b4e2682d4ff56d6bfa4a2ef76c963f5e75f4ff4785f9cf338a80"}, + {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:913fac5d594ccabf5e8fbac15b9b3bb9c576d537d49eeec9f664e7a64dde4c4b"}, + {file = "orjson-3.6.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:58f244775f20476e5851e7546df109f75160a5178d44257d437ba6d7e562bfe8"}, + {file = "orjson-3.6.7-cp37-none-win_amd64.whl", hash = "sha256:2d5f45c6b85e5f14646df2d32ecd7ff20fcccc71c0ea1155f4d3df8c5299bbb7"}, + {file = "orjson-3.6.7-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:612d242493afeeb2068bc72ff2544aa3b1e627578fcf92edee9daebb5893ffea"}, + {file = "orjson-3.6.7-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:539cdc5067db38db27985e257772d073cd2eb9462d0a41bde96da4e4e60bd99b"}, + {file = "orjson-3.6.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6d103b721bbc4f5703f62b3882e638c0b65fcdd48622531c7ffd45047ef8e87c"}, + {file = "orjson-3.6.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb10a20f80e95102dd35dfbc3a22531661b44a09b55236b012a446955846b023"}, + {file = "orjson-3.6.7-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:bb68d0da349cf8a68971a48ad179434f75256159fe8b0715275d9b49fa23b7a3"}, + {file = "orjson-3.6.7-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:4a2c7d0a236aaeab7f69c17b7ab4c078874e817da1bfbb9827cb8c73058b3050"}, + {file = "orjson-3.6.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3be045ca3b96119f592904cf34b962969ce97bd7843cbfca084009f6c8d2f268"}, + {file = "orjson-3.6.7-cp38-none-win_amd64.whl", hash = "sha256:bd765c06c359d8a814b90f948538f957fa8a1f55ad1aaffcdc5771996aaea061"}, + {file = "orjson-3.6.7-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7dd9e1e46c0776eee9e0649e3ae9584ea368d96851bcaeba18e217fa5d755283"}, + {file = "orjson-3.6.7-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:c4b4f20a1e3df7e7c83717aff0ef4ab69e42ce2fb1f5234682f618153c458406"}, + {file = "orjson-3.6.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7107a5673fd0b05adbb58bf71c1578fc84d662d29c096eb6d998982c8635c221"}, + {file = "orjson-3.6.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a08b6940dd9a98ccf09785890112a0f81eadb4f35b51b9a80736d1725437e22c"}, + {file = "orjson-3.6.7-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:f5d1648e5a9d1070f3628a69a7c6c17634dbb0caf22f2085eca6910f7427bf1f"}, + {file = "orjson-3.6.7-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:e6201494e8dff2ce7fd21da4e3f6dfca1a3fed38f9dcefc972f552f6596a7621"}, + {file = "orjson-3.6.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:70d0386abe02879ebaead2f9632dd2acb71000b4721fd8c1a2fb8c031a38d4d5"}, + {file = "orjson-3.6.7-cp39-none-win_amd64.whl", hash = "sha256:d9a3288861bfd26f3511fb4081561ca768674612bac59513cb9081bb61fcc87f"}, + {file = "orjson-3.6.7.tar.gz", hash = "sha256:a4bb62b11289b7620eead2f25695212e9ac77fcfba76f050fa8a540fb5c32401"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -514,13 +604,24 @@ pathspec = [ {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] platformdirs = [ - {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, - {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, + {file = "platformdirs-2.5.1-py3-none-any.whl", hash = "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"}, + {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, +] +plpygis = [ + {file = "plpygis-0.2.0.tar.gz", hash = "sha256:f9d1bb3913970b6c40c67188be3716f9fa490c1441e6c0d915221c8291826079"}, ] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +psycopg = [ + {file = "psycopg-3.0.11-py3-none-any.whl", hash = "sha256:8fcbac023ef84922e107645fa8e21b7dc3b0fec0ca49c4270b53d7e64fb08e5f"}, + {file = "psycopg-3.0.11.tar.gz", hash = "sha256:5dfe409eefc91890602dff18ee17b3815e803d48f87de48f3fa8587653a5a2fb"}, +] +psycopg-pool = [ + {file = "psycopg-pool-3.1.1.tar.gz", hash = "sha256:bc579078dc8209f1ce280228460f96770756f24babb5d8ab2418800e9082a973"}, + {file = "psycopg_pool-3.1.1-py3-none-any.whl", hash = "sha256:397beaa082f17255e6267850a00700aec4427fa214b4c55d2d49c7c154508ed5"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -529,13 +630,50 @@ pycodestyle = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, ] +pydantic = [ + {file = "pydantic-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb23bcc093697cdea2708baae4f9ba0e972960a835af22560f6ae4e7e47d33f5"}, + {file = "pydantic-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d5278bd9f0eee04a44c712982343103bba63507480bfd2fc2790fa70cd64cf4"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab624700dc145aa809e6f3ec93fb8e7d0f99d9023b713f6a953637429b437d37"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d7da6f1c1049eefb718d43d99ad73100c958a5367d30b9321b092771e96c25"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c3b035103bd4e2e4a28da9da7ef2fa47b00ee4a9cf4f1a735214c1bcd05e0f6"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3011b975c973819883842c5ab925a4e4298dffccf7782c55ec3580ed17dc464c"}, + {file = "pydantic-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:086254884d10d3ba16da0588604ffdc5aab3f7f09557b998373e885c690dd398"}, + {file = "pydantic-1.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0fe476769acaa7fcddd17cadd172b156b53546ec3614a4d880e5d29ea5fbce65"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8e9dcf1ac499679aceedac7e7ca6d8641f0193c591a2d090282aaf8e9445a46"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e4c28f30e767fd07f2ddc6f74f41f034d1dd6bc526cd59e63a82fe8bb9ef4c"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c86229333cabaaa8c51cf971496f10318c4734cf7b641f08af0a6fbf17ca3054"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c0727bda6e38144d464daec31dff936a82917f431d9c39c39c60a26567eae3ed"}, + {file = "pydantic-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:dee5ef83a76ac31ab0c78c10bd7d5437bfdb6358c95b91f1ba7ff7b76f9996a1"}, + {file = "pydantic-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9c9bdb3af48e242838f9f6e6127de9be7063aad17b32215ccc36a09c5cf1070"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee7e3209db1e468341ef41fe263eb655f67f5c5a76c924044314e139a1103a2"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b6037175234850ffd094ca77bf60fb54b08b5b22bc85865331dd3bda7a02fa1"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b2571db88c636d862b35090ccf92bf24004393f85c8870a37f42d9f23d13e032"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b5ac0f1c83d31b324e57a273da59197c83d1bb18171e512908fe5dc7278a1d6"}, + {file = "pydantic-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bbbc94d0c94dd80b3340fc4f04fd4d701f4b038ebad72c39693c794fd3bc2d9d"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e0896200b6a40197405af18828da49f067c2fa1f821491bc8f5bde241ef3f7d7"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bdfdadb5994b44bd5579cfa7c9b0e1b0e540c952d56f627eb227851cda9db77"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:574936363cd4b9eed8acdd6b80d0143162f2eb654d96cb3a8ee91d3e64bf4cf9"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c556695b699f648c58373b542534308922c46a1cda06ea47bc9ca45ef5b39ae6"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f947352c3434e8b937e3aa8f96f47bdfe6d92779e44bb3f41e4c213ba6a32145"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5e48ef4a8b8c066c4a31409d91d7ca372a774d0212da2787c0d32f8045b1e034"}, + {file = "pydantic-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:96f240bce182ca7fe045c76bcebfa0b0534a1bf402ed05914a6f1dadff91877f"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:815ddebb2792efd4bba5488bc8fde09c29e8ca3227d27cf1c6990fc830fd292b"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c5b77947b9e85a54848343928b597b4f74fc364b70926b3c4441ff52620640c"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c68c3bc88dbda2a6805e9a142ce84782d3930f8fdd9655430d8576315ad97ce"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a79330f8571faf71bf93667d3ee054609816f10a259a109a0738dac983b23c3"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5a64b64ddf4c99fe201ac2724daada8595ada0d102ab96d019c1555c2d6441d"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a733965f1a2b4090a5238d40d983dcd78f3ecea221c7af1497b845a9709c1721"}, + {file = "pydantic-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cc6a4cb8a118ffec2ca5fcb47afbacb4f16d0ab8b7350ddea5e8ef7bcc53a16"}, + {file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"}, + {file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"}, +] pyflakes = [ {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pyparsing = [ - {file = "pyparsing-3.0.6-py3-none-any.whl", hash = "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4"}, - {file = "pyparsing-3.0.6.tar.gz", hash = "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"}, + {file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"}, + {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, ] pytest = [ {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, @@ -552,13 +690,20 @@ six = [ smart-open = [ {file = "smart_open-4.2.0.tar.gz", hash = "sha256:d9f5a0f173ccb9bbae528db5a3804f57145815774f77ef755b9b0f3b4b2a9dcb"}, ] +tenacity = [ + {file = "tenacity-8.0.1-py3-none-any.whl", hash = "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a"}, + {file = "tenacity-8.0.1.tar.gz", hash = "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f"}, +] +termcolor = [ + {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomli = [ - {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, - {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] typed-ast = [ {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, @@ -592,23 +737,23 @@ typed-ast = [ {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, ] -typer = [ - {file = "typer-0.4.0-py3-none-any.whl", hash = "sha256:d81169725140423d072df464cad1ff25ee154ef381aaf5b8225352ea187ca338"}, - {file = "typer-0.4.0.tar.gz", hash = "sha256:63c3aeab0549750ffe40da79a1b524f60e08a2cbc3126c520ebf2eeaf507f5dd"}, -] types-orjson = [ {file = "types-orjson-0.1.1.tar.gz", hash = "sha256:7454bfbaed27900a844bb9d8e211b69f1c335f0b9e3541d4950a793db41c104d"}, {file = "types_orjson-0.1.1-py2.py3-none-any.whl", hash = "sha256:92f85986261ea1a5cb215e4b35e4016631d35163a372f023918750f340ea737f"}, ] typing-extensions = [ - {file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"}, - {file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"}, + {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, + {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, +] +tzdata = [ + {file = "tzdata-2022.1-py2.py3-none-any.whl", hash = "sha256:238e70234214138ed7b4e8a0fab0e5e13872edab3be586ab8198c407620e2ab9"}, + {file = "tzdata-2022.1.tar.gz", hash = "sha256:8b536a8ec63dc0751342b3984193a3118f8fca2afe25752bb9b7fffd398552d3"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] zipp = [ - {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, - {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, + {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, + {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, ] diff --git a/pypgstac/pypgstac/__init__.py b/pypgstac/pypgstac/__init__.py index 963da0cc..fd81f05c 100644 --- a/pypgstac/pypgstac/__init__.py +++ b/pypgstac/pypgstac/__init__.py @@ -1,2 +1,2 @@ """PyPGStac Version.""" -__version__ = "0.4.5" +__version__ = "0.5.0" diff --git a/pypgstac/pypgstac/db.py b/pypgstac/pypgstac/db.py new file mode 100644 index 00000000..deff87e9 --- /dev/null +++ b/pypgstac/pypgstac/db.py @@ -0,0 +1,254 @@ +"""Base library for database interaction with PgStac.""" +import time +from types import TracebackType +from typing import Any, List, Optional, Tuple, Type, Union, Generator +import orjson +import psycopg +from psycopg import Connection, sql +from psycopg.types.json import set_json_dumps, set_json_loads +from psycopg_pool import ConnectionPool +import atexit +import logging +from pydantic import BaseSettings +from tenacity import retry, retry_if_exception_type, stop_after_attempt + + +def dumps(data: dict) -> str: + """Dump dictionary as string.""" + return orjson.dumps(data).decode() + + +set_json_dumps(dumps) +set_json_loads(orjson.loads) + + +def pg_notice_handler(notice: psycopg.errors.Diagnostic) -> None: + """Add PG messages to logging.""" + msg = f"{notice.severity} - {notice.message_primary}" + logging.info(msg) + + +class Settings(BaseSettings): + """Base Settings for Database Connection.""" + + db_min_conn_size: int = 0 + db_max_conn_size: int = 1 + db_max_queries: int = 5 + db_max_idle: int = 5 + db_num_workers: int = 1 + db_retries: int = 3 + + class Config: + """Use .env file if available.""" + + env_file = ".env" + + +settings = Settings() + + +class PgstacDB: + """Base class for interacting with PgStac Database.""" + + def __init__( + self, + dsn: Optional[str] = "", + pool: Optional[ConnectionPool] = None, + connection: Optional[Connection] = None, + commit_on_exit: bool = True, + debug: bool = False, + ) -> None: + """Initialize Database.""" + self.dsn: str + if dsn is not None: + self.dsn = dsn + else: + self.dsn = "" + self.pool = pool + self.connection = connection + self.commit_on_exit = commit_on_exit + self.initial_version = "0.1.9" + self.debug = debug + if self.debug: + logging.basicConfig(level=logging.DEBUG) + + def get_pool(self) -> ConnectionPool: + """Get Database Pool.""" + if self.pool is None: + self.pool = ConnectionPool( + conninfo=self.dsn, + min_size=settings.db_min_conn_size, + max_size=settings.db_max_conn_size, + max_waiting=settings.db_max_queries, + max_idle=settings.db_max_idle, + num_workers=settings.db_num_workers, + kwargs={ + "options": "-c search_path=pgstac,public" + " -c application_name=pypgstac" + }, + ) + return self.pool + + def open(self) -> None: + """Open database pool connection.""" + self.get_pool() + + def close(self) -> None: + """Close database pool connection.""" + if self.pool is not None: + self.pool.close() + + def connect(self) -> Connection: + """Return database connection.""" + pool = self.get_pool() + if self.connection is None: + self.connection = pool.getconn() + if self.debug: + self.connection.add_notice_handler(pg_notice_handler) + atexit.register(self.disconnect) + return self.connection + + def wait(self) -> None: + """Block until database connection is ready.""" + cnt: int = 0 + while cnt < 60: + try: + self.connect() + self.query("SELECT 1;") + return None + except psycopg.errors.OperationalError: + time.sleep(1) + cnt += 1 + raise psycopg.errors.CannotConnectNow + + def disconnect(self) -> None: + """Disconnect from database.""" + try: + if self.commit_on_exit: + if self.connection is not None: + self.connection.commit() + if self.connection is not None: + self.connection.rollback() + except: + pass + try: + if self.pool is not None and self.connection is not None: + self.pool.putconn(self.connection) + except: + pass + + self.connection = None + self.pool = None + + def __enter__(self) -> Any: + """Enter used for context.""" + self.connect() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + """Exit used for context.""" + self.disconnect() + + @retry( + stop=stop_after_attempt(settings.db_retries), + retry=retry_if_exception_type(psycopg.errors.OperationalError), + reraise=True, + ) + def query( + self, + query: Union[str, sql.Composed], + args: Optional[List] = None, + row_factory: psycopg.rows.BaseRowFactory = psycopg.rows.tuple_row, + ) -> Generator: + """Query the database with parameters.""" + conn = self.connect() + try: + with conn.cursor(row_factory=row_factory) as cursor: + if args is None: + rows = cursor.execute(query, prepare=False) + for row in rows: + yield row + else: + rows = cursor.execute(query, args) + for row in rows: + yield row + except psycopg.errors.OperationalError as e: + # If we get an operational error check the pool and retry + logging.warning(f"OPERATIONAL ERROR: {e}") + if self.pool is None: + self.get_pool() + else: + self.pool.check() + raise e + except psycopg.errors.DatabaseError as e: + if conn is not None: + conn.rollback() + raise e + + def query_one(self, *args: Any, **kwargs: Any) -> Union[Tuple, str, None]: + """Return results from a query that returns a single row.""" + try: + r = next(self.query(*args, **kwargs)) + except StopIteration: + return None + + if r is None: + return None + if len(r) == 1: + return r[0] + return r + + @property + def version(self) -> Optional[str]: + """Get the current version number from a pgstac database.""" + try: + version = self.query_one( + """ + SELECT version from pgstac.migrations + order by datetime desc, version desc limit 1; + """ + ) + logging.debug(f"VERSION: {version}") + if isinstance(version, str): + return version + except psycopg.errors.UndefinedTable: + logging.debug("PGStac is not installed.") + if self.connection is not None: + self.connection.rollback() + return None + + @property + def pg_version(self) -> str: + """Get the current pg version number from a pgstac database.""" + version = self.query_one( + """ + SHOW server_version; + """ + ) + logging.debug(f"PG VERSION: {version}.") + if isinstance(version, str): + if int(version.split(".")[0]) < 13: + raise Exception("PGStac requires PostgreSQL 13+") + return version + else: + if self.connection is not None: + self.connection.rollback() + raise Exception("Could not find PG version.") + + def func(self, function_name: str, *args: Any) -> Generator: + """Call a database function.""" + placeholders = sql.SQL(", ").join(sql.Placeholder() * len(args)) + func = sql.Identifier(function_name) + base_query = sql.SQL("SELECT * FROM {}({});").format(func, placeholders) + return self.query(base_query, *args) + + def search(self, query: Union[dict, str, psycopg.types.json.Jsonb] = "{}") -> str: + """Search PgStac.""" + if isinstance(query, dict): + query = psycopg.types.json.Jsonb(query) + return dumps(next(self.func("search", query))[0]) diff --git a/pypgstac/pypgstac/load.py b/pypgstac/pypgstac/load.py index 19617d54..e2e00c39 100644 --- a/pypgstac/pypgstac/load.py +++ b/pypgstac/pypgstac/load.py @@ -1,246 +1,537 @@ """Utilities to bulk load data into pgstac from json/ndjson.""" -from enum import Enum -from typing import Any, AsyncGenerator, Dict, Iterable, Optional, TypeVar +import contextlib +import itertools +import logging +import sys +import time +from dataclasses import dataclass +from functools import lru_cache +from typing import ( + Any, + BinaryIO, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + Union, + Generator, + TextIO, +) -import asyncpg import orjson -import typer -from asyncpg.connection import Connection +import psycopg +from orjson import JSONDecodeError +from plpygis.geometry import Geometry +from psycopg import sql from smart_open import open +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_random_exponential, +) + +from .db import PgstacDB +from enum import Enum -app = typer.Typer() - - -async def con_init(conn: Connection) -> None: - """Use orjson for json returns.""" - await conn.set_type_codec( - "json", - encoder=orjson.dumps, - decoder=orjson.loads, - schema="pg_catalog", - ) - await conn.set_type_codec( - "jsonb", - encoder=orjson.dumps, - decoder=orjson.loads, - schema="pg_catalog", - ) - - -class DB: - """Database connection context manager.""" - - pg_connection_string: Optional[str] = None - connection: Optional[Connection] = None - - def __init__(self, pg_connection_string: Optional[str] = None) -> None: - """Initialize DB class.""" - self.pg_connection_string = pg_connection_string - - async def create_connection(self) -> Connection: - """Create database connection and set search_path.""" - connection: Connection = await asyncpg.connect( - self.pg_connection_string, - server_settings={ - "search_path": "pgstac,public", - "application_name": "pypgstac", - }, - ) - await con_init(connection) - self.connection = connection - return self.connection - async def __aenter__(self) -> Connection: - """Enter DB Connection.""" - if self.connection is None: - await self.create_connection() - assert self.connection is not None - return self.connection +class Tables(str, Enum): + """Available tables for loading.""" - async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: - """Exit DB Connection.""" - if self.connection: - await self.connection.close() + items = "items" + collections = "collections" -class loadopt(str, Enum): - """Options for how to load data.""" +class Methods(str, Enum): + """Available methods for loading data.""" insert = "insert" - insert_ignore = "insert_ignore" + ignore = "ignore" upsert = "upsert" + delsert = "delsert" + insert_ignore = "insert_ignore" -class tables(str, Enum): - """Tables available to load data into.""" - - items = "items" - collections = "collections" +@contextlib.contextmanager +def open_std( + filename: str, mode: str = "r", *args: Any, **kwargs: Any +) -> Generator[Any, None, None]: + """Open files and i/o streams transparently.""" + fh: Union[TextIO, BinaryIO] + if ( + filename is None + or filename == "-" + or filename == "stdin" + or filename == "stdout" + ): + if "r" in mode: + stream = sys.stdin + else: + stream = sys.stdout + if "b" in mode: + fh = stream.buffer + else: + fh = stream + close = False + else: + fh = open(filename, mode, *args, **kwargs) + close = True + + try: + yield fh + finally: + if close: + try: + fh.close() + except AttributeError: + pass + + +def name_array_asdict(na: List) -> dict: + """Create a dict from a list with key from name field.""" + out = {} + for i in na: + out[i["name"]] = i + return out + + +def name_array_diff(a: List, b: List) -> List: + """Diff an array by name attribute.""" + diff = dict_minus(name_array_asdict(a), name_array_asdict(b)) + vals = diff.values() + return [v for v in vals if v != {}] + + +def dict_minus(a: dict, b: dict) -> dict: + """Get a recursive difference between two dicts.""" + out: dict = {} + for key, value in b.items(): + if isinstance(value, list): + try: + arraydiff = name_array_diff(a[key], value) + if arraydiff is not None and arraydiff != []: + out[key] = arraydiff + continue + except KeyError: + pass + except TypeError: + pass + + if value is None or value == []: + continue + if a is None or key not in a: + out[key] = value + continue + + if a.get(key) != value: + if isinstance(value, dict): + out[key] = dict_minus(a[key], value) + continue + out[key] = value + + return out + + +def read_json(file: Union[str, Iterator[Any]] = "stdin") -> Iterable: + """Load data from an ndjson or json file.""" + if file is None: + file = "stdin" + if isinstance(file, str): + open_file: Any = open_std(file, "r") + with open_file as f: + # Try reading line by line as ndjson + try: + for line in f: + line = line.strip().replace("\\\\", "\\").replace("\\\\", "\\") + yield orjson.loads(line) + except JSONDecodeError: + # If reading first line as json fails, try reading entire file + logging.info( + "First line could not be parsed as json, trying full file." + ) + try: + f.seek(0) + json = orjson.loads(f.read()) + if isinstance(json, list): + for record in json: + yield record + else: + yield json + except JSONDecodeError: + logging.info("File cannot be read as json") + raise + elif isinstance(file, Iterable): + for line in file: + if isinstance(line, Dict): + yield line + else: + yield orjson.loads(line) + + +@dataclass +class Loader: + """Utilities for loading data.""" + + db: PgstacDB + + @lru_cache + def collection_json(self, collection_id: str) -> Tuple[dict, int, str]: + """Get collection.""" + res = self.db.query_one( + "SELECT base_item, key, partition_trunc FROM collections WHERE id=%s", + (collection_id,), + ) + if isinstance(res, tuple): + base_item, key, partition_trunc = res + else: + raise Exception(f"Error getting info for {collection_id}.") + if key is None: + raise Exception( + f"Collection {collection_id} is not present in the database" + ) + return base_item, key, partition_trunc + + def load_collections( + self, + file: Union[str, Iterator[Any]] = "stdin", + insert_mode: Optional[Methods] = Methods.insert, + ) -> None: + """Load a collections json or ndjson file.""" + if file is None: + file = "stdin" + conn = self.db.connect() + with conn.cursor() as cur: + with conn.transaction(): + cur.execute( + """ + DROP TABLE IF EXISTS tmp_collections; + CREATE TEMP TABLE tmp_collections + (content jsonb) ON COMMIT DROP; + """ + ) + with cur.copy("COPY tmp_collections (content) FROM stdin;") as copy: + for collection in read_json(file): + copy.write_row((orjson.dumps(collection).decode(),)) + if insert_mode is None or insert_mode == "insert": + cur.execute( + """ + INSERT INTO collections (content) + SELECT content FROM tmp_collections; + """ + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + elif insert_mode in ("insert_ignore", "ignore"): + cur.execute( + """ + INSERT INTO collections (content) + SELECT content FROM tmp_collections + ON CONFLICT DO NOTHING; + """ + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + elif insert_mode == "upsert": + cur.execute( + """ + INSERT INTO collections (content) + SELECT content FROM tmp_collections + ON CONFLICT (id) DO + UPDATE SET content=EXCLUDED.content; + """ + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + else: + raise Exception( + "Available modes are insert, ignore, and upsert." + f"You entered {insert_mode}." + ) + + @retry( + stop=stop_after_attempt(5), + wait=wait_random_exponential(multiplier=1, max=120), + retry=( + retry_if_exception_type(psycopg.errors.CheckViolation) + | retry_if_exception_type(psycopg.errors.DeadlockDetected) + ), + reraise=True, + ) + def load_partition( + self, + partition: dict, + items: Iterable, + insert_mode: Optional[Methods] = Methods.insert, + ) -> None: + """Load items data for a single partition.""" + conn = self.db.connect() + t = time.perf_counter() + + logging.debug(f"Loading data for partition: {partition}.") + with conn.cursor() as cur: + with conn.transaction(): + cur.execute( + "SELECT * FROM partitions WHERE name = %s FOR UPDATE;", + (partition["partition"],), + ) + cur.execute( + """ + INSERT INTO partitions + (collection, datetime_range, end_datetime_range) + VALUES + (%s, tstzrange(%s, %s, '[]'), tstzrange(%s,%s, '[]')) + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + """, + ( + partition["collection"], + partition["mindt"], + partition["maxdt"], + partition["minedt"], + partition["maxedt"], + ), + ) + with conn.transaction(): + logging.debug( + f"Adding partition {partition} took {time.perf_counter() - t}s" + ) + t = time.perf_counter() + if insert_mode is None or insert_mode == "insert": + with cur.copy( + sql.SQL( + """ + COPY {} + (id, collection, datetime, end_datetime, geometry, content) + FROM stdin; + """ + ).format(sql.Identifier(partition["partition"])) + ) as copy: + for item in items: + item.pop("partition") + copy.write_row(list(item.values())) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + elif insert_mode in ( + "ignore_dupes", + "upsert", + "delsert", + "ignore", + ): + cur.execute( + """ + DROP TABLE IF EXISTS items_ingest_temp; + CREATE TEMP TABLE items_ingest_temp + ON COMMIT DROP AS SELECT * FROM items LIMIT 0; + """ + ) + with cur.copy( + """ + COPY items_ingest_temp + (id, collection, datetime, end_datetime, geometry, content) + FROM stdin; + """ + ) as copy: + for item in items: + item.pop("partition") + copy.write_row(list(item.values())) + logging.debug(cur.statusmessage) + logging.debug(f"Copied rows: {cur.rowcount}") + + cur.execute( + sql.SQL( + """ + LOCK TABLE ONLY {} IN EXCLUSIVE MODE; + """ + ).format(sql.Identifier(partition["partition"])) + ) + if insert_mode in ("ignore", "insert_ignore"): + cur.execute( + sql.SQL( + """ + INSERT INTO {} + SELECT * + FROM items_ingest_temp ON CONFLICT DO NOTHING; + """ + ).format(sql.Identifier(partition["partition"])) + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + elif insert_mode == "upsert": + cur.execute( + sql.SQL( + """ + INSERT INTO {} AS t SELECT * FROM items_ingest_temp + ON CONFLICT (id) DO UPDATE + SET + datetime = EXCLUDED.datetime, + end_datetime = EXCLUDED.end_datetime, + geometry = EXCLUDED.geometry, + collection = EXCLUDED.collection, + content = EXCLUDED.content + WHERE t IS DISTINCT FROM EXCLUDED + ; + """ + ).format(sql.Identifier(partition["partition"])) + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + elif insert_mode == "delsert": + cur.execute( + sql.SQL( + """ + WITH deletes AS ( + DELETE FROM items i USING items_ingest_temp s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO {} + SELECT s.* FROM + items_ingest_temp s + JOIN deletes d + USING (id, collection); + ; + """ + ).format(sql.Identifier(partition["partition"])) + ) + logging.debug(cur.statusmessage) + logging.debug(f"Rows affected: {cur.rowcount}") + else: + raise Exception( + "Available modes are insert, ignore, upsert, and delsert." + f"You entered {insert_mode}." + ) + logging.debug( + f"Copying data for {partition} took {time.perf_counter() - t} seconds" + ) + def load_items( + self, + file: Union[str, Iterator[Any]] = "stdin", + insert_mode: Optional[Methods] = Methods.insert, + ) -> None: + """Load items json records.""" + if file is None: + file = "stdin" + t = time.perf_counter() + items: List = [] + partitions: dict = {} + for line in read_json(file): + item = self.format_item(line) + items.append(item) + partition = partitions.get( + item["partition"], + { + "partition": None, + "collection": None, + "mindt": None, + "maxdt": None, + "minedt": None, + "maxedt": None, + }, + ) + partition["partition"] = item["partition"] + partition["collection"] = item["collection"] + if partition["mindt"] is None or item["datetime"] < partition["mindt"]: + partition["mindt"] = item["datetime"] + + if partition["maxdt"] is None or item["datetime"] > partition["maxdt"]: + partition["maxdt"] = item["datetime"] + + if ( + partition["minedt"] is None + or item["end_datetime"] < partition["minedt"] + ): + partition["minedt"] = item["end_datetime"] + + if ( + partition["maxedt"] is None + or item["end_datetime"] > partition["maxedt"] + ): + partition["maxedt"] = item["end_datetime"] + partitions[item["partition"]] = partition + logging.debug( + f"Loading and parsing data took {time.perf_counter() - t} seconds." + ) + t = time.perf_counter() + items.sort(key=lambda x: x["partition"]) + logging.debug(f"Sorting data took {time.perf_counter() - t} seconds.") + t = time.perf_counter() -# Types of iterable that load_iterator can support -T = TypeVar("T", Iterable[bytes], Iterable[Dict[str, Any]], Iterable[str]) + for k, g in itertools.groupby(items, lambda x: x["partition"]): + self.load_partition(partitions[k], g, insert_mode) + logging.debug( + f"Adding data to database took {time.perf_counter() - t} seconds." + ) -async def aiter(list: T) -> AsyncGenerator[bytes, None]: - """Async Iterator to convert data to be suitable for pg copy.""" - for item in list: - item_str: str - if isinstance(item, bytes): - item_str = item.decode("utf-8") - elif isinstance(item, dict): - item_str = orjson.dumps(item).decode("utf-8") - elif isinstance(item, str): - item_str = item + def format_item(self, _item: Union[str, dict]) -> dict: + """Format an item to insert into a record.""" + out = {} + item: dict + if not isinstance(_item, dict): + try: + item = orjson.loads(_item.replace("\\\\", "\\")) + except: + raise Exception(f"Could not load {_item}") else: - raise ValueError( - f"Cannot load iterator with values of type {type(item)} (value {item})" - ) - - lines = "\n".join( - [item_str.rstrip().replace(r"\n", r"\\n").replace(r"\t", r"\\t")] - ) - encoded_lines = (lines + "\n").encode("utf-8") + item = _item - yield encoded_lines + base_item, key, partition_trunc = self.collection_json(item["collection"]) + out["id"] = item.pop("id") + out["collection"] = item.pop("collection") + properties: dict = item.get("properties", {}) -async def copy(iter: T, table: tables, conn: asyncpg.Connection) -> None: - """Directly use copy to load data.""" - bytes_iter = aiter(iter) - async with conn.transaction(): - if table == "collections": - await conn.execute( - """ - CREATE TEMP TABLE pgstactemp (content jsonb) - ON COMMIT DROP; - """ - ) - await conn.copy_to_table( - "pgstactemp", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), - ) - await conn.execute( - """ - INSERT INTO collections (content) - SELECT content FROM pgstactemp; - """ - ) - if table == "items": - await conn.copy_to_table( - "items_staging", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), - ) + dt = properties.get("datetime") + edt = properties.get("end_datetime") + sdt = properties.get("start_datetime") + if edt is not None and sdt is not None: + out["datetime"] = sdt + out["end_datetime"] = edt + elif dt is not None: + out["datetime"] = dt + out["end_datetime"] = dt + else: + raise Exception("Invalid datetime encountered") -async def copy_ignore_duplicates( - iter: T, table: tables, conn: asyncpg.Connection -) -> None: - """Load data first into a temp table to ignore duplicates.""" - bytes_iter = aiter(iter) - async with conn.transaction(): - if table == "collections": - await conn.execute( - """ - CREATE TEMP TABLE pgstactemp (content jsonb) - ON COMMIT DROP; - """ - ) - await conn.copy_to_table( - "pgstactemp", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), - ) - await conn.execute( - """ - INSERT INTO collections (content) - SELECT content FROM pgstactemp - ON CONFLICT DO NOTHING; - """ - ) - if table == "items": - await conn.copy_to_table( - "items_staging_ignore", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), + if out["datetime"] is None or out["end_datetime"] is None: + raise Exception( + f"Datetime must be set. OUT: {out} Properties: {properties}" ) + if partition_trunc == "year": + pd = out["datetime"].replace("-", "")[:4] + partition = f"_items_{key}_{pd}" + elif partition_trunc == "month": + pd = out["datetime"].replace("-", "")[:6] + partition = f"_items_{key}_{pd}" + else: + partition = f"_items_{key}" -async def copy_upsert(iter: T, table: tables, conn: asyncpg.Connection) -> None: - """Insert data into a temp table to be able merge data.""" - bytes_iter = aiter(iter) - async with conn.transaction(): - if table == "collections": - await conn.execute( - """ - CREATE TEMP TABLE pgstactemp (content jsonb) - ON COMMIT DROP; - """ - ) - await conn.copy_to_table( - "pgstactemp", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), - ) - await conn.execute( - """ - INSERT INTO collections (content) - SELECT content FROM pgstactemp - ON CONFLICT (id) DO UPDATE - SET content = EXCLUDED.content - WHERE collections.content IS DISTINCT FROM EXCLUDED.content; - """ - ) - if table == "items": - await conn.copy_to_table( - "items_staging_upsert", - source=bytes_iter, - columns=["content"], - format="csv", - quote=chr(27), - delimiter=chr(31), - ) + out["partition"] = partition + bbox = item.pop("bbox") + geojson = item.pop("geometry") + if geojson is None and bbox is not None: + pass + else: + geometry = str(Geometry.from_geojson(geojson).wkb) + out["geometry"] = geometry -async def load_iterator( - iter: T, - table: tables, - conn: asyncpg.Connection, - method: loadopt = loadopt.insert, -): - """Use appropriate method to load data from a file like iterator.""" - if method == loadopt.insert: - await copy(iter, table, conn) - elif method == loadopt.insert_ignore: - await copy_ignore_duplicates(iter, table, conn) - else: - await copy_upsert(iter, table, conn) + content = dict_minus(base_item, item) + out["content"] = orjson.dumps(content).decode() -async def load_ndjson( - file: str, table: tables, method: loadopt = loadopt.insert, dsn: str = None -) -> None: - """Load data from an ndjson file.""" - typer.echo(f"loading {file} into {table} using {method}") - open_file: Any = open(file, "rb") + return out - with open_file as f: - async with DB(dsn) as conn: - await load_iterator(f, table, conn, method) + def __hash__(self) -> int: + """Return hash so that the LRU deocrator can cache without the class.""" + return 0 diff --git a/pypgstac/pypgstac/migrate.py b/pypgstac/pypgstac/migrate.py index 39cfdaa0..b1aa5a60 100644 --- a/pypgstac/pypgstac/migrate.py +++ b/pypgstac/pypgstac/migrate.py @@ -3,12 +3,11 @@ import os from collections import defaultdict from typing import Optional, Dict, List, Iterator, Any - -import asyncpg -import typer from smart_open import open +import logging -from pypgstac import __version__ as version +from .db import PgstacDB +from . import __version__ dirname = os.path.dirname(__file__) migrations_dir = os.path.join(dirname, "migrations") @@ -17,7 +16,7 @@ class MigrationPath: """Calculate path from migration files to get from one version to the next.""" - def __init__(self, path: str, f: str, t: str): + def __init__(self, path: str, f: str, t: str) -> None: """Initialize MigrationPath.""" self.path = path if f is None: @@ -78,7 +77,7 @@ def migrations(self) -> List[str]: path = self.build_path() if path is None: raise Exception( - "Could not determine path to get f %s to %s.", self.f, self.t + f"Could not determine path to get from {self.f} to {self.t}." ) if len(path) == 1: return [f"pgstac.{path[0]}.sql"] @@ -101,89 +100,59 @@ def get_sql(file: str) -> str: return "\n".join(sqlstrs) -def get_initial_version() -> str: - """Get initial version available in migrations.""" - return "0.1.9" +class Migrate: + """Utilities for migrating pgstac database.""" + def __init__(self, db: PgstacDB, schema: str = "pgstac"): + """Prepare for migration.""" + self.db = db + self.schema = schema -async def get_version(conn: asyncpg.Connection) -> str: - """Get the current version number from a pgstac database.""" - async with conn.transaction(): - try: - version = await conn.fetchval( - """ - SELECT version from pgstac.migrations - order by datetime desc, version desc limit 1; - """ - ) - except asyncpg.exceptions.UndefinedTableError: - version = None - return version - - -async def check_pg_version(conn: asyncpg.Connection) -> str: - """Get the current pg version number from a pgstac database.""" - async with conn.transaction(): - version = await conn.fetchval( - """ - SHOW server_version; - """ - ) - if int(version.split(".")[0]) < 13: - raise Exception("PGStac requires PostgreSQL 13+") - return version - - -async def get_version_dsn(dsn: Optional[str] = None) -> str: - """Get current version from a specified database.""" - conn = await asyncpg.connect(dsn=dsn) - version = await get_version(conn) - await conn.close() - return version - - -async def run_migration( - dsn: Optional[str] = None, toversion: Optional[str] = None -) -> str: - """Migrate a pgstac database to current version.""" - if toversion is None: - toversion = version - files = [] - - conn = await asyncpg.connect(dsn=dsn) - pgversion = await check_pg_version(conn) - typer.echo(f"Migrating PGStac on PostgreSQL Version {pgversion}") - oldversion = await get_version(conn) - try: + def run_migration(self, toversion: Optional[str] = None) -> str: + """Migrate a pgstac database to current version.""" + if toversion is None: + toversion = __version__ + files = [] + + pg_version = self.db.pg_version + logging.info(f"Migrating PGStac on PostgreSQL Version {pg_version}") + oldversion = self.db.version if oldversion == toversion: - typer.echo(f"Target database already at version: {toversion}") - await conn.close() + logging.info(f"Target database already at version: {toversion}") return toversion if oldversion is None: - typer.echo( - "**** You can ignore error above that says relation " - "pgstac.migrations does not exist *****" - ) - typer.echo(f"No pgstac version set, installing {toversion} from scratch.") + logging.info(f"No pgstac version set, installing {toversion} from scratch.") files.append(os.path.join(migrations_dir, f"pgstac.{toversion}.sql")) else: - typer.echo(f"Migrating from {oldversion} to {toversion}.") + logging.info(f"Migrating from {oldversion} to {toversion}.") m = MigrationPath(migrations_dir, oldversion, toversion) files = m.migrations() if len(files) < 1: raise Exception("Could not find migration files") - typer.echo(f"Running migrations for {files}.") + conn = self.db.connect() + + with conn.cursor() as cur: + for file in files: + logging.debug(f"Running migration file {file}.") + migration_sql = get_sql(file) + cur.execute(migration_sql) + logging.debug(cur.statusmessage) + logging.debug(cur.rowcount) - for file in files: - migration_sql = get_sql(file) - async with conn.transaction(): - await conn.execute(migration_sql) + logging.debug(f"Database migrated to {toversion}") + + newversion = self.db.version + if conn is not None: + if newversion == toversion: + conn.commit() + else: + conn.rollback() + raise Exception( + "Migration failed, database rolled back to previous state." + ) - newversion = await get_version(conn) - await conn.close() + logging.debug(f"New Version: {newversion}") return newversion - finally: - await conn.close() diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.5-0.5.0.sql b/pypgstac/pypgstac/migrations/pgstac.0.4.5-0.5.0.sql new file mode 100644 index 00000000..b95a20e8 --- /dev/null +++ b/pypgstac/pypgstac/migrations/pgstac.0.4.5-0.5.0.sql @@ -0,0 +1,2669 @@ +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS btree_gist; + +ALTER SCHEMA pgstac rename to pgstac_045; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + CREATE ROLE pgstac_read; + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +GRANT pgstac_admin TO current_user; + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + +SET SEARCH_PATH TO pgstac, public; + + +CREATE TABLE IF NOT EXISTS migrations ( + version text PRIMARY KEY, + datetime timestamptz DEFAULT clock_timestamp() NOT NULL +); + +CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ + SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ + INSERT INTO pgstac.migrations (version) VALUES ($1) + ON CONFLICT DO NOTHING + RETURNING version; +$$ LANGUAGE SQL; + + +CREATE TABLE IF NOT EXISTS pgstac_settings ( + name text PRIMARY KEY, + value text NOT NULL +); + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default-filter-lang', 'cql2-json'), + ('additional_properties', 'true') +ON CONFLICT DO NOTHING +; + + +CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ +SELECT COALESCE( + conf->>_setting, + current_setting(concat('pgstac.',_setting), TRUE), + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) +); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ + SELECT pgstac.get_setting('context', conf); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ + SELECT pgstac.get_setting('context_estimated_count', conf)::int; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_estimated_cost(); +CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ + SELECT pgstac.get_setting('context_estimated_cost', conf)::float; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_stats_ttl(); +CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ + SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ +DECLARE +debug boolean := current_setting('pgstac.debug', true); +BEGIN + IF debug THEN + RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); + RETURN TRUE; + END IF; + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ +SELECT CASE + WHEN $1 IS NULL THEN TRUE + WHEN cardinality($1)<1 THEN TRUE +ELSE FALSE +END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ + SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ +SELECT ARRAY( + SELECT $1[i] + FROM generate_subscripts($1,1) AS s(i) + ORDER BY i DESC +); +$$ LANGUAGE SQL STRICT IMMUTABLE; +CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ + SELECT floor(($1->>0)::float)::int; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ + SELECT ($1->>0)::float; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ + SELECT ($1->>0)::timestamptz; +$$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; + + +CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ + SELECT $1->>0; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ + SELECT + CASE jsonb_typeof($1) + WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) + ELSE ARRAY[$1->>0] + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ +SELECT CASE jsonb_array_length(_bbox) + WHEN 4 THEN + ST_SetSRID(ST_MakeEnvelope( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float, + (_bbox->>3)::float + ),4326) + WHEN 6 THEN + ST_SetSRID(ST_3DMakeBox( + ST_MakePoint( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float + ), + ST_MakePoint( + (_bbox->>3)::float, + (_bbox->>4)::float, + (_bbox->>5)::float + ) + ),4326) + ELSE null END; +; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ + SELECT jsonb_build_array( + st_xmin(_geom), + st_ymin(_geom), + st_xmax(_geom), + st_ymax(_geom) + ); +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ + SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +/* looks for a geometry in a stac item first from geometry and falling back to bbox */ +CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ +SELECT + CASE + WHEN value ? 'intersects' THEN + ST_GeomFromGeoJSON(value->>'intersects') + WHEN value ? 'geometry' THEN + ST_GeomFromGeoJSON(value->>'geometry') + WHEN value ? 'bbox' THEN + pgstac.bbox_geom(value->'bbox') + ELSE NULL + END as geometry +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION stac_daterange( + value jsonb +) RETURNS tstzrange AS $$ +DECLARE + props jsonb := value; + dt timestamptz; + edt timestamptz; +BEGIN + IF props ? 'properties' THEN + props := props->'properties'; + END IF; + IF props ? 'start_datetime' AND props ? 'end_datetime' THEN + dt := props->'start_datetime'; + edt := props->'end_datetime'; + IF dt > edt THEN + RAISE EXCEPTION 'start_datetime must be < end_datetime'; + END IF; + ELSE + dt := props->'datetime'; + edt := props->'datetime'; + END IF; + IF dt is NULL OR edt IS NULL THEN + RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; + END IF; + RETURN tstzrange(dt, edt, '[]'); +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT lower(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT upper(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + + +CREATE TABLE IF NOT EXISTS stac_extensions( + name text PRIMARY KEY, + url text, + enbabled_by_default boolean NOT NULL DEFAULT TRUE, + enableable boolean NOT NULL DEFAULT TRUE +); + +INSERT INTO stac_extensions (name, url) VALUES + ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), + ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), + ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), + ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), + ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') +ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; + + + +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id', + 'links', '[]'::jsonb + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); + +CREATE TABLE IF NOT EXISTS collections ( + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + partition_trunc partition_trunc_strategy +); + + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + SELECT relid::text INTO partition_name + FROM pg_partition_tree('items') + WHERE relid::text = partition_name; + IF FOUND THEN + partition_exists := true; + partition_empty := table_empty(partition_name); + ELSE + partition_exists := false; + partition_empty := true; + partition_name := format('_items_%s', NEW.key); + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN + q := format($q$ + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name + ); + EXECUTE q; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN + q := format($q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name, + partition_name + ); + EXECUTE q; + loadtemp := TRUE; + partition_empty := TRUE; + partition_exists := FALSE; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN + RETURN NEW; + END IF; + IF NEW.partition_trunc IS NULL AND partition_empty THEN + RAISE NOTICE '% % % %', + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); + ELSIF partition_empty THEN + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) + PARTITION BY RANGE (datetime); + $q$, + partition_name, + NEW.id + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + ELSE + RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; + END IF; + IF loadtemp THEN + RAISE NOTICE 'Moving data into new partitions.'; + q := format($q$ + WITH p AS ( + SELECT + collection, + datetime as datetime, + end_datetime as end_datetime, + (partition_name( + collection, + datetime + )).partition_name as name + FROM changepartitionstaging + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + INSERT INTO %I SELECT * FROM changepartitionstaging; + DROP TABLE IF EXISTS changepartitionstaging; + $q$, + partition_name + ); + EXECUTE q; + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + +CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW +EXECUTE FUNCTION collections_trigger_func(); + +CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ + UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; +$$ LANGUAGE SQL; + +CREATE TABLE IF NOT EXISTS partitions ( + collection text REFERENCES collections(id), + name text PRIMARY KEY, + partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), + datetime_range tstzrange, + end_datetime_range tstzrange, + CONSTRAINT prange EXCLUDE USING GIST ( + collection WITH =, + partition_range WITH && + ) +) WITH (FILLFACTOR=90); +CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); + + + +CREATE OR REPLACE FUNCTION partition_name( + IN collection text, + IN dt timestamptz, + OUT partition_name text, + OUT partition_range tstzrange +) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + + +CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + cq text; + parent_name text; + partition_trunc text; + partition_name text := NEW.name; + partition_exists boolean := false; + partition_empty boolean := true; + partition_range tstzrange; + datetime_range tstzrange; + end_datetime_range tstzrange; + err_context text; + mindt timestamptz := lower(NEW.datetime_range); + maxdt timestamptz := upper(NEW.datetime_range); + minedt timestamptz := lower(NEW.end_datetime_range); + maxedt timestamptz := upper(NEW.end_datetime_range); + t_mindt timestamptz; + t_maxdt timestamptz; + t_minedt timestamptz; + t_maxedt timestamptz; +BEGIN + RAISE NOTICE 'Partitions Trigger. %', NEW; + datetime_range := NEW.datetime_range; + end_datetime_range := NEW.end_datetime_range; + + SELECT + format('_items_%s', key), + c.partition_trunc::text + INTO + parent_name, + partition_trunc + FROM pgstac.collections c + WHERE c.id = NEW.collection; + SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; + NEW.name := partition_name; + + IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + + NEW.partition_range := partition_range; + IF TG_OP = 'UPDATE' THEN + mindt := least(mindt, lower(OLD.datetime_range)); + maxdt := greatest(maxdt, upper(OLD.datetime_range)); + minedt := least(minedt, lower(OLD.end_datetime_range)); + maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + END IF; + IF TG_OP = 'INSERT' THEN + + IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN + + RAISE NOTICE '% % %', partition_name, parent_name, partition_range; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + parent_name, + lower(partition_range), + upper(partition_range), + format('%s_pkey', partition_name), + partition_name + ); + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + END IF; + + END IF; + + -- Update constraints + EXECUTE format($q$ + SELECT + min(datetime), + max(datetime), + min(end_datetime), + max(end_datetime) + FROM %I; + $q$, partition_name) + INTO t_mindt, t_maxdt, t_minedt, t_maxedt; + mindt := least(mindt, t_mindt); + maxdt := greatest(maxdt, t_maxdt); + minedt := least(mindt, minedt, t_minedt); + maxedt := greatest(maxdt, maxedt, t_maxedt); + + mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); + maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); + maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + + + IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + IF + TG_OP='UPDATE' + AND OLD.datetime_range @> NEW.datetime_range + AND OLD.end_datetime_range @> NEW.end_datetime_range + THEN + RAISE NOTICE 'Range unchanged, not updating constraints.'; + ELSE + + RAISE NOTICE ' + SETTING CONSTRAINTS + mindt: %, maxdt: % + minedt: %, maxedt: % + ', mindt, maxdt, minedt, maxedt; + IF partition_trunc IS NULL THEN + cq := format($q$ + ALTER TABLE %7$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ( + (datetime >= %3$L) + AND (datetime <= %4$L) + AND (end_datetime >= %5$L) + AND (end_datetime <= %6$L) + ) NOT VALID + ; + ALTER TABLE %7$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + mindt, + maxdt, + minedt, + maxedt, + partition_name + ); + ELSE + cq := format($q$ + ALTER TABLE %5$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %2$I + CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID + ; + ALTER TABLE %5$I + VALIDATE CONSTRAINT %2$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + minedt, + maxedt, + partition_name + ); + + END IF; + RAISE NOTICE 'Altering Constraints. %', cq; + EXECUTE cq; + END IF; + ELSE + NEW.datetime_range = NULL; + NEW.end_datetime_range = NULL; + + cq := format($q$ + ALTER TABLE %3$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID + ; + ALTER TABLE %3$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + partition_name + ); + EXECUTE cq; + END IF; + + RETURN NEW; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW +EXECUTE FUNCTION partitions_trigger_func(); + + +CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ON CONFLICT (id) DO + UPDATE + SET content=EXCLUDED.content + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ + SELECT content FROM collections + WHERE id=$1 + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ + SELECT jsonb_agg(content) FROM collections; +; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; +CREATE TABLE queryables ( + id bigint GENERATED ALWAYS AS identity PRIMARY KEY, + name text UNIQUE NOT NULL, + collection_ids text[], -- used to determine what partitions to create indexes on + definition jsonb, + property_path text, + property_wrapper text, + property_index_type text +); +CREATE INDEX queryables_name_idx ON queryables (name); +CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); + + +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') +ON CONFLICT DO NOTHING; + + + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + +CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ + SELECT string_agg( + quote_literal(v), + '->' + ) FROM unnest(arr) v; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + + + +CREATE OR REPLACE FUNCTION queryable( + IN dotpath text, + OUT path text, + OUT expression text, + OUT wrapper text +) AS $$ +DECLARE + q RECORD; + path_elements text[]; +BEGIN + IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN + path := dotpath; + expression := dotpath; + wrapper := NULL; + RETURN; + END IF; + SELECT * INTO q FROM queryables WHERE name=dotpath; + IF q.property_wrapper IS NULL THEN + IF q.definition->>'type' = 'number' THEN + wrapper := 'to_float'; + ELSIF q.definition->>'format' = 'date-time' THEN + wrapper := 'to_tstz'; + ELSE + wrapper := 'to_text'; + END IF; + ELSE + wrapper := q.property_wrapper; + END IF; + IF q.property_path IS NOT NULL THEN + path := q.property_path; + ELSE + path_elements := string_to_array(dotpath, '.'); + IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN + path := format('content->%s', array_to_path(path_elements)); + ELSIF path_elements[1] = 'properties' THEN + path := format('content->%s', array_to_path(path_elements)); + ELSE + path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); + END IF; + END IF; + expression := format('%I(%s)', wrapper, path); + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ +DECLARE + queryable RECORD; + q text; +BEGIN + FOR queryable IN + SELECT + queryables.id as qid, + CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, + property_index_type, + expression + FROM + queryables + LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) + JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) + LOOP + q := format( + $q$ + CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); + $q$, + format('%s_%s_idx', queryable.part, queryable.qid), + queryable.part, + COALESCE(queryable.property_index_type, 'to_text'), + queryable.expression + ); + RAISE NOTICE '%',q; + EXECUTE q; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ +DECLARE +BEGIN +PERFORM create_queryable_indexes(); +RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); + +CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate jsonb, + relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) +) RETURNS tstzrange AS $$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate text, + relative_base timestamptz DEFAULT CURRENT_TIMESTAMP +) RETURNS tstzrange AS $$ + SELECT parse_dtrange(to_jsonb(_indate), relative_base); +$$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + ll text := 'datetime'; + lh text := 'end_datetime'; + rrange tstzrange; + rl text; + rh text; + outq text; +BEGIN + rrange := parse_dtrange(args->1); + RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; + op := lower(op); + rl := format('%L::timestamptz', lower(rrange)); + rh := format('%L::timestamptz', upper(rrange)); + outq := CASE op + WHEN 't_before' THEN 'lh < rl' + WHEN 't_after' THEN 'll > rh' + WHEN 't_meets' THEN 'lh = rl' + WHEN 't_metby' THEN 'll = rh' + WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' + WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' + WHEN 't_starts' THEN 'll = rl AND lh < rh' + WHEN 't_startedby' THEN 'll = rl AND lh > rh' + WHEN 't_during' THEN 'll > rl AND lh < rh' + WHEN 't_contains' THEN 'll < rl AND lh > rh' + WHEN 't_finishes' THEN 'll > rl AND lh = rh' + WHEN 't_finishedby' THEN 'll < rl AND lh = rh' + WHEN 't_equals' THEN 'll = rl AND lh = rh' + WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' + WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' + WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' + END; + outq := regexp_replace(outq, '\mll\M', ll); + outq := regexp_replace(outq, '\mlh\M', lh); + outq := regexp_replace(outq, '\mrl\M', rl); + outq := regexp_replace(outq, '\mrh\M', rh); + outq := format('(%s)', outq); + RETURN outq; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + + +CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + geom text; + j jsonb := args->1; +BEGIN + op := lower(op); + RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; + IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN + RAISE EXCEPTION 'Spatial Operator % Not Supported', op; + END IF; + op := regexp_replace(op, '^s_', 'st_'); + IF op = 'intersects' THEN + op := 'st_intersects'; + END IF; + -- Convert geometry to WKB string + IF j ? 'type' AND j ? 'coordinates' THEN + geom := st_geomfromgeojson(j)::text; + ELSIF jsonb_typeof(j) = 'array' THEN + geom := bbox_geom(j)::text; + END IF; + + RETURN format('%s(geometry, %L::geometry)', op, geom); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ +-- Translates anything passed in through the deprecated "query" into equivalent CQL2 +WITH t AS ( + SELECT key as property, value as ops + FROM jsonb_each(q) +), t2 AS ( + SELECT property, (jsonb_each(ops)).* + FROM t WHERE jsonb_typeof(ops) = 'object' + UNION ALL + SELECT property, 'eq', ops + FROM t WHERE jsonb_typeof(ops) != 'object' +) +SELECT + jsonb_strip_nulls(jsonb_build_object( + 'op', 'and', + 'args', jsonb_agg( + jsonb_build_object( + 'op', key, + 'args', jsonb_build_array( + jsonb_build_object('property',property), + value + ) + ) + ) + ) +) as qcql FROM t2 +; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ +DECLARE + args jsonb; + ret jsonb; +BEGIN + RAISE NOTICE 'CQL1_TO_CQL2: %', j; + IF j ? 'filter' THEN + RETURN cql1_to_cql2(j->'filter'); + END IF; + IF j ? 'property' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'array' THEN + SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; + RETURN args; + END IF; + IF jsonb_typeof(j) = 'number' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'string' THEN + RETURN j; + END IF; + + IF jsonb_typeof(j) = 'object' THEN + SELECT jsonb_build_object( + 'op', key, + 'args', cql1_to_cql2(value) + ) INTO ret + FROM jsonb_each(j) + WHERE j IS NOT NULL; + RETURN ret; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT; + +CREATE TABLE cql2_ops ( + op text PRIMARY KEY, + template text, + types text[] +); +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('in', '%s = ANY (%s)', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), + ('isnull', '%s IS NULL', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template; +; + + +CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ +#variable_conflict use_variable +DECLARE + args jsonb := j->'args'; + arg jsonb; + op text := lower(j->>'op'); + cql2op RECORD; + literal text; +BEGIN + IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'CQL2_QUERY: %', j; + IF j ? 'filter' THEN + RETURN cql2_query(j->'filter'); + END IF; + + IF j ? 'upper' THEN + RETURN format('upper(%s)', cql2_query(j->'upper')); + END IF; + + IF j ? 'lower' THEN + RETURN format('lower(%s)', cql2_query(j->'lower')); + END IF; + + -- Temporal Query + IF op ilike 't_%' or op = 'anyinteracts' THEN + RETURN temporal_op_query(op, args); + END IF; + + -- If property is a timestamp convert it to text to use with + -- general operators + IF j ? 'timestamp' THEN + RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); + END IF; + IF j ? 'interval' THEN + RAISE EXCEPTION 'Please use temporal operators when using intervals.'; + RETURN NONE; + END IF; + + -- Spatial Query + IF op ilike 's_%' or op = 'intersects' THEN + RETURN spatial_op_query(op, args); + END IF; + + + IF op = 'in' THEN + RETURN format( + '%s = ANY (%L)', + cql2_query(args->0), + to_text_array(args->1) + ); + END IF; + + + + IF op = 'between' THEN + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + RETURN format( + '%s BETWEEN %s and %s', + cql2_query(args->0, wrapper), + cql2_query(args->1->0, wrapper), + cql2_query(args->1->1, wrapper) + ); + END IF; + + -- Make sure that args is an array and run cql2_query on + -- each element of the array + RAISE NOTICE 'ARGS PRE: %', args; + IF j ? 'args' THEN + IF jsonb_typeof(args) != 'array' THEN + args := jsonb_build_array(args); + END IF; + + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + SELECT jsonb_agg(cql2_query(a, wrapper)) + INTO args + FROM jsonb_array_elements(args) a; + END IF; + RAISE NOTICE 'ARGS: %', args; + + IF op IN ('and', 'or') THEN + RETURN + format( + '(%s)', + array_to_string(to_text_array(args), format(' %s ', upper(op))) + ); + END IF; + + -- Look up template from cql2_ops + IF j ? 'op' THEN + SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; + IF FOUND THEN + -- If specific index set in queryables for a property cast other arguments to that type + RETURN format( + cql2op.template, + VARIADIC (to_text_array(args)) + ); + ELSE + RAISE EXCEPTION 'Operator % Not Supported.', op; + END IF; + END IF; + + + IF j ? 'property' THEN + RETURN (queryable(j->>'property')).expression; + END IF; + + IF wrapper IS NOT NULL THEN + EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; + RAISE NOTICE '% % %',wrapper, j, literal; + RETURN format('%I(%L)', wrapper, j); + END IF; + + RETURN quote_literal(to_text(j)); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION paging_dtrange( + j jsonb +) RETURNS tstzrange AS $$ +DECLARE + op text; + filter jsonb := j->'filter'; + dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); + sdate timestamptz := '-infinity'::timestamptz; + edate timestamptz := 'infinity'::timestamptz; + jpitem jsonb; +BEGIN + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP + op := lower(jpitem->>'op'); + dtrange := parse_dtrange(jpitem->'args'->1); + IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN + sdate := greatest(sdate,'-infinity'); + edate := least(edate, upper(dtrange)); + ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN + edate := least(edate, 'infinity'); + sdate := greatest(sdate, lower(dtrange)); + ELSIF op IN ('=', 'eq') THEN + edate := least(edate, upper(dtrange)); + sdate := greatest(sdate, lower(dtrange)); + END IF; + RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; + END LOOP; + END IF; + IF sdate > edate THEN + RETURN 'empty'::tstzrange; + END IF; + RETURN tstzrange(sdate,edate, '[]'); +END; +$$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION paging_collections( + IN j jsonb +) RETURNS text[] AS $$ +DECLARE + filter jsonb := j->'filter'; + jpitem jsonb; + op text; + args jsonb; + arg jsonb; + collections text[]; +BEGIN + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP + RAISE NOTICE 'JPITEM: %', jpitem; + op := jpitem->>'op'; + args := jpitem->'args'; + IF op IN ('=', 'eq', 'in') THEN + FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP + IF jsonb_typeof(arg) IN ('string', 'array') THEN + RAISE NOTICE 'arg: %, collections: %', arg, collections; + IF collections IS NULL OR collections = '{}'::text[] THEN + collections := to_text_array(arg); + ELSE + collections := array_intersection(collections, to_text_array(arg)); + END IF; + END IF; + END LOOP; + END IF; + END LOOP; + END IF; + IF collections = '{}'::text[] THEN + RETURN NULL; + END IF; + RETURN collections; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; +CREATE TABLE items ( + id text NOT NULL, + geometry geometry NOT NULL, + collection text NOT NULL, + datetime timestamptz NOT NULL, + end_datetime timestamptz NOT NULL, + content JSONB NOT NULL +) +PARTITION BY LIST (collection) +; + +CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); +CREATE INDEX "geometry_idx" ON items USING GIST (geometry); + +CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; + + +ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; + + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ + SELECT + jsonb_object_agg( + key, + CASE + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_slim(i.value, c.value) + ELSE i.value + END + ) + FROM + jsonb_each(_item) as i + LEFT JOIN + jsonb_each(_collection) as c + USING (key) + WHERE + i.value IS DISTINCT FROM c.value + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ + SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ + SELECT + content->>'id' as id, + stac_geom(content) as geometry, + content->>'collection' as collection, + stac_datetime(content) as datetime, + stac_end_datetime(content) as end_datetime, + content_slim(content) as content + ; +$$ LANGUAGE SQL STABLE; + + +CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ +DECLARE + includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); + excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); +BEGIN + IF f IS NULL THEN + RETURN NULL; + ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN + RETURN TRUE; + ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN + RETURN FALSE; + ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN + RETURN FALSE; + END IF; + RETURN TRUE; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ +DECLARE + includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); + excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); +BEGIN + RAISE NOTICE '% % %', k, val, kf; + + include := TRUE; + IF k = 'properties' AND NOT excludes ? 'properties' THEN + excludes := excludes || '["properties"]'; + include := TRUE; + RAISE NOTICE 'Prop include %', include; + ELSIF + jsonb_array_length(excludes)>0 AND excludes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND NOT includes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND includes ? k THEN + includes := '[]'::jsonb; + RAISE NOTICE 'KF: %', kf; + END IF; + kf := jsonb_build_object('includes', includes, 'excludes', excludes); + RETURN; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ + WITH t AS (SELECT * FROM jsonb_each(a)) + SELECT jsonb_object_agg(key, value) FROM t + WHERE value ? 'href'; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION content_hydrate( + _item jsonb, + _collection jsonb, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ + SELECT + jsonb_strip_nulls(jsonb_object_agg( + key, + CASE + WHEN key = 'properties' AND include_field('properties', fields) THEN + i.value + WHEN key = 'properties' THEN + content_hydrate(i.value, c.value, kf) + WHEN + c.value IS NULL AND key != 'properties' + THEN i.value + WHEN + key = 'assets' + AND + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN strip_assets(content_hydrate(i.value, c.value, kf)) + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_hydrate(i.value, c.value, kf) + ELSE coalesce(i.value, c.value) + END + )) + FROM + jsonb_each(coalesce(_item,'{}'::jsonb)) as i + FULL JOIN + jsonb_each(coalesce(_collection,'{}'::jsonb)) as c + USING (key) + JOIN LATERAL ( + SELECT kf, include FROM key_filter(key, i.value, fields) + ) as k ON (include) + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; + content jsonb; + base_item jsonb := _collection.base_item; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry)::jsonb; + END IF; + IF include_field('bbox', fields) THEN + bbox := geom_bbox(_item.geometry)::jsonb; + END IF; + output := content_hydrate( + jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'bbox',bbox, + 'collection', _item.collection + ) || _item.content, + _collection.base_item, + fields + ); + + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ + SELECT content_hydrate( + _item, + (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), + fields + ); +$$ LANGUAGE SQL STABLE; + + +CREATE UNLOGGED TABLE items_staging ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_ignore ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_upsert ( + content JSONB NOT NULL +); + +CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p record; + _partitions text[]; + ts timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + WITH ranges AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr + FROM newdata n + ), p AS ( + SELECT + collection, + lower(dtr) as datetime, + upper(dtr) as end_datetime, + (partition_name( + collection, + lower(dtr) + )).partition_name as name + FROM ranges + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata; + DELETE FROM items_staging; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata + ON CONFLICT DO NOTHING; + DELETE FROM items_staging_ignore; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + WITH staging_formatted AS ( + SELECT (content_dehydrate(content)).* FROM newdata + ), deletes AS ( + DELETE FROM items i USING staging_formatted s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO items + SELECT s.* FROM + staging_formatted s + JOIN deletes d + USING (id, collection); + DELETE FROM items_staging_upsert; + END IF; + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + + + + +CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS +$$ +DECLARE + i items%ROWTYPE; +BEGIN + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; + RETURN i; +END; +$$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE +out items%ROWTYPE; +BEGIN + DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL STABLE; + +--/* +CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ +DECLARE + old items %ROWTYPE; + out items%ROWTYPE; +BEGIN + SELECT delete_item(content->>'id', content->>'collection'); + SELECT create_item(content); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ + SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb + FROM items WHERE collection=$1; + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ + SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) + FROM items WHERE collection=$1; +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ +UPDATE collections SET + content = content || + jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', collection_bbox(collections.id) + ), + 'temporal', jsonb_build_object( + 'interval', collection_temporal_extent(collections.id) + ) + ) + ) +; +$$ LANGUAGE SQL; +CREATE VIEW partition_steps AS +SELECT + name, + date_trunc('month',lower(datetime_range)) as sdate, + date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate + FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange + ORDER BY datetime_range ASC +; + +CREATE OR REPLACE FUNCTION chunker( + IN _where text, + OUT s timestamptz, + OUT e timestamptz +) RETURNS SETOF RECORD AS $$ +DECLARE + explain jsonb; +BEGIN + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; + + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION partition_queries( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL +) RETURNS SETOF text AS $$ +DECLARE + query text; + sdate timestamptz; + edate timestamptz; +BEGIN +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + $q$, _where, _orderby + ); + + RETURN NEXT query; + RETURN; +END IF; + +RETURN; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION partition_query_view( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN _limit int DEFAULT 10 +) RETURNS text AS $$ + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total LIMIT %s + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ), + _limit + )) + ELSE NULL + END FROM p; +$$ LANGUAGE SQL IMMUTABLE; + + + + +CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ +DECLARE + where_segments text[]; + _where text; + dtrange tstzrange; + collections text[]; + geom geometry; + sdate timestamptz; + edate timestamptz; + filterlang text; + filter jsonb := j->'filter'; +BEGIN + IF j ? 'ids' THEN + where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); + END IF; + + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + where_segments := where_segments || format('collection = ANY (%L) ', collections); + END IF; + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + + where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', + edate, + sdate + ); + END IF; + + geom := stac_geom(j); + IF geom IS NOT NULL THEN + where_segments := where_segments || format('st_intersects(geometry, %L)',geom); + END IF; + + filterlang := COALESCE( + j->>'filter-lang', + get_setting('default-filter-lang', j->'conf') + ); + IF NOT filter @? '$.**.op' THEN + filterlang := 'cql-json'; + END IF; + + IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN + RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; + END IF; + + IF j ? 'query' AND j ? 'filter' THEN + RAISE EXCEPTION 'Can only use either query or filter at one time.'; + END IF; + + IF j ? 'query' THEN + filter := query_to_cql2(j->'query'); + ELSIF filterlang = 'cql-json' THEN + filter := cql1_to_cql2(filter); + END IF; + RAISE NOTICE 'FILTER: %', filter; + where_segments := where_segments || cql2_query(filter); + IF cardinality(where_segments) < 1 THEN + RETURN ' TRUE '; + END IF; + + _where := array_to_string(array_remove(where_segments, NULL), ' AND '); + + IF _where IS NULL OR BTRIM(_where) = '' THEN + RETURN ' TRUE '; + END IF; + RETURN _where; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN NOT reverse THEN d + WHEN d = 'ASC' THEN 'DESC' + WHEN d = 'DESC' THEN 'ASC' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN d = 'ASC' AND prev THEN '<=' + WHEN d = 'DESC' AND prev THEN '>=' + WHEN d = 'ASC' THEN '>=' + WHEN d = 'DESC' THEN '<=' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION sort_sqlorderby( + _search jsonb DEFAULT NULL, + reverse boolean DEFAULT FALSE +) RETURNS text AS $$ + WITH sortby AS ( + SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort + ), withid AS ( + SELECT CASE + WHEN sort @? '$[*] ? (@.field == "id")' THEN sort + ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb + END as sort + FROM sortby + ), withid_rows AS ( + SELECT jsonb_array_elements(sort) as value FROM withid + ),sorts AS ( + SELECT + coalesce( + -- field_orderby((items_path(value->>'field')).path_txt), + (queryable(value->>'field')).expression + ) as key, + parse_sort_dir(value->>'direction', reverse) as dir + FROM withid_rows + ) + SELECT array_to_string( + array_agg(concat(key, ' ', dir)), + ', ' + ) FROM sorts; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ + SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ +DECLARE + token_id text; + filters text[] := '{}'::text[]; + prev boolean := TRUE; + field text; + dir text; + sort record; + orfilters text[] := '{}'::text[]; + andfilters text[] := '{}'::text[]; + output text; + token_where text; +BEGIN + RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; + -- If no token provided return NULL + IF token_rec IS NULL THEN + IF NOT (_search ? 'token' AND + ( + (_search->>'token' ILIKE 'prev:%') + OR + (_search->>'token' ILIKE 'next:%') + ) + ) THEN + RETURN NULL; + END IF; + prev := (_search->>'token' ILIKE 'prev:%'); + token_id := substr(_search->>'token', 6); + SELECT to_jsonb(items) INTO token_rec + FROM items WHERE id=token_id; + END IF; + RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; + + CREATE TEMP TABLE sorts ( + _row int GENERATED ALWAYS AS IDENTITY NOT NULL, + _field text PRIMARY KEY, + _dir text NOT NULL, + _val text + ) ON COMMIT DROP; + + -- Make sure we only have distinct columns to sort with taking the first one we get + INSERT INTO sorts (_field, _dir) + SELECT + (queryable(value->>'field')).expression, + get_sort_dir(value) + FROM + jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) + ON CONFLICT DO NOTHING + ; + RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + -- Get the first sort direction provided. As the id is a primary key, if there are any + -- sorts after id they won't do anything, so make sure that id is the last sort item. + SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; + IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN + DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); + ELSE + INSERT INTO sorts (_field, _dir) VALUES ('id', dir); + END IF; + + -- Add value from looked up item to the sorts table + UPDATE sorts SET _val=quote_literal(token_rec->>_field); + + -- Check if all sorts are the same direction and use row comparison + -- to filter + RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + + IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN + SELECT format( + '(%s) %s (%s)', + concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), + CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, + concat_ws(', ', VARIADIC array_agg(_val)) + ) INTO output FROM sorts + WHERE token_rec ? _field + ; + ELSE + FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP + RAISE NOTICE 'SORT: %', sort; + IF sort._row = 1 THEN + orfilters := orfilters || format('(%s %s %s)', + quote_ident(sort._field), + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + ELSE + orfilters := orfilters || format('(%s AND %s %s %s)', + array_to_string(andfilters, ' AND '), + quote_ident(sort._field), + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + + END IF; + andfilters := andfilters || format('%s = %s', + quote_ident(sort._field), + sort._val + ); + END LOOP; + output := array_to_string(orfilters, ' OR '); + END IF; + DROP TABLE IF EXISTS sorts; + token_where := concat('(',coalesce(output,'true'),')'); + IF trim(token_where) = '' THEN + token_where := NULL; + END IF; + RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; + RETURN token_where; + END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ + SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ + SELECT md5(concat(search_tohash($1)::text,$2::text)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS searches( + hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, + search jsonb NOT NULL, + _where text, + orderby text, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); +CREATE TABLE IF NOT EXISTS search_wheres( + id bigint generated always as identity primary key, + _where text NOT NULL, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + statslastupdated timestamptz, + estimated_count bigint, + estimated_cost float, + time_to_estimate float, + total_count bigint, + time_to_count float, + partitions text[] +); + +CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); +CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); + +CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + partitions text[]; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost float := context_estimated_cost(conf); + _estimated_count int := context_estimated_count(conf); +BEGIN + IF _context = 'off' THEN + sw._where := inwhere; + return sw; + END IF; + + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + + -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired + IF NOT updatestats THEN + RAISE NOTICE 'Checking if update is needed for: % .', inwhere; + RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; + RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; + RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; + IF + sw.statslastupdated IS NULL + OR (now() - sw.statslastupdated) > _stats_ttl + OR (context(conf) != 'off' AND sw.total_count IS NULL) + THEN + updatestats := TRUE; + END IF; + END IF; + + sw._where := inwhere; + sw.lastused := now(); + sw.usecount := coalesce(sw.usecount,0) + 1; + + IF NOT updatestats THEN + UPDATE search_wheres SET + lastused = sw.lastused, + usecount = sw.usecount + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw + ; + RETURN sw; + END IF; + + -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.statslastupdated := now(); + sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; + sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.time_to_estimate := extract(epoch from i); + + RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; + RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; + + -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough + IF + _context = 'on' + OR + ( _context = 'auto' AND + ( + sw.estimated_count < _estimated_count + AND + sw.estimated_cost < _estimated_cost + ) + ) + THEN + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + ELSE + sw.total_count := NULL; + sw.time_to_count := NULL; + END IF; + + + INSERT INTO search_wheres + (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) + SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count + ON CONFLICT ((md5(_where))) + DO UPDATE + SET + lastused = sw.lastused, + usecount = sw.usecount, + statslastupdated = sw.statslastupdated, + estimated_count = sw.estimated_count, + estimated_cost = sw.estimated_cost, + time_to_estimate = sw.time_to_estimate, + total_count = sw.total_count, + time_to_count = sw.time_to_count + ; + RETURN sw; +END; +$$ LANGUAGE PLPGSQL ; + + + +DROP FUNCTION IF EXISTS search_query; +CREATE OR REPLACE FUNCTION search_query( + _search jsonb = '{}'::jsonb, + updatestats boolean = false, + _metadata jsonb = '{}'::jsonb +) RETURNS searches AS $$ +DECLARE + search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; +BEGIN + SELECT * INTO search FROM searches + WHERE hash=search_hash(_search, _metadata) FOR UPDATE; + + -- Calculate the where clause if not already calculated + IF search._where IS NULL THEN + search._where := stac_search_to_where(_search); + END IF; + + -- Calculate the order by clause if not already calculated + IF search.orderby IS NULL THEN + search.orderby := sort_sqlorderby(_search); + END IF; + + PERFORM where_stats(search._where, updatestats, _search->'conf'); + + search.lastused := now(); + search.usecount := coalesce(search.usecount, 0) + 1; + INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) + VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) + ON CONFLICT (hash) DO + UPDATE SET + _where = EXCLUDED._where, + orderby = EXCLUDED.orderby, + lastused = EXCLUDED.lastused, + usecount = EXCLUDED.usecount, + metadata = EXCLUDED.metadata + RETURNING * INTO search + ; + RETURN search; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + searches searches%ROWTYPE; + _where text; + token_where text; + full_where text; + orderby text; + query text; + token_type text := substr(_search->>'token',1,4); + _limit int := coalesce((_search->>'limit')::int, 10); + curs refcursor; + cntr int := 0; + iter_record items%ROWTYPE; + first_record jsonb; + last_record jsonb; + out_records jsonb := '[]'::jsonb; + prev_query text; + next text; + prev_id text; + has_next boolean := false; + has_prev boolean := false; + prev text; + total_count bigint; + context jsonb; + collection jsonb; + includes text[]; + excludes text[]; + exit_flag boolean := FALSE; + batches int := 0; + timer timestamptz := clock_timestamp(); + pstart timestamptz; + pend timestamptz; + pcurs refcursor; + search_where search_wheres%ROWTYPE; + id text; +BEGIN +CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; +-- if ids is set, short circuit and just use direct ids query for each id +-- skip any paging or caching +-- hard codes ordering in the same order as the array of ids +IF _search ? 'ids' THEN + INSERT INTO results + SELECT content_hydrate(items, _search->'fields') + FROM items WHERE + items.id = ANY(to_text_array(_search->'ids')) + AND + CASE WHEN _search ? 'collections' THEN + items.collection = ANY(to_text_array(_search->'collections')) + ELSE TRUE + END + ORDER BY items.datetime desc, items.id desc + ; + SELECT INTO total_count count(*) FROM results; +ELSE + searches := search_query(_search); + _where := searches._where; + orderby := searches.orderby; + search_where := where_stats(_where); + total_count := coalesce(search_where.total_count, search_where.estimated_count); + + IF token_type='prev' THEN + token_where := get_token_filter(_search, null::jsonb); + orderby := sort_sqlorderby(_search, TRUE); + END IF; + IF token_type='next' THEN + token_where := get_token_filter(_search, null::jsonb); + END IF; + + full_where := concat_ws(' AND ', _where, token_where); + RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; + timer := clock_timestamp(); + + FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP + timer := clock_timestamp(); + query := format('%s LIMIT %s', query, _limit + 1); + RAISE NOTICE 'Partition Query: %', query; + batches := batches + 1; + -- curs = create_cursor(query); + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs into iter_record; + EXIT WHEN NOT FOUND; + cntr := cntr + 1; + last_record := content_hydrate(iter_record, _search->'fields'); + IF cntr = 1 THEN + first_record := last_record; + END IF; + IF cntr <= _limit THEN + INSERT INTO results (content) VALUES (last_record); + ELSIF cntr > _limit THEN + has_next := true; + exit_flag := true; + EXIT; + END IF; + END LOOP; + CLOSE curs; + RAISE NOTICE 'Query took %.', clock_timestamp()-timer; + timer := clock_timestamp(); + EXIT WHEN exit_flag; + END LOOP; + RAISE NOTICE 'Scanned through % partitions.', batches; +END IF; + +SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; + +DROP TABLE results; + + +-- Flip things around if this was the result of a prev token query +IF token_type='prev' THEN + out_records := flip_jsonb_array(out_records); + first_record := last_record; +END IF; + +-- If this query has a token, see if there is data before the first record +IF _search ? 'token' THEN + prev_query := format( + 'SELECT 1 FROM items WHERE %s LIMIT 1', + concat_ws( + ' AND ', + _where, + trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) + ) + ); + RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; + EXECUTE prev_query INTO has_prev; + IF FOUND and has_prev IS NOT NULL THEN + RAISE NOTICE 'Query results from prev query: %', has_prev; + has_prev := TRUE; + END IF; +END IF; +has_prev := COALESCE(has_prev, FALSE); + +IF has_prev THEN + prev := out_records->0->>'id'; +END IF; +IF has_next OR token_type='prev' THEN + next := out_records->-1->>'id'; +END IF; + +IF context(_search->'conf') != 'off' THEN + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'matched', total_count, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +ELSE + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +END IF; + +collection := jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb), + 'next', next, + 'prev', prev, + 'context', context +); + +RETURN collection; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + +CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ +DECLARE + curs refcursor; + searches searches%ROWTYPE; + _where text; + _orderby text; + q text; + +BEGIN + searches := search_query(_search); + _where := searches._where; + _orderby := searches.orderby; + + OPEN curs FOR + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ) + )) + ELSE NULL + END FROM p; + RETURN curs; +END; +$$ LANGUAGE PLPGSQL; +SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ +WITH t AS ( + SELECT + 20037508.3427892 as merc_max, + -20037508.3427892 as merc_min, + (2 * 20037508.3427892) / (2 ^ zoom) as tile_size +) +SELECT st_makeenvelope( + merc_min + (tile_size * x), + merc_max - (tile_size * (y + 1)), + merc_min + (tile_size * (x + 1)), + merc_max - (tile_size * y), + 3857 +) FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; + + +CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ +SELECT age(clock_timestamp(), transaction_timestamp()); +$$ LANGUAGE SQL; +SET SEARCH_PATH to pgstac, public; + +DROP FUNCTION IF EXISTS geometrysearch; +CREATE OR REPLACE FUNCTION geometrysearch( + IN geom geometry, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered + IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items +) RETURNS jsonb AS $$ +DECLARE + search searches%ROWTYPE; + curs refcursor; + _where text; + query text; + iter_record items%ROWTYPE; + out_records jsonb := '{}'::jsonb[]; + exit_flag boolean := FALSE; + counter int := 1; + scancounter int := 1; + remaining_limit int := _scanlimit; + tilearea float; + unionedgeom geometry; + clippedgeom geometry; + unionedgeom_area float := 0; + prev_area float := 0; + excludes text[]; + includes text[]; + +BEGIN + DROP TABLE IF EXISTS pgstac_results; + CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; + + -- If skipcovered is true then you will always want to exit when the passed in geometry is full + IF skipcovered THEN + exitwhenfull := TRUE; + END IF; + + SELECT * INTO search FROM searches WHERE hash=queryhash; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; + END IF; + + tilearea := st_area(geom); + _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); + + + FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP + query := format('%s LIMIT %L', query, remaining_limit); + RAISE NOTICE '%', query; + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs INTO iter_record; + EXIT WHEN NOT FOUND; + IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations + clippedgeom := st_intersection(geom, iter_record.geometry); + + IF unionedgeom IS NULL THEN + unionedgeom := clippedgeom; + ELSE + unionedgeom := st_union(unionedgeom, clippedgeom); + END IF; + + unionedgeom_area := st_area(unionedgeom); + + IF skipcovered AND prev_area = unionedgeom_area THEN + scancounter := scancounter + 1; + CONTINUE; + END IF; + + prev_area := unionedgeom_area; + + RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); + END IF; + RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); + INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); + + IF counter >= _limit + OR scancounter > _scanlimit + OR ftime() > _timelimit + OR (exitwhenfull AND unionedgeom_area >= tilearea) + THEN + exit_flag := TRUE; + EXIT; + END IF; + counter := counter + 1; + scancounter := scancounter + 1; + + END LOOP; + CLOSE curs; + EXIT WHEN exit_flag; + remaining_limit := _scanlimit - scancounter; + END LOOP; + + SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; + + RETURN jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb) + ); +END; +$$ LANGUAGE PLPGSQL; + +DROP FUNCTION IF EXISTS geojsonsearch; +CREATE OR REPLACE FUNCTION geojsonsearch( + IN geojson jsonb, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_geomfromgeojson(geojson), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS xyzsearch; +CREATE OR REPLACE FUNCTION xyzsearch( + IN _x int, + IN _y int, + IN _z int, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_transform(tileenvelope(_z, _x, _y), 4326), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +RESET ROLE; + +INSERT INTO pgstac.collections (content) SELECT content FROM pgstac_045.collections; +INSERT INTO pgstac.items_staging SELECT content FROM pgstac_045.items; + + + +SELECT set_version('0.5.0'); diff --git a/pypgstac/pypgstac/migrations/pgstac.0.5.0.sql b/pypgstac/pypgstac/migrations/pgstac.0.5.0.sql new file mode 100644 index 00000000..a975bec7 --- /dev/null +++ b/pypgstac/pypgstac/migrations/pgstac.0.5.0.sql @@ -0,0 +1,2659 @@ +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS btree_gist; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + CREATE ROLE pgstac_read; + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +GRANT pgstac_admin TO current_user; + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + +SET SEARCH_PATH TO pgstac, public; + + +CREATE TABLE IF NOT EXISTS migrations ( + version text PRIMARY KEY, + datetime timestamptz DEFAULT clock_timestamp() NOT NULL +); + +CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ + SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ + INSERT INTO pgstac.migrations (version) VALUES ($1) + ON CONFLICT DO NOTHING + RETURNING version; +$$ LANGUAGE SQL; + + +CREATE TABLE IF NOT EXISTS pgstac_settings ( + name text PRIMARY KEY, + value text NOT NULL +); + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default-filter-lang', 'cql2-json'), + ('additional_properties', 'true') +ON CONFLICT DO NOTHING +; + + +CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ +SELECT COALESCE( + conf->>_setting, + current_setting(concat('pgstac.',_setting), TRUE), + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) +); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ + SELECT pgstac.get_setting('context', conf); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ + SELECT pgstac.get_setting('context_estimated_count', conf)::int; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_estimated_cost(); +CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ + SELECT pgstac.get_setting('context_estimated_cost', conf)::float; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_stats_ttl(); +CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ + SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ +DECLARE +debug boolean := current_setting('pgstac.debug', true); +BEGIN + IF debug THEN + RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); + RETURN TRUE; + END IF; + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ +SELECT CASE + WHEN $1 IS NULL THEN TRUE + WHEN cardinality($1)<1 THEN TRUE +ELSE FALSE +END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ + SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ +SELECT ARRAY( + SELECT $1[i] + FROM generate_subscripts($1,1) AS s(i) + ORDER BY i DESC +); +$$ LANGUAGE SQL STRICT IMMUTABLE; +CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ + SELECT floor(($1->>0)::float)::int; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ + SELECT ($1->>0)::float; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ + SELECT ($1->>0)::timestamptz; +$$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; + + +CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ + SELECT $1->>0; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ + SELECT + CASE jsonb_typeof($1) + WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) + ELSE ARRAY[$1->>0] + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ +SELECT CASE jsonb_array_length(_bbox) + WHEN 4 THEN + ST_SetSRID(ST_MakeEnvelope( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float, + (_bbox->>3)::float + ),4326) + WHEN 6 THEN + ST_SetSRID(ST_3DMakeBox( + ST_MakePoint( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float + ), + ST_MakePoint( + (_bbox->>3)::float, + (_bbox->>4)::float, + (_bbox->>5)::float + ) + ),4326) + ELSE null END; +; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ + SELECT jsonb_build_array( + st_xmin(_geom), + st_ymin(_geom), + st_xmax(_geom), + st_ymax(_geom) + ); +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ + SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +/* looks for a geometry in a stac item first from geometry and falling back to bbox */ +CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ +SELECT + CASE + WHEN value ? 'intersects' THEN + ST_GeomFromGeoJSON(value->>'intersects') + WHEN value ? 'geometry' THEN + ST_GeomFromGeoJSON(value->>'geometry') + WHEN value ? 'bbox' THEN + pgstac.bbox_geom(value->'bbox') + ELSE NULL + END as geometry +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION stac_daterange( + value jsonb +) RETURNS tstzrange AS $$ +DECLARE + props jsonb := value; + dt timestamptz; + edt timestamptz; +BEGIN + IF props ? 'properties' THEN + props := props->'properties'; + END IF; + IF props ? 'start_datetime' AND props ? 'end_datetime' THEN + dt := props->'start_datetime'; + edt := props->'end_datetime'; + IF dt > edt THEN + RAISE EXCEPTION 'start_datetime must be < end_datetime'; + END IF; + ELSE + dt := props->'datetime'; + edt := props->'datetime'; + END IF; + IF dt is NULL OR edt IS NULL THEN + RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; + END IF; + RETURN tstzrange(dt, edt, '[]'); +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT lower(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT upper(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + + +CREATE TABLE IF NOT EXISTS stac_extensions( + name text PRIMARY KEY, + url text, + enbabled_by_default boolean NOT NULL DEFAULT TRUE, + enableable boolean NOT NULL DEFAULT TRUE +); + +INSERT INTO stac_extensions (name, url) VALUES + ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), + ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), + ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), + ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), + ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') +ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; + + + +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id', + 'links', '[]'::jsonb + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); + +CREATE TABLE IF NOT EXISTS collections ( + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + partition_trunc partition_trunc_strategy +); + + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + SELECT relid::text INTO partition_name + FROM pg_partition_tree('items') + WHERE relid::text = partition_name; + IF FOUND THEN + partition_exists := true; + partition_empty := table_empty(partition_name); + ELSE + partition_exists := false; + partition_empty := true; + partition_name := format('_items_%s', NEW.key); + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN + q := format($q$ + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name + ); + EXECUTE q; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN + q := format($q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name, + partition_name + ); + EXECUTE q; + loadtemp := TRUE; + partition_empty := TRUE; + partition_exists := FALSE; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN + RETURN NEW; + END IF; + IF NEW.partition_trunc IS NULL AND partition_empty THEN + RAISE NOTICE '% % % %', + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); + ELSIF partition_empty THEN + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) + PARTITION BY RANGE (datetime); + $q$, + partition_name, + NEW.id + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + ELSE + RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; + END IF; + IF loadtemp THEN + RAISE NOTICE 'Moving data into new partitions.'; + q := format($q$ + WITH p AS ( + SELECT + collection, + datetime as datetime, + end_datetime as end_datetime, + (partition_name( + collection, + datetime + )).partition_name as name + FROM changepartitionstaging + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + INSERT INTO %I SELECT * FROM changepartitionstaging; + DROP TABLE IF EXISTS changepartitionstaging; + $q$, + partition_name + ); + EXECUTE q; + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + +CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW +EXECUTE FUNCTION collections_trigger_func(); + +CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ + UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; +$$ LANGUAGE SQL; + +CREATE TABLE IF NOT EXISTS partitions ( + collection text REFERENCES collections(id), + name text PRIMARY KEY, + partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), + datetime_range tstzrange, + end_datetime_range tstzrange, + CONSTRAINT prange EXCLUDE USING GIST ( + collection WITH =, + partition_range WITH && + ) +) WITH (FILLFACTOR=90); +CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); + + + +CREATE OR REPLACE FUNCTION partition_name( + IN collection text, + IN dt timestamptz, + OUT partition_name text, + OUT partition_range tstzrange +) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + + +CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + cq text; + parent_name text; + partition_trunc text; + partition_name text := NEW.name; + partition_exists boolean := false; + partition_empty boolean := true; + partition_range tstzrange; + datetime_range tstzrange; + end_datetime_range tstzrange; + err_context text; + mindt timestamptz := lower(NEW.datetime_range); + maxdt timestamptz := upper(NEW.datetime_range); + minedt timestamptz := lower(NEW.end_datetime_range); + maxedt timestamptz := upper(NEW.end_datetime_range); + t_mindt timestamptz; + t_maxdt timestamptz; + t_minedt timestamptz; + t_maxedt timestamptz; +BEGIN + RAISE NOTICE 'Partitions Trigger. %', NEW; + datetime_range := NEW.datetime_range; + end_datetime_range := NEW.end_datetime_range; + + SELECT + format('_items_%s', key), + c.partition_trunc::text + INTO + parent_name, + partition_trunc + FROM pgstac.collections c + WHERE c.id = NEW.collection; + SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; + NEW.name := partition_name; + + IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + + NEW.partition_range := partition_range; + IF TG_OP = 'UPDATE' THEN + mindt := least(mindt, lower(OLD.datetime_range)); + maxdt := greatest(maxdt, upper(OLD.datetime_range)); + minedt := least(minedt, lower(OLD.end_datetime_range)); + maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + END IF; + IF TG_OP = 'INSERT' THEN + + IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN + + RAISE NOTICE '% % %', partition_name, parent_name, partition_range; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + parent_name, + lower(partition_range), + upper(partition_range), + format('%s_pkey', partition_name), + partition_name + ); + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + END IF; + + END IF; + + -- Update constraints + EXECUTE format($q$ + SELECT + min(datetime), + max(datetime), + min(end_datetime), + max(end_datetime) + FROM %I; + $q$, partition_name) + INTO t_mindt, t_maxdt, t_minedt, t_maxedt; + mindt := least(mindt, t_mindt); + maxdt := greatest(maxdt, t_maxdt); + minedt := least(mindt, minedt, t_minedt); + maxedt := greatest(maxdt, maxedt, t_maxedt); + + mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); + maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); + maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + + + IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + IF + TG_OP='UPDATE' + AND OLD.datetime_range @> NEW.datetime_range + AND OLD.end_datetime_range @> NEW.end_datetime_range + THEN + RAISE NOTICE 'Range unchanged, not updating constraints.'; + ELSE + + RAISE NOTICE ' + SETTING CONSTRAINTS + mindt: %, maxdt: % + minedt: %, maxedt: % + ', mindt, maxdt, minedt, maxedt; + IF partition_trunc IS NULL THEN + cq := format($q$ + ALTER TABLE %7$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ( + (datetime >= %3$L) + AND (datetime <= %4$L) + AND (end_datetime >= %5$L) + AND (end_datetime <= %6$L) + ) NOT VALID + ; + ALTER TABLE %7$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + mindt, + maxdt, + minedt, + maxedt, + partition_name + ); + ELSE + cq := format($q$ + ALTER TABLE %5$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %2$I + CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID + ; + ALTER TABLE %5$I + VALIDATE CONSTRAINT %2$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + minedt, + maxedt, + partition_name + ); + + END IF; + RAISE NOTICE 'Altering Constraints. %', cq; + EXECUTE cq; + END IF; + ELSE + NEW.datetime_range = NULL; + NEW.end_datetime_range = NULL; + + cq := format($q$ + ALTER TABLE %3$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID + ; + ALTER TABLE %3$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + partition_name + ); + EXECUTE cq; + END IF; + + RETURN NEW; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW +EXECUTE FUNCTION partitions_trigger_func(); + + +CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ON CONFLICT (id) DO + UPDATE + SET content=EXCLUDED.content + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ + SELECT content FROM collections + WHERE id=$1 + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ + SELECT jsonb_agg(content) FROM collections; +; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; +CREATE TABLE queryables ( + id bigint GENERATED ALWAYS AS identity PRIMARY KEY, + name text UNIQUE NOT NULL, + collection_ids text[], -- used to determine what partitions to create indexes on + definition jsonb, + property_path text, + property_wrapper text, + property_index_type text +); +CREATE INDEX queryables_name_idx ON queryables (name); +CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); + + +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') +ON CONFLICT DO NOTHING; + + + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + +CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ + SELECT string_agg( + quote_literal(v), + '->' + ) FROM unnest(arr) v; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + + + +CREATE OR REPLACE FUNCTION queryable( + IN dotpath text, + OUT path text, + OUT expression text, + OUT wrapper text +) AS $$ +DECLARE + q RECORD; + path_elements text[]; +BEGIN + IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN + path := dotpath; + expression := dotpath; + wrapper := NULL; + RETURN; + END IF; + SELECT * INTO q FROM queryables WHERE name=dotpath; + IF q.property_wrapper IS NULL THEN + IF q.definition->>'type' = 'number' THEN + wrapper := 'to_float'; + ELSIF q.definition->>'format' = 'date-time' THEN + wrapper := 'to_tstz'; + ELSE + wrapper := 'to_text'; + END IF; + ELSE + wrapper := q.property_wrapper; + END IF; + IF q.property_path IS NOT NULL THEN + path := q.property_path; + ELSE + path_elements := string_to_array(dotpath, '.'); + IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN + path := format('content->%s', array_to_path(path_elements)); + ELSIF path_elements[1] = 'properties' THEN + path := format('content->%s', array_to_path(path_elements)); + ELSE + path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); + END IF; + END IF; + expression := format('%I(%s)', wrapper, path); + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ +DECLARE + queryable RECORD; + q text; +BEGIN + FOR queryable IN + SELECT + queryables.id as qid, + CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, + property_index_type, + expression + FROM + queryables + LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) + JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) + LOOP + q := format( + $q$ + CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); + $q$, + format('%s_%s_idx', queryable.part, queryable.qid), + queryable.part, + COALESCE(queryable.property_index_type, 'to_text'), + queryable.expression + ); + RAISE NOTICE '%',q; + EXECUTE q; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ +DECLARE +BEGIN +PERFORM create_queryable_indexes(); +RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); + +CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate jsonb, + relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) +) RETURNS tstzrange AS $$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate text, + relative_base timestamptz DEFAULT CURRENT_TIMESTAMP +) RETURNS tstzrange AS $$ + SELECT parse_dtrange(to_jsonb(_indate), relative_base); +$$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + ll text := 'datetime'; + lh text := 'end_datetime'; + rrange tstzrange; + rl text; + rh text; + outq text; +BEGIN + rrange := parse_dtrange(args->1); + RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; + op := lower(op); + rl := format('%L::timestamptz', lower(rrange)); + rh := format('%L::timestamptz', upper(rrange)); + outq := CASE op + WHEN 't_before' THEN 'lh < rl' + WHEN 't_after' THEN 'll > rh' + WHEN 't_meets' THEN 'lh = rl' + WHEN 't_metby' THEN 'll = rh' + WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' + WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' + WHEN 't_starts' THEN 'll = rl AND lh < rh' + WHEN 't_startedby' THEN 'll = rl AND lh > rh' + WHEN 't_during' THEN 'll > rl AND lh < rh' + WHEN 't_contains' THEN 'll < rl AND lh > rh' + WHEN 't_finishes' THEN 'll > rl AND lh = rh' + WHEN 't_finishedby' THEN 'll < rl AND lh = rh' + WHEN 't_equals' THEN 'll = rl AND lh = rh' + WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' + WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' + WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' + END; + outq := regexp_replace(outq, '\mll\M', ll); + outq := regexp_replace(outq, '\mlh\M', lh); + outq := regexp_replace(outq, '\mrl\M', rl); + outq := regexp_replace(outq, '\mrh\M', rh); + outq := format('(%s)', outq); + RETURN outq; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + + +CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + geom text; + j jsonb := args->1; +BEGIN + op := lower(op); + RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; + IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN + RAISE EXCEPTION 'Spatial Operator % Not Supported', op; + END IF; + op := regexp_replace(op, '^s_', 'st_'); + IF op = 'intersects' THEN + op := 'st_intersects'; + END IF; + -- Convert geometry to WKB string + IF j ? 'type' AND j ? 'coordinates' THEN + geom := st_geomfromgeojson(j)::text; + ELSIF jsonb_typeof(j) = 'array' THEN + geom := bbox_geom(j)::text; + END IF; + + RETURN format('%s(geometry, %L::geometry)', op, geom); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ +-- Translates anything passed in through the deprecated "query" into equivalent CQL2 +WITH t AS ( + SELECT key as property, value as ops + FROM jsonb_each(q) +), t2 AS ( + SELECT property, (jsonb_each(ops)).* + FROM t WHERE jsonb_typeof(ops) = 'object' + UNION ALL + SELECT property, 'eq', ops + FROM t WHERE jsonb_typeof(ops) != 'object' +) +SELECT + jsonb_strip_nulls(jsonb_build_object( + 'op', 'and', + 'args', jsonb_agg( + jsonb_build_object( + 'op', key, + 'args', jsonb_build_array( + jsonb_build_object('property',property), + value + ) + ) + ) + ) +) as qcql FROM t2 +; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ +DECLARE + args jsonb; + ret jsonb; +BEGIN + RAISE NOTICE 'CQL1_TO_CQL2: %', j; + IF j ? 'filter' THEN + RETURN cql1_to_cql2(j->'filter'); + END IF; + IF j ? 'property' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'array' THEN + SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; + RETURN args; + END IF; + IF jsonb_typeof(j) = 'number' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'string' THEN + RETURN j; + END IF; + + IF jsonb_typeof(j) = 'object' THEN + SELECT jsonb_build_object( + 'op', key, + 'args', cql1_to_cql2(value) + ) INTO ret + FROM jsonb_each(j) + WHERE j IS NOT NULL; + RETURN ret; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT; + +CREATE TABLE cql2_ops ( + op text PRIMARY KEY, + template text, + types text[] +); +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('in', '%s = ANY (%s)', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), + ('isnull', '%s IS NULL', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template; +; + + +CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ +#variable_conflict use_variable +DECLARE + args jsonb := j->'args'; + arg jsonb; + op text := lower(j->>'op'); + cql2op RECORD; + literal text; +BEGIN + IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'CQL2_QUERY: %', j; + IF j ? 'filter' THEN + RETURN cql2_query(j->'filter'); + END IF; + + IF j ? 'upper' THEN + RETURN format('upper(%s)', cql2_query(j->'upper')); + END IF; + + IF j ? 'lower' THEN + RETURN format('lower(%s)', cql2_query(j->'lower')); + END IF; + + -- Temporal Query + IF op ilike 't_%' or op = 'anyinteracts' THEN + RETURN temporal_op_query(op, args); + END IF; + + -- If property is a timestamp convert it to text to use with + -- general operators + IF j ? 'timestamp' THEN + RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); + END IF; + IF j ? 'interval' THEN + RAISE EXCEPTION 'Please use temporal operators when using intervals.'; + RETURN NONE; + END IF; + + -- Spatial Query + IF op ilike 's_%' or op = 'intersects' THEN + RETURN spatial_op_query(op, args); + END IF; + + + IF op = 'in' THEN + RETURN format( + '%s = ANY (%L)', + cql2_query(args->0), + to_text_array(args->1) + ); + END IF; + + + + IF op = 'between' THEN + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + RETURN format( + '%s BETWEEN %s and %s', + cql2_query(args->0, wrapper), + cql2_query(args->1->0, wrapper), + cql2_query(args->1->1, wrapper) + ); + END IF; + + -- Make sure that args is an array and run cql2_query on + -- each element of the array + RAISE NOTICE 'ARGS PRE: %', args; + IF j ? 'args' THEN + IF jsonb_typeof(args) != 'array' THEN + args := jsonb_build_array(args); + END IF; + + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + SELECT jsonb_agg(cql2_query(a, wrapper)) + INTO args + FROM jsonb_array_elements(args) a; + END IF; + RAISE NOTICE 'ARGS: %', args; + + IF op IN ('and', 'or') THEN + RETURN + format( + '(%s)', + array_to_string(to_text_array(args), format(' %s ', upper(op))) + ); + END IF; + + -- Look up template from cql2_ops + IF j ? 'op' THEN + SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; + IF FOUND THEN + -- If specific index set in queryables for a property cast other arguments to that type + RETURN format( + cql2op.template, + VARIADIC (to_text_array(args)) + ); + ELSE + RAISE EXCEPTION 'Operator % Not Supported.', op; + END IF; + END IF; + + + IF j ? 'property' THEN + RETURN (queryable(j->>'property')).expression; + END IF; + + IF wrapper IS NOT NULL THEN + EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; + RAISE NOTICE '% % %',wrapper, j, literal; + RETURN format('%I(%L)', wrapper, j); + END IF; + + RETURN quote_literal(to_text(j)); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION paging_dtrange( + j jsonb +) RETURNS tstzrange AS $$ +DECLARE + op text; + filter jsonb := j->'filter'; + dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); + sdate timestamptz := '-infinity'::timestamptz; + edate timestamptz := 'infinity'::timestamptz; + jpitem jsonb; +BEGIN + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP + op := lower(jpitem->>'op'); + dtrange := parse_dtrange(jpitem->'args'->1); + IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN + sdate := greatest(sdate,'-infinity'); + edate := least(edate, upper(dtrange)); + ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN + edate := least(edate, 'infinity'); + sdate := greatest(sdate, lower(dtrange)); + ELSIF op IN ('=', 'eq') THEN + edate := least(edate, upper(dtrange)); + sdate := greatest(sdate, lower(dtrange)); + END IF; + RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; + END LOOP; + END IF; + IF sdate > edate THEN + RETURN 'empty'::tstzrange; + END IF; + RETURN tstzrange(sdate,edate, '[]'); +END; +$$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION paging_collections( + IN j jsonb +) RETURNS text[] AS $$ +DECLARE + filter jsonb := j->'filter'; + jpitem jsonb; + op text; + args jsonb; + arg jsonb; + collections text[]; +BEGIN + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP + RAISE NOTICE 'JPITEM: %', jpitem; + op := jpitem->>'op'; + args := jpitem->'args'; + IF op IN ('=', 'eq', 'in') THEN + FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP + IF jsonb_typeof(arg) IN ('string', 'array') THEN + RAISE NOTICE 'arg: %, collections: %', arg, collections; + IF collections IS NULL OR collections = '{}'::text[] THEN + collections := to_text_array(arg); + ELSE + collections := array_intersection(collections, to_text_array(arg)); + END IF; + END IF; + END LOOP; + END IF; + END LOOP; + END IF; + IF collections = '{}'::text[] THEN + RETURN NULL; + END IF; + RETURN collections; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; +CREATE TABLE items ( + id text NOT NULL, + geometry geometry NOT NULL, + collection text NOT NULL, + datetime timestamptz NOT NULL, + end_datetime timestamptz NOT NULL, + content JSONB NOT NULL +) +PARTITION BY LIST (collection) +; + +CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); +CREATE INDEX "geometry_idx" ON items USING GIST (geometry); + +CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; + + +ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; + + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ + SELECT + jsonb_object_agg( + key, + CASE + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_slim(i.value, c.value) + ELSE i.value + END + ) + FROM + jsonb_each(_item) as i + LEFT JOIN + jsonb_each(_collection) as c + USING (key) + WHERE + i.value IS DISTINCT FROM c.value + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ + SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ + SELECT + content->>'id' as id, + stac_geom(content) as geometry, + content->>'collection' as collection, + stac_datetime(content) as datetime, + stac_end_datetime(content) as end_datetime, + content_slim(content) as content + ; +$$ LANGUAGE SQL STABLE; + + +CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ +DECLARE + includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); + excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); +BEGIN + IF f IS NULL THEN + RETURN NULL; + ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN + RETURN TRUE; + ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN + RETURN FALSE; + ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN + RETURN FALSE; + END IF; + RETURN TRUE; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ +DECLARE + includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); + excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); +BEGIN + RAISE NOTICE '% % %', k, val, kf; + + include := TRUE; + IF k = 'properties' AND NOT excludes ? 'properties' THEN + excludes := excludes || '["properties"]'; + include := TRUE; + RAISE NOTICE 'Prop include %', include; + ELSIF + jsonb_array_length(excludes)>0 AND excludes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND NOT includes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND includes ? k THEN + includes := '[]'::jsonb; + RAISE NOTICE 'KF: %', kf; + END IF; + kf := jsonb_build_object('includes', includes, 'excludes', excludes); + RETURN; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ + WITH t AS (SELECT * FROM jsonb_each(a)) + SELECT jsonb_object_agg(key, value) FROM t + WHERE value ? 'href'; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION content_hydrate( + _item jsonb, + _collection jsonb, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ + SELECT + jsonb_strip_nulls(jsonb_object_agg( + key, + CASE + WHEN key = 'properties' AND include_field('properties', fields) THEN + i.value + WHEN key = 'properties' THEN + content_hydrate(i.value, c.value, kf) + WHEN + c.value IS NULL AND key != 'properties' + THEN i.value + WHEN + key = 'assets' + AND + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN strip_assets(content_hydrate(i.value, c.value, kf)) + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_hydrate(i.value, c.value, kf) + ELSE coalesce(i.value, c.value) + END + )) + FROM + jsonb_each(coalesce(_item,'{}'::jsonb)) as i + FULL JOIN + jsonb_each(coalesce(_collection,'{}'::jsonb)) as c + USING (key) + JOIN LATERAL ( + SELECT kf, include FROM key_filter(key, i.value, fields) + ) as k ON (include) + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; + content jsonb; + base_item jsonb := _collection.base_item; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry)::jsonb; + END IF; + IF include_field('bbox', fields) THEN + bbox := geom_bbox(_item.geometry)::jsonb; + END IF; + output := content_hydrate( + jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'bbox',bbox, + 'collection', _item.collection + ) || _item.content, + _collection.base_item, + fields + ); + + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ + SELECT content_hydrate( + _item, + (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), + fields + ); +$$ LANGUAGE SQL STABLE; + + +CREATE UNLOGGED TABLE items_staging ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_ignore ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_upsert ( + content JSONB NOT NULL +); + +CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p record; + _partitions text[]; + ts timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + WITH ranges AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr + FROM newdata n + ), p AS ( + SELECT + collection, + lower(dtr) as datetime, + upper(dtr) as end_datetime, + (partition_name( + collection, + lower(dtr) + )).partition_name as name + FROM ranges + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata; + DELETE FROM items_staging; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata + ON CONFLICT DO NOTHING; + DELETE FROM items_staging_ignore; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + WITH staging_formatted AS ( + SELECT (content_dehydrate(content)).* FROM newdata + ), deletes AS ( + DELETE FROM items i USING staging_formatted s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO items + SELECT s.* FROM + staging_formatted s + JOIN deletes d + USING (id, collection); + DELETE FROM items_staging_upsert; + END IF; + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + + + + +CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS +$$ +DECLARE + i items%ROWTYPE; +BEGIN + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; + RETURN i; +END; +$$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE +out items%ROWTYPE; +BEGIN + DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL STABLE; + +--/* +CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ +DECLARE + old items %ROWTYPE; + out items%ROWTYPE; +BEGIN + SELECT delete_item(content->>'id', content->>'collection'); + SELECT create_item(content); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ + SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb + FROM items WHERE collection=$1; + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ + SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) + FROM items WHERE collection=$1; +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ +UPDATE collections SET + content = content || + jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', collection_bbox(collections.id) + ), + 'temporal', jsonb_build_object( + 'interval', collection_temporal_extent(collections.id) + ) + ) + ) +; +$$ LANGUAGE SQL; +CREATE VIEW partition_steps AS +SELECT + name, + date_trunc('month',lower(datetime_range)) as sdate, + date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate + FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange + ORDER BY datetime_range ASC +; + +CREATE OR REPLACE FUNCTION chunker( + IN _where text, + OUT s timestamptz, + OUT e timestamptz +) RETURNS SETOF RECORD AS $$ +DECLARE + explain jsonb; +BEGIN + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; + + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION partition_queries( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL +) RETURNS SETOF text AS $$ +DECLARE + query text; + sdate timestamptz; + edate timestamptz; +BEGIN +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + $q$, _where, _orderby + ); + + RETURN NEXT query; + RETURN; +END IF; + +RETURN; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION partition_query_view( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN _limit int DEFAULT 10 +) RETURNS text AS $$ + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total LIMIT %s + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ), + _limit + )) + ELSE NULL + END FROM p; +$$ LANGUAGE SQL IMMUTABLE; + + + + +CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ +DECLARE + where_segments text[]; + _where text; + dtrange tstzrange; + collections text[]; + geom geometry; + sdate timestamptz; + edate timestamptz; + filterlang text; + filter jsonb := j->'filter'; +BEGIN + IF j ? 'ids' THEN + where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); + END IF; + + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + where_segments := where_segments || format('collection = ANY (%L) ', collections); + END IF; + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + + where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', + edate, + sdate + ); + END IF; + + geom := stac_geom(j); + IF geom IS NOT NULL THEN + where_segments := where_segments || format('st_intersects(geometry, %L)',geom); + END IF; + + filterlang := COALESCE( + j->>'filter-lang', + get_setting('default-filter-lang', j->'conf') + ); + IF NOT filter @? '$.**.op' THEN + filterlang := 'cql-json'; + END IF; + + IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN + RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; + END IF; + + IF j ? 'query' AND j ? 'filter' THEN + RAISE EXCEPTION 'Can only use either query or filter at one time.'; + END IF; + + IF j ? 'query' THEN + filter := query_to_cql2(j->'query'); + ELSIF filterlang = 'cql-json' THEN + filter := cql1_to_cql2(filter); + END IF; + RAISE NOTICE 'FILTER: %', filter; + where_segments := where_segments || cql2_query(filter); + IF cardinality(where_segments) < 1 THEN + RETURN ' TRUE '; + END IF; + + _where := array_to_string(array_remove(where_segments, NULL), ' AND '); + + IF _where IS NULL OR BTRIM(_where) = '' THEN + RETURN ' TRUE '; + END IF; + RETURN _where; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN NOT reverse THEN d + WHEN d = 'ASC' THEN 'DESC' + WHEN d = 'DESC' THEN 'ASC' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN d = 'ASC' AND prev THEN '<=' + WHEN d = 'DESC' AND prev THEN '>=' + WHEN d = 'ASC' THEN '>=' + WHEN d = 'DESC' THEN '<=' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION sort_sqlorderby( + _search jsonb DEFAULT NULL, + reverse boolean DEFAULT FALSE +) RETURNS text AS $$ + WITH sortby AS ( + SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort + ), withid AS ( + SELECT CASE + WHEN sort @? '$[*] ? (@.field == "id")' THEN sort + ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb + END as sort + FROM sortby + ), withid_rows AS ( + SELECT jsonb_array_elements(sort) as value FROM withid + ),sorts AS ( + SELECT + coalesce( + -- field_orderby((items_path(value->>'field')).path_txt), + (queryable(value->>'field')).expression + ) as key, + parse_sort_dir(value->>'direction', reverse) as dir + FROM withid_rows + ) + SELECT array_to_string( + array_agg(concat(key, ' ', dir)), + ', ' + ) FROM sorts; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ + SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ +DECLARE + token_id text; + filters text[] := '{}'::text[]; + prev boolean := TRUE; + field text; + dir text; + sort record; + orfilters text[] := '{}'::text[]; + andfilters text[] := '{}'::text[]; + output text; + token_where text; +BEGIN + RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; + -- If no token provided return NULL + IF token_rec IS NULL THEN + IF NOT (_search ? 'token' AND + ( + (_search->>'token' ILIKE 'prev:%') + OR + (_search->>'token' ILIKE 'next:%') + ) + ) THEN + RETURN NULL; + END IF; + prev := (_search->>'token' ILIKE 'prev:%'); + token_id := substr(_search->>'token', 6); + SELECT to_jsonb(items) INTO token_rec + FROM items WHERE id=token_id; + END IF; + RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; + + CREATE TEMP TABLE sorts ( + _row int GENERATED ALWAYS AS IDENTITY NOT NULL, + _field text PRIMARY KEY, + _dir text NOT NULL, + _val text + ) ON COMMIT DROP; + + -- Make sure we only have distinct columns to sort with taking the first one we get + INSERT INTO sorts (_field, _dir) + SELECT + (queryable(value->>'field')).expression, + get_sort_dir(value) + FROM + jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) + ON CONFLICT DO NOTHING + ; + RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + -- Get the first sort direction provided. As the id is a primary key, if there are any + -- sorts after id they won't do anything, so make sure that id is the last sort item. + SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; + IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN + DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); + ELSE + INSERT INTO sorts (_field, _dir) VALUES ('id', dir); + END IF; + + -- Add value from looked up item to the sorts table + UPDATE sorts SET _val=quote_literal(token_rec->>_field); + + -- Check if all sorts are the same direction and use row comparison + -- to filter + RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + + IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN + SELECT format( + '(%s) %s (%s)', + concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), + CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, + concat_ws(', ', VARIADIC array_agg(_val)) + ) INTO output FROM sorts + WHERE token_rec ? _field + ; + ELSE + FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP + RAISE NOTICE 'SORT: %', sort; + IF sort._row = 1 THEN + orfilters := orfilters || format('(%s %s %s)', + quote_ident(sort._field), + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + ELSE + orfilters := orfilters || format('(%s AND %s %s %s)', + array_to_string(andfilters, ' AND '), + quote_ident(sort._field), + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + + END IF; + andfilters := andfilters || format('%s = %s', + quote_ident(sort._field), + sort._val + ); + END LOOP; + output := array_to_string(orfilters, ' OR '); + END IF; + DROP TABLE IF EXISTS sorts; + token_where := concat('(',coalesce(output,'true'),')'); + IF trim(token_where) = '' THEN + token_where := NULL; + END IF; + RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; + RETURN token_where; + END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ + SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ + SELECT md5(concat(search_tohash($1)::text,$2::text)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS searches( + hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, + search jsonb NOT NULL, + _where text, + orderby text, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); +CREATE TABLE IF NOT EXISTS search_wheres( + id bigint generated always as identity primary key, + _where text NOT NULL, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + statslastupdated timestamptz, + estimated_count bigint, + estimated_cost float, + time_to_estimate float, + total_count bigint, + time_to_count float, + partitions text[] +); + +CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); +CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); + +CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + partitions text[]; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost float := context_estimated_cost(conf); + _estimated_count int := context_estimated_count(conf); +BEGIN + IF _context = 'off' THEN + sw._where := inwhere; + return sw; + END IF; + + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + + -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired + IF NOT updatestats THEN + RAISE NOTICE 'Checking if update is needed for: % .', inwhere; + RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; + RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; + RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; + IF + sw.statslastupdated IS NULL + OR (now() - sw.statslastupdated) > _stats_ttl + OR (context(conf) != 'off' AND sw.total_count IS NULL) + THEN + updatestats := TRUE; + END IF; + END IF; + + sw._where := inwhere; + sw.lastused := now(); + sw.usecount := coalesce(sw.usecount,0) + 1; + + IF NOT updatestats THEN + UPDATE search_wheres SET + lastused = sw.lastused, + usecount = sw.usecount + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw + ; + RETURN sw; + END IF; + + -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.statslastupdated := now(); + sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; + sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.time_to_estimate := extract(epoch from i); + + RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; + RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; + + -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough + IF + _context = 'on' + OR + ( _context = 'auto' AND + ( + sw.estimated_count < _estimated_count + AND + sw.estimated_cost < _estimated_cost + ) + ) + THEN + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + ELSE + sw.total_count := NULL; + sw.time_to_count := NULL; + END IF; + + + INSERT INTO search_wheres + (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) + SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count + ON CONFLICT ((md5(_where))) + DO UPDATE + SET + lastused = sw.lastused, + usecount = sw.usecount, + statslastupdated = sw.statslastupdated, + estimated_count = sw.estimated_count, + estimated_cost = sw.estimated_cost, + time_to_estimate = sw.time_to_estimate, + total_count = sw.total_count, + time_to_count = sw.time_to_count + ; + RETURN sw; +END; +$$ LANGUAGE PLPGSQL ; + + + +DROP FUNCTION IF EXISTS search_query; +CREATE OR REPLACE FUNCTION search_query( + _search jsonb = '{}'::jsonb, + updatestats boolean = false, + _metadata jsonb = '{}'::jsonb +) RETURNS searches AS $$ +DECLARE + search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; +BEGIN + SELECT * INTO search FROM searches + WHERE hash=search_hash(_search, _metadata) FOR UPDATE; + + -- Calculate the where clause if not already calculated + IF search._where IS NULL THEN + search._where := stac_search_to_where(_search); + END IF; + + -- Calculate the order by clause if not already calculated + IF search.orderby IS NULL THEN + search.orderby := sort_sqlorderby(_search); + END IF; + + PERFORM where_stats(search._where, updatestats, _search->'conf'); + + search.lastused := now(); + search.usecount := coalesce(search.usecount, 0) + 1; + INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) + VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) + ON CONFLICT (hash) DO + UPDATE SET + _where = EXCLUDED._where, + orderby = EXCLUDED.orderby, + lastused = EXCLUDED.lastused, + usecount = EXCLUDED.usecount, + metadata = EXCLUDED.metadata + RETURNING * INTO search + ; + RETURN search; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + searches searches%ROWTYPE; + _where text; + token_where text; + full_where text; + orderby text; + query text; + token_type text := substr(_search->>'token',1,4); + _limit int := coalesce((_search->>'limit')::int, 10); + curs refcursor; + cntr int := 0; + iter_record items%ROWTYPE; + first_record jsonb; + last_record jsonb; + out_records jsonb := '[]'::jsonb; + prev_query text; + next text; + prev_id text; + has_next boolean := false; + has_prev boolean := false; + prev text; + total_count bigint; + context jsonb; + collection jsonb; + includes text[]; + excludes text[]; + exit_flag boolean := FALSE; + batches int := 0; + timer timestamptz := clock_timestamp(); + pstart timestamptz; + pend timestamptz; + pcurs refcursor; + search_where search_wheres%ROWTYPE; + id text; +BEGIN +CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; +-- if ids is set, short circuit and just use direct ids query for each id +-- skip any paging or caching +-- hard codes ordering in the same order as the array of ids +IF _search ? 'ids' THEN + INSERT INTO results + SELECT content_hydrate(items, _search->'fields') + FROM items WHERE + items.id = ANY(to_text_array(_search->'ids')) + AND + CASE WHEN _search ? 'collections' THEN + items.collection = ANY(to_text_array(_search->'collections')) + ELSE TRUE + END + ORDER BY items.datetime desc, items.id desc + ; + SELECT INTO total_count count(*) FROM results; +ELSE + searches := search_query(_search); + _where := searches._where; + orderby := searches.orderby; + search_where := where_stats(_where); + total_count := coalesce(search_where.total_count, search_where.estimated_count); + + IF token_type='prev' THEN + token_where := get_token_filter(_search, null::jsonb); + orderby := sort_sqlorderby(_search, TRUE); + END IF; + IF token_type='next' THEN + token_where := get_token_filter(_search, null::jsonb); + END IF; + + full_where := concat_ws(' AND ', _where, token_where); + RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; + timer := clock_timestamp(); + + FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP + timer := clock_timestamp(); + query := format('%s LIMIT %s', query, _limit + 1); + RAISE NOTICE 'Partition Query: %', query; + batches := batches + 1; + -- curs = create_cursor(query); + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs into iter_record; + EXIT WHEN NOT FOUND; + cntr := cntr + 1; + last_record := content_hydrate(iter_record, _search->'fields'); + IF cntr = 1 THEN + first_record := last_record; + END IF; + IF cntr <= _limit THEN + INSERT INTO results (content) VALUES (last_record); + ELSIF cntr > _limit THEN + has_next := true; + exit_flag := true; + EXIT; + END IF; + END LOOP; + CLOSE curs; + RAISE NOTICE 'Query took %.', clock_timestamp()-timer; + timer := clock_timestamp(); + EXIT WHEN exit_flag; + END LOOP; + RAISE NOTICE 'Scanned through % partitions.', batches; +END IF; + +SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; + +DROP TABLE results; + + +-- Flip things around if this was the result of a prev token query +IF token_type='prev' THEN + out_records := flip_jsonb_array(out_records); + first_record := last_record; +END IF; + +-- If this query has a token, see if there is data before the first record +IF _search ? 'token' THEN + prev_query := format( + 'SELECT 1 FROM items WHERE %s LIMIT 1', + concat_ws( + ' AND ', + _where, + trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) + ) + ); + RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; + EXECUTE prev_query INTO has_prev; + IF FOUND and has_prev IS NOT NULL THEN + RAISE NOTICE 'Query results from prev query: %', has_prev; + has_prev := TRUE; + END IF; +END IF; +has_prev := COALESCE(has_prev, FALSE); + +IF has_prev THEN + prev := out_records->0->>'id'; +END IF; +IF has_next OR token_type='prev' THEN + next := out_records->-1->>'id'; +END IF; + +IF context(_search->'conf') != 'off' THEN + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'matched', total_count, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +ELSE + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +END IF; + +collection := jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb), + 'next', next, + 'prev', prev, + 'context', context +); + +RETURN collection; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + +CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ +DECLARE + curs refcursor; + searches searches%ROWTYPE; + _where text; + _orderby text; + q text; + +BEGIN + searches := search_query(_search); + _where := searches._where; + _orderby := searches.orderby; + + OPEN curs FOR + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ) + )) + ELSE NULL + END FROM p; + RETURN curs; +END; +$$ LANGUAGE PLPGSQL; +SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ +WITH t AS ( + SELECT + 20037508.3427892 as merc_max, + -20037508.3427892 as merc_min, + (2 * 20037508.3427892) / (2 ^ zoom) as tile_size +) +SELECT st_makeenvelope( + merc_min + (tile_size * x), + merc_max - (tile_size * (y + 1)), + merc_min + (tile_size * (x + 1)), + merc_max - (tile_size * y), + 3857 +) FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; + + +CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ +SELECT age(clock_timestamp(), transaction_timestamp()); +$$ LANGUAGE SQL; +SET SEARCH_PATH to pgstac, public; + +DROP FUNCTION IF EXISTS geometrysearch; +CREATE OR REPLACE FUNCTION geometrysearch( + IN geom geometry, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered + IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items +) RETURNS jsonb AS $$ +DECLARE + search searches%ROWTYPE; + curs refcursor; + _where text; + query text; + iter_record items%ROWTYPE; + out_records jsonb := '{}'::jsonb[]; + exit_flag boolean := FALSE; + counter int := 1; + scancounter int := 1; + remaining_limit int := _scanlimit; + tilearea float; + unionedgeom geometry; + clippedgeom geometry; + unionedgeom_area float := 0; + prev_area float := 0; + excludes text[]; + includes text[]; + +BEGIN + DROP TABLE IF EXISTS pgstac_results; + CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; + + -- If skipcovered is true then you will always want to exit when the passed in geometry is full + IF skipcovered THEN + exitwhenfull := TRUE; + END IF; + + SELECT * INTO search FROM searches WHERE hash=queryhash; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; + END IF; + + tilearea := st_area(geom); + _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); + + + FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP + query := format('%s LIMIT %L', query, remaining_limit); + RAISE NOTICE '%', query; + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs INTO iter_record; + EXIT WHEN NOT FOUND; + IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations + clippedgeom := st_intersection(geom, iter_record.geometry); + + IF unionedgeom IS NULL THEN + unionedgeom := clippedgeom; + ELSE + unionedgeom := st_union(unionedgeom, clippedgeom); + END IF; + + unionedgeom_area := st_area(unionedgeom); + + IF skipcovered AND prev_area = unionedgeom_area THEN + scancounter := scancounter + 1; + CONTINUE; + END IF; + + prev_area := unionedgeom_area; + + RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); + END IF; + RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); + INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); + + IF counter >= _limit + OR scancounter > _scanlimit + OR ftime() > _timelimit + OR (exitwhenfull AND unionedgeom_area >= tilearea) + THEN + exit_flag := TRUE; + EXIT; + END IF; + counter := counter + 1; + scancounter := scancounter + 1; + + END LOOP; + CLOSE curs; + EXIT WHEN exit_flag; + remaining_limit := _scanlimit - scancounter; + END LOOP; + + SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; + + RETURN jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb) + ); +END; +$$ LANGUAGE PLPGSQL; + +DROP FUNCTION IF EXISTS geojsonsearch; +CREATE OR REPLACE FUNCTION geojsonsearch( + IN geojson jsonb, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_geomfromgeojson(geojson), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS xyzsearch; +CREATE OR REPLACE FUNCTION xyzsearch( + IN _x int, + IN _y int, + IN _z int, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_transform(tileenvelope(_z, _x, _y), 4326), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; +SELECT set_version('0.5.0'); diff --git a/pypgstac/pypgstac/pypgstac.py b/pypgstac/pypgstac/pypgstac.py index a032dabf..a54835a3 100755 --- a/pypgstac/pypgstac/pypgstac.py +++ b/pypgstac/pypgstac/pypgstac.py @@ -1,72 +1,70 @@ """Command utilities for managing pgstac.""" -import asyncio -import time -from typing import Optional - -import asyncpg -import typer - -from .migrate import run_migration, get_version_dsn, get_initial_version -from .load import loadopt, tables, load_ndjson - -app = typer.Typer() - - -@app.command() -def version(dsn: Optional[str] = None) -> None: - """Get version from a pgstac database.""" - version = asyncio.run(get_version_dsn(dsn)) - typer.echo(f"{version}") - - -@app.command() -def initversion() -> None: - """Get initial version.""" - typer.echo(get_initial_version()) - -@app.command() -def migrate(dsn: Optional[str] = None, toversion: Optional[str] = None) -> None: - """Migrate a pgstac database.""" - version = asyncio.run(run_migration(dsn, toversion)) - typer.echo(f"pgstac version {version}") - - -@app.command() -def load( - table: tables, - file: str, - dsn: str = None, - method: loadopt = typer.Option("insert", prompt="How to deal conflicting ids"), -) -> None: - """Load STAC data into a pgstac database.""" - typer.echo(asyncio.run(load_ndjson(file=file, table=table, dsn=dsn, method=method))) - - -@app.command() -def pgready(dsn: Optional[str] = None) -> None: - """Wait for a pgstac database to accept connections.""" - - async def wait_on_connection() -> bool: - cnt = 0 - - print("Waiting for pgstac to come online...", end="", flush=True) - while True: - if cnt > 150: - raise Exception("Unable to connect to database") - try: - print(".", end="", flush=True) - conn = await asyncpg.connect(dsn=dsn) - await conn.execute("SELECT 1") - await conn.close() - print("success!") - return True - except Exception: - time.sleep(0.1) - cnt += 1 - - asyncio.run(wait_on_connection()) +from typing import Optional +import sys +import fire +from pypgstac.db import PgstacDB +from pypgstac.migrate import Migrate +from pypgstac.load import Loader, Methods, Tables +import logging + +# sys.tracebacklimit = 0 + + +class PgstacCLI: + """CLI for PgStac.""" + + def __init__(self, dsn: Optional[str] = "", debug: bool = False): + """Initialize PgStac CLI.""" + self.dsn = dsn + self._db = PgstacDB(dsn=dsn, debug=debug) + if debug: + logging.basicConfig(level=logging.DEBUG) + sys.tracebacklimit = 1000 + + @property + def initversion(self) -> str: + """Return earliest migration version.""" + return "0.1.9" + + @property + def version(self) -> Optional[str]: + """Get PGStac version installed on database.""" + return self._db.version + + @property + def pg_version(self) -> str: + """Get PostgreSQL server version installed on database.""" + return self._db.pg_version + + def pgready(self) -> None: + """Wait for a pgstac database to accept connections.""" + self._db.wait() + + def search(self, query: str) -> str: + """Search PgStac.""" + return self._db.search(query) + + def migrate(self, toversion: Optional[str] = None) -> str: + """Migrate PgStac Database.""" + migrator = Migrate(self._db) + return migrator.run_migration(toversion=toversion) + + def load( + self, table: Tables, file: str, method: Optional[Methods] = Methods.insert + ) -> None: + """Load collections or items into PGStac.""" + loader = Loader(db=self._db) + if table == "collections": + loader.load_collections(file, method) + if table == "items": + loader.load_items(file, method) + + +def cli() -> fire.Fire: + """Wrap fire call for CLI.""" + fire.Fire(PgstacCLI) if __name__ == "__main__": - app() + fire.Fire(PgstacCLI) diff --git a/pypgstac/pyproject.toml b/pypgstac/pyproject.toml index b26049c6..67215510 100644 --- a/pypgstac/pyproject.toml +++ b/pypgstac/pyproject.toml @@ -1,9 +1,9 @@ [tool.poetry] name = "pypgstac" -version = "0.4.5" +version = "0.5.0" description = "" authors = ["David Bitner "] -keywords = ["stac", "asyncpg"] +keywords = ["stac", "postgres"] readme = "README" homepage = "https://github.com/stac-utils/pgstac" repository = "https://github.com/stac-utils/pgstac" @@ -13,20 +13,24 @@ include = ["pypgstac/migrations/pgstac*.sql"] [tool.poetry.dependencies] python = ">=3.7" smart-open = "^4.2.0" -typer = ">=0.4.0" orjson = ">=3.5.2" python-dateutil = "^2.8.2" -asyncpg = "^0.25.0" +fire = "^0.4.0" +psycopg = "^3.0.10" +psycopg-pool = "^3.1.1" +plpygis = "^0.2.0" +pydantic = "^1.9.0" +tenacity = "^8.0.1" [tool.poetry.dev-dependencies] pytest = "^5.2" flake8 = "^3.9.2" -black = "^21.7b0" +black = ">=21.7b0" mypy = "^0.910" types-orjson = "^0.1.1" [tool.poetry.scripts] -pypgstac = "pypgstac.pypgstac:app" +pypgstac = "pypgstac.pypgstac:cli" [build-system] requires = ["setuptools", "poetry-core>=1.0.0"] diff --git a/pypgstac/tests/conftest.py b/pypgstac/tests/conftest.py new file mode 100644 index 00000000..69b89404 --- /dev/null +++ b/pypgstac/tests/conftest.py @@ -0,0 +1,44 @@ +"""Fixtures for pypgstac tests.""" +from typing import Generator +import pytest +import os +import psycopg +from pypgstac.db import PgstacDB +from pypgstac.migrate import Migrate +from pypgstac.load import Loader + + +@pytest.fixture(scope="function") +def db() -> Generator: + """Fixture to get a fresh database.""" + origdb: str = os.getenv("PGDATABASE", "") + + with psycopg.connect(autocommit=True) as conn: + try: + conn.execute("CREATE DATABASE pgstactestdb;") + except psycopg.errors.DuplicateDatabase: + conn.execute("DROP DATABASE pgstactestdb WITH (FORCE);") + conn.execute("CREATE DATABASE pgstactestdb;") + + os.environ["PGDATABASE"] = "pgstactestdb" + + pgdb = PgstacDB() + + yield pgdb + + print("Closing Connection and Dropping DB") + pgdb.close() + os.environ["PGDATABASE"] = origdb + + with psycopg.connect(autocommit=True) as conn: + conn.execute("DROP DATABASE pgstactestdb WITH (FORCE);") + + +@pytest.fixture(scope="function") +def loader(db: PgstacDB) -> Generator: + """Fixture to get a loader and an empty pgstac.""" + db.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") + migrator = Migrate(db) + print(migrator.run_migration()) + ldr = Loader(db) + yield ldr diff --git a/pypgstac/tests/test_load.py b/pypgstac/tests/test_load.py index 55084cfe..f6683389 100644 --- a/pypgstac/tests/test_load.py +++ b/pypgstac/tests/test_load.py @@ -1,28 +1,224 @@ """Tests for pypgstac.""" -import asyncio from pathlib import Path -import unittest - -from pypgstac.pypgstac import load_ndjson, loadopt, tables +from pypgstac.load import Methods, Loader +from psycopg.errors import UniqueViolation +import pytest HERE = Path(__file__).parent TEST_DATA_DIR = HERE.parent.parent / "test" / "testdata" +TEST_COLLECTIONS_JSON = TEST_DATA_DIR / "collections.json" TEST_COLLECTIONS = TEST_DATA_DIR / "collections.ndjson" TEST_ITEMS = TEST_DATA_DIR / "items.ndjson" -class LoadTest(unittest.TestCase): - """Tests pypgstac data loader.""" +def test_load_collections_succeeds(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.insert, + ) + + +def test_load_collections_json_succeeds(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) - def test_load_testdata_succeeds(self) -> None: - """Test pypgstac data loader.""" - asyncio.run( - load_ndjson( - str(TEST_COLLECTIONS), - table=tables.collections, - method=loadopt.upsert, - ) + +def test_load_collections_json_duplicates_fails(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) + with pytest.raises(UniqueViolation): + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, ) - asyncio.run( - load_ndjson(str(TEST_ITEMS), table=tables.items, method=loadopt.upsert) + + +def test_load_collections_json_duplicates_with_upsert(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.upsert, + ) + + +def test_load_collections_json_duplicates_with_ignore(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.ignore, + ) + + +def test_load_items_duplicates_fails(loader: Loader) -> None: + """Test pypgstac collections loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.insert, + ) + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + with pytest.raises(UniqueViolation): + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, ) + + +def test_load_items_succeeds(loader: Loader) -> None: + """Test pypgstac items loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.upsert, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + +def test_load_items_ignore_succeeds(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.ignore, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.ignore, + ) + + +def test_load_items_upsert_succeeds(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.ignore, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.upsert, + ) + + +def test_load_items_delsert_succeeds(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS), + insert_mode=Methods.ignore, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.delsert, + ) + + +def test_partition_loads_default(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.ignore, + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + partitions = loader.db.query_one( + """ + SELECT count(*) from partitions; + """ + ) + + assert partitions == 1 + + +def test_partition_loads_month(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.ignore, + ) + if loader.db.connection is not None: + loader.db.connection.execute( + """ + UPDATE collections SET partition_trunc='month'; + """ + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + partitions = loader.db.query_one( + """ + SELECT count(*) from partitions; + """ + ) + + assert partitions == 2 + + +def test_partition_loads_year(loader: Loader) -> None: + """Test pypgstac items ignore loader.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.ignore, + ) + if loader.db.connection is not None: + loader.db.connection.execute( + """ + UPDATE collections SET partition_trunc='year'; + """ + ) + + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + + partitions = loader.db.query_one( + """ + SELECT count(*) from partitions; + """ + ) + + assert partitions == 1 diff --git a/scripts/bin/test b/scripts/bin/test index 4a4271f4..594de714 100755 --- a/scripts/bin/test +++ b/scripts/bin/test @@ -28,7 +28,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pypgstac/pypgstac pypgstac/tests echo "Running unit tests..." - python -m unittest discover pypgstac/tests + pytest pypgstac/tests echo "Checking if there are any staged migrations." find /opt/src/pypgstac/pypgstac/migrations | grep 'staged' && { echo "There are staged migrations in pypgstac/pypgstac/migrations. Please check migrations and remove staged suffix."; exit 1; } diff --git a/scripts/bin/testdb b/scripts/bin/testdb index 80a4c167..135aef0d 100755 --- a/scripts/bin/testdb +++ b/scripts/bin/testdb @@ -49,7 +49,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then create_migra_dbs trap drop_migra_dbs 0 2 3 15 - echo "Using latest base migrations on Temp DB 2" + echo "Using latest base migrations on Temp DB 2 $TODBURL" pypgstac migrate --dsn $TODBURL echo "Running PGTap Tests on Temp DB 2" pgtap $TODBURL diff --git a/scripts/migrate b/scripts/migrate index 808773c4..c399221d 100755 --- a/scripts/migrate +++ b/scripts/migrate @@ -19,6 +19,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then docker-compose \ -f docker-compose.yml \ run --rm dev \ - bash -c "pypgstac pgready && pypgstac migrate" + bash -c "pypgstac pgready && pypgstac migrate --debug" -fi \ No newline at end of file +fi diff --git a/sql/001_core.sql b/sql/001_core.sql index 6408c49c..f2bac507 100644 --- a/sql/001_core.sql +++ b/sql/001_core.sql @@ -1,21 +1,53 @@ CREATE EXTENSION IF NOT EXISTS postgis; -CREATE SCHEMA IF NOT EXISTS pgstac; +CREATE EXTENSION IF NOT EXISTS btree_gist; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + CREATE ROLE pgstac_read; + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +GRANT pgstac_admin TO current_user; + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + SET SEARCH_PATH TO pgstac, public; -CREATE TABLE migrations ( + +CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ - SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; + SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ - INSERT INTO migrations (version) VALUES ($1) + INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; +$$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( @@ -26,9 +58,10 @@ CREATE TABLE IF NOT EXISTS pgstac_settings ( INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), - ('context_estimated_cost', '1000000'), + ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), - ('default-filter-lang', 'cql2-json') + ('default-filter-lang', 'cql2-json'), + ('additional_properties', 'true') ON CONFLICT DO NOTHING ; @@ -37,28 +70,26 @@ CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT N SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), - (SELECT value FROM pgstac_settings WHERE name=_setting) + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; -DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ - SELECT get_setting('context', conf); + SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; -DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ - SELECT get_setting('context_estimated_count', conf)::int; + SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ - SELECT get_setting('context_estimated_cost', conf)::float; + SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ - SELECT get_setting('context_stats_ttl', conf)::interval; + SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; @@ -82,6 +113,11 @@ ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ + SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); +$$ LANGUAGE SQL IMMUTABLE; + + CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; @@ -92,24 +128,6 @@ CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ -DECLARE -rec record; -rows bigint; -BEGIN - FOR rec in EXECUTE format( - $q$ - EXPLAIN SELECT 1 FROM items WHERE %s - $q$, - _where) - LOOP - rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); - EXIT WHEN rows IS NOT NULL; - END LOOP; - - RETURN rows; -END; -$$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( @@ -117,4 +135,4 @@ SELECT ARRAY( FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); -$$ LANGUAGE 'sql' STRICT IMMUTABLE; +$$ LANGUAGE SQL STRICT IMMUTABLE; diff --git a/sql/001a_jsonutils.sql b/sql/001a_jsonutils.sql index 183cd4a4..a9214b44 100644 --- a/sql/001a_jsonutils.sql +++ b/sql/001a_jsonutils.sql @@ -1,138 +1,65 @@ -/* converts a jsonb text array to a pg text[] array */ -CREATE OR REPLACE FUNCTION textarr(_js jsonb) - RETURNS text[] AS $$ - SELECT - CASE jsonb_typeof(_js) - WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) - ELSE ARRAY[_js->>0] - END -; -$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; - - -CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS -SETOF RECORD AS $$ -with recursive extract_all as -( - select - ARRAY[key]::text[] as path, - value - FROM jsonb_each(jdata) -union all - select - path || coalesce(obj_key, (arr_key- 1)::text), - coalesce(obj_value, arr_value) - from extract_all - left join lateral - jsonb_each(case jsonb_typeof(value) when 'object' then value end) - as o(obj_key, obj_value) - on jsonb_typeof(value) = 'object' - left join lateral - jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) - with ordinality as a(arr_value, arr_key) - on jsonb_typeof(value) = 'array' - where obj_key is not null or arr_key is not null -) -select * -from extract_all; -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS -SETOF RECORD AS $$ -with recursive extract_all as -( - select - ARRAY[key]::text[] as path, - value - FROM jsonb_each(jdata) -union all - select - path || obj_key, - obj_value - from extract_all - left join lateral - jsonb_each(case jsonb_typeof(value) when 'object' then value end) - as o(obj_key, obj_value) - on jsonb_typeof(value) = 'object' - where obj_key is not null -) -select * -from extract_all; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ + SELECT floor(($1->>0)::float)::int; +$$ LANGUAGE SQL IMMUTABLE STRICT; -CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS -SETOF RECORD AS $$ -SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ + SELECT ($1->>0)::float; +$$ LANGUAGE SQL IMMUTABLE STRICT; +CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ + SELECT ($1->>0)::timestamptz; +$$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; -CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ -WITH t AS (SELECT unnest(includes) i) -SELECT EXISTS ( - SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') -); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ -WITH t AS (SELECT unnest(excludes) e) -SELECT NOT EXISTS ( - SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') -); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ + SELECT $1->>0; +$$ LANGUAGE SQL IMMUTABLE STRICT; +CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ + SELECT + CASE jsonb_typeof($1) + WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) + ELSE ARRAY[$1->>0] + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; -CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( - IN jdata jsonb, - IN includes text[] DEFAULT ARRAY[]::text[], - IN excludes text[] DEFAULT ARRAY[]::text[], - OUT path text[], - OUT value jsonb -) RETURNS -SETOF RECORD AS $$ -SELECT path, value -FROM jsonb_obj_paths(jdata) -WHERE - CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END - AND - path_excludes(path, excludes) +CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ +SELECT CASE jsonb_array_length(_bbox) + WHEN 4 THEN + ST_SetSRID(ST_MakeEnvelope( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float, + (_bbox->>3)::float + ),4326) + WHEN 6 THEN + ST_SetSRID(ST_3DMakeBox( + ST_MakePoint( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float + ), + ST_MakePoint( + (_bbox->>3)::float, + (_bbox->>4)::float, + (_bbox->>5)::float + ) + ),4326) + ELSE null END; ; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - -CREATE OR REPLACE FUNCTION filter_jsonb( - IN jdata jsonb, - IN includes text[] DEFAULT ARRAY[]::text[], - IN excludes text[] DEFAULT ARRAY[]::text[] -) RETURNS jsonb AS $$ -DECLARE -rec RECORD; -outj jsonb := '{}'::jsonb; -created_paths text[] := '{}'::text[]; -BEGIN +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; -IF empty_arr(includes) AND empty_arr(excludes) THEN -RAISE NOTICE 'no filter'; - RETURN jdata; -END IF; -FOR rec in -SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) -WHERE jsonb_typeof(value) != 'object' -LOOP - IF array_length(rec.path,1)>1 THEN - FOR i IN 1..(array_length(rec.path,1)-1) LOOP - IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN - outj := jsonb_set(outj, rec.path[1:i],'{}', true); - created_paths := created_paths || array_to_string(rec.path[1:i],'.'); - END IF; - END LOOP; - END IF; - outj := jsonb_set(outj, rec.path, rec.value, true); - created_paths := created_paths || array_to_string(rec.path,'.'); -END LOOP; -RETURN outj; -END; -$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ + SELECT jsonb_build_array( + st_xmin(_geom), + st_ymin(_geom), + st_xmax(_geom), + st_ymax(_geom) + ); +$$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ -SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; + SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; diff --git a/sql/001b_cursorutils.sql b/sql/001b_cursorutils.sql deleted file mode 100644 index 1ca644e3..00000000 --- a/sql/001b_cursorutils.sql +++ /dev/null @@ -1,227 +0,0 @@ -/* Functions to create an iterable of cursors over partitions. */ -CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ -DECLARE - curs refcursor; -BEGIN - OPEN curs FOR EXECUTE q; - RETURN curs; -END; -$$ LANGUAGE PLPGSQL; - -DROP FUNCTION IF EXISTS partition_queries; -CREATE OR REPLACE FUNCTION partition_queries( - IN _where text DEFAULT 'TRUE', - IN _orderby text DEFAULT 'datetime DESC, id DESC', - IN partitions text[] DEFAULT '{items}' -) RETURNS SETOF text AS $$ -DECLARE - partition_query text; - query text; - p text; - cursors refcursor; - dstart timestamptz; - dend timestamptz; - step interval := '10 weeks'::interval; -BEGIN - -IF _orderby ILIKE 'datetime d%' THEN - partitions := partitions; -ELSIF _orderby ILIKE 'datetime a%' THEN - partitions := array_reverse(partitions); -ELSE - query := format($q$ - SELECT * FROM items - WHERE %s - ORDER BY %s - $q$, _where, _orderby - ); - - RETURN NEXT query; - RETURN; -END IF; -RAISE NOTICE 'PARTITIONS ---> %',partitions; -IF cardinality(partitions) > 0 THEN - FOREACH p IN ARRAY partitions - --EXECUTE partition_query - LOOP - query := format($q$ - SELECT * FROM %I - WHERE %s - ORDER BY %s - $q$, - p, - _where, - _orderby - ); - RETURN NEXT query; - END LOOP; -END IF; -RETURN; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; - -CREATE OR REPLACE FUNCTION partition_cursor( - IN _where text DEFAULT 'TRUE', - IN _orderby text DEFAULT 'datetime DESC, id DESC' -) RETURNS SETOF refcursor AS $$ -DECLARE - partition_query text; - query text; - p record; - cursors refcursor; -BEGIN -FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP - RETURN NEXT create_cursor(query); -END LOOP; -RETURN; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; - -CREATE OR REPLACE FUNCTION partition_count( - IN _where text DEFAULT 'TRUE' -) RETURNS bigint AS $$ -DECLARE - partition_query text; - query text; - p record; - subtotal bigint; - total bigint := 0; -BEGIN -partition_query := format($q$ - SELECT partition, tstzrange - FROM items_partitions - ORDER BY tstzrange DESC; -$q$); -RAISE NOTICE 'Partition Query: %', partition_query; -FOR p IN - EXECUTE partition_query -LOOP - query := format($q$ - SELECT count(*) FROM items - WHERE datetime BETWEEN %L AND %L AND %s - $q$, lower(p.tstzrange), upper(p.tstzrange), _where - ); - RAISE NOTICE 'Query %', query; - RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; - EXECUTE query INTO subtotal; - total := subtotal + total; -END LOOP; -RETURN total; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; - - -CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ -DECLARE - q text; - end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); - collections_constraint text := concat(partition, '_collections_constraint'); -BEGIN - q := format($q$ - ALTER TABLE %I - DROP CONSTRAINT IF EXISTS %I, - DROP CONSTRAINT IF EXISTS %I; - $q$, - partition, - end_datetime_constraint, - collections_constraint - ); - - EXECUTE q; - RETURN; - -END; -$$ LANGUAGE PLPGSQL; - -DROP FUNCTION IF EXISTS partition_checks; -CREATE OR REPLACE FUNCTION partition_checks( - IN partition text, - OUT min_datetime timestamptz, - OUT max_datetime timestamptz, - OUT min_end_datetime timestamptz, - OUT max_end_datetime timestamptz, - OUT collections text[], - OUT cnt bigint -) RETURNS RECORD AS $$ -DECLARE -q text; -end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); -collections_constraint text := concat(partition, '_collections_constraint'); -BEGIN -RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; -q := format($q$ - SELECT - min(datetime), - max(datetime), - min(end_datetime), - max(end_datetime), - array_agg(DISTINCT collection_id), - count(*) - FROM %I; - $q$, - partition -); -EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; -RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); -IF cnt IS NULL or cnt = 0 THEN - RAISE NOTICE 'Partition % is empty, removing...', partition; - q := format($q$ - DROP TABLE IF EXISTS %I; - $q$, partition - ); - EXECUTE q; - RETURN; -END IF; -RAISE NOTICE 'Running Constraint DDL %', ftime(); -q := format($q$ - ALTER TABLE %I - DROP CONSTRAINT IF EXISTS %I, - ADD CONSTRAINT %I - check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, - DROP CONSTRAINT IF EXISTS %I, - ADD CONSTRAINT %I - check((collection_id = ANY(%L))) NOT VALID; - $q$, - partition, - end_datetime_constraint, - end_datetime_constraint, - min_end_datetime, - max_end_datetime, - collections_constraint, - collections_constraint, - collections, - partition -); -RAISE NOTICE 'q: %', q; - -EXECUTE q; -RAISE NOTICE 'Returning %', ftime(); -RETURN; - -END; -$$ LANGUAGE PLPGSQL; - -CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ -DECLARE -q text; -BEGIN -FOR q IN - SELECT FORMAT( - 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', - current_database(), - nsp.nspname, - cls.relname, - con.conname - ) - FROM pg_constraint AS con - JOIN pg_class AS cls - ON con.conrelid = cls.oid - JOIN pg_namespace AS nsp - ON cls.relnamespace = nsp.oid - WHERE convalidated IS FALSE - AND nsp.nspname = 'pgstac' -LOOP - EXECUTE q; -END LOOP; -END; -$$ LANGUAGE PLPGSQL; diff --git a/sql/001s_stacutils.sql b/sql/001s_stacutils.sql index 42e0afe3..9a4aab77 100644 --- a/sql/001s_stacutils.sql +++ b/sql/001s_stacutils.sql @@ -2,36 +2,67 @@ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE - WHEN value->>'geometry' IS NOT NULL THEN + WHEN value ? 'intersects' THEN + ST_GeomFromGeoJSON(value->>'intersects') + WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') - WHEN value->>'bbox' IS NOT NULL THEN - ST_MakeEnvelope( - (value->'bbox'->>0)::float, - (value->'bbox'->>1)::float, - (value->'bbox'->>2)::float, - (value->'bbox'->>3)::float, - 4326 - ) + WHEN value ? 'bbox' THEN + pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION stac_daterange( + value jsonb +) RETURNS tstzrange AS $$ +DECLARE + props jsonb := value; + dt timestamptz; + edt timestamptz; +BEGIN + IF props ? 'properties' THEN + props := props->'properties'; + END IF; + IF props ? 'start_datetime' AND props ? 'end_datetime' THEN + dt := props->'start_datetime'; + edt := props->'end_datetime'; + IF dt > edt THEN + RAISE EXCEPTION 'start_datetime must be < end_datetime'; + END IF; + ELSE + dt := props->'datetime'; + edt := props->'datetime'; + END IF; + IF dt is NULL OR edt IS NULL THEN + RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; + END IF; + RETURN tstzrange(dt, edt, '[]'); +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ -SELECT COALESCE( - (value->'properties'->>'datetime')::timestamptz, - (value->'properties'->>'start_datetime')::timestamptz -); + SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ -SELECT COALESCE( - (value->'properties'->>'datetime')::timestamptz, - (value->'properties'->>'end_datetime')::timestamptz -); + SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; -CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ -SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; +CREATE TABLE IF NOT EXISTS stac_extensions( + name text PRIMARY KEY, + url text, + enbabled_by_default boolean NOT NULL DEFAULT TRUE, + enableable boolean NOT NULL DEFAULT TRUE +); + +INSERT INTO stac_extensions (name, url) VALUES + ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), + ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), + ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), + ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), + ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') +ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; diff --git a/sql/002_collections.sql b/sql/002_collections.sql index 45bb6232..748a96b3 100644 --- a/sql/002_collections.sql +++ b/sql/002_collections.sql @@ -1,10 +1,436 @@ -SET SEARCH_PATH TO pgstac, public; + + + +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id', + 'links', '[]'::jsonb + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( - id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, - content JSONB + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + partition_trunc partition_trunc_strategy ); + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + SELECT relid::text INTO partition_name + FROM pg_partition_tree('items') + WHERE relid::text = partition_name; + IF FOUND THEN + partition_exists := true; + partition_empty := table_empty(partition_name); + ELSE + partition_exists := false; + partition_empty := true; + partition_name := format('_items_%s', NEW.key); + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN + q := format($q$ + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name + ); + EXECUTE q; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN + q := format($q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + $q$, + partition_name, + partition_name + ); + EXECUTE q; + loadtemp := TRUE; + partition_empty := TRUE; + partition_exists := FALSE; + END IF; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN + RETURN NEW; + END IF; + IF NEW.partition_trunc IS NULL AND partition_empty THEN + RAISE NOTICE '% % % %', + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + NEW.id, + concat(partition_name,'_id_idx'), + partition_name + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); + ELSIF partition_empty THEN + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) + PARTITION BY RANGE (datetime); + $q$, + partition_name, + NEW.id + ); + RAISE NOTICE 'q: %', q; + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; + ELSE + RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; + END IF; + IF loadtemp THEN + RAISE NOTICE 'Moving data into new partitions.'; + q := format($q$ + WITH p AS ( + SELECT + collection, + datetime as datetime, + end_datetime as end_datetime, + (partition_name( + collection, + datetime + )).partition_name as name + FROM changepartitionstaging + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range + ; + INSERT INTO %I SELECT * FROM changepartitionstaging; + DROP TABLE IF EXISTS changepartitionstaging; + $q$, + partition_name + ); + EXECUTE q; + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + +CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW +EXECUTE FUNCTION collections_trigger_func(); + +CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ + UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; +$$ LANGUAGE SQL; + +CREATE TABLE IF NOT EXISTS partitions ( + collection text REFERENCES collections(id), + name text PRIMARY KEY, + partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), + datetime_range tstzrange, + end_datetime_range tstzrange, + CONSTRAINT prange EXCLUDE USING GIST ( + collection WITH =, + partition_range WITH && + ) +) WITH (FILLFACTOR=90); +CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); + + + +CREATE OR REPLACE FUNCTION partition_name( + IN collection text, + IN dt timestamptz, + OUT partition_name text, + OUT partition_range tstzrange +) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + + +CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + cq text; + parent_name text; + partition_trunc text; + partition_name text := NEW.name; + partition_exists boolean := false; + partition_empty boolean := true; + partition_range tstzrange; + datetime_range tstzrange; + end_datetime_range tstzrange; + err_context text; + mindt timestamptz := lower(NEW.datetime_range); + maxdt timestamptz := upper(NEW.datetime_range); + minedt timestamptz := lower(NEW.end_datetime_range); + maxedt timestamptz := upper(NEW.end_datetime_range); + t_mindt timestamptz; + t_maxdt timestamptz; + t_minedt timestamptz; + t_maxedt timestamptz; +BEGIN + RAISE NOTICE 'Partitions Trigger. %', NEW; + datetime_range := NEW.datetime_range; + end_datetime_range := NEW.end_datetime_range; + + SELECT + format('_items_%s', key), + c.partition_trunc::text + INTO + parent_name, + partition_trunc + FROM pgstac.collections c + WHERE c.id = NEW.collection; + SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; + NEW.name := partition_name; + + IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + + NEW.partition_range := partition_range; + IF TG_OP = 'UPDATE' THEN + mindt := least(mindt, lower(OLD.datetime_range)); + maxdt := greatest(maxdt, upper(OLD.datetime_range)); + minedt := least(minedt, lower(OLD.end_datetime_range)); + maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + END IF; + IF TG_OP = 'INSERT' THEN + + IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN + + RAISE NOTICE '% % %', partition_name, parent_name, partition_range; + q := format($q$ + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + partition_name, + parent_name, + lower(partition_range), + upper(partition_range), + format('%s_pkey', partition_name), + partition_name + ); + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + END IF; + + END IF; + + -- Update constraints + EXECUTE format($q$ + SELECT + min(datetime), + max(datetime), + min(end_datetime), + max(end_datetime) + FROM %I; + $q$, partition_name) + INTO t_mindt, t_maxdt, t_minedt, t_maxedt; + mindt := least(mindt, t_mindt); + maxdt := greatest(maxdt, t_maxdt); + minedt := least(mindt, minedt, t_minedt); + maxedt := greatest(maxdt, maxedt, t_maxedt); + + mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); + maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); + maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; + + + IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN + NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); + NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); + IF + TG_OP='UPDATE' + AND OLD.datetime_range @> NEW.datetime_range + AND OLD.end_datetime_range @> NEW.end_datetime_range + THEN + RAISE NOTICE 'Range unchanged, not updating constraints.'; + ELSE + + RAISE NOTICE ' + SETTING CONSTRAINTS + mindt: %, maxdt: % + minedt: %, maxedt: % + ', mindt, maxdt, minedt, maxedt; + IF partition_trunc IS NULL THEN + cq := format($q$ + ALTER TABLE %7$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ( + (datetime >= %3$L) + AND (datetime <= %4$L) + AND (end_datetime >= %5$L) + AND (end_datetime <= %6$L) + ) NOT VALID + ; + ALTER TABLE %7$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + mindt, + maxdt, + minedt, + maxedt, + partition_name + ); + ELSE + cq := format($q$ + ALTER TABLE %5$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %2$I + CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID + ; + ALTER TABLE %5$I + VALIDATE CONSTRAINT %2$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + minedt, + maxedt, + partition_name + ); + + END IF; + RAISE NOTICE 'Altering Constraints. %', cq; + EXECUTE cq; + END IF; + ELSE + NEW.datetime_range = NULL; + NEW.end_datetime_range = NULL; + + cq := format($q$ + ALTER TABLE %3$I + DROP CONSTRAINT IF EXISTS %1$I, + DROP CONSTRAINT IF EXISTS %2$I, + ADD CONSTRAINT %1$I + CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID + ; + ALTER TABLE %3$I + VALIDATE CONSTRAINT %1$I; + $q$, + format('%s_dt', partition_name), + format('%s_edt', partition_name), + partition_name + ); + EXECUTE cq; + END IF; + + RETURN NEW; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW +EXECUTE FUNCTION partitions_trigger_func(); + + CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) @@ -13,7 +439,7 @@ $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE -out collections%ROWTYPE; + out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; @@ -30,7 +456,7 @@ $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE -out collections%ROWTYPE; + out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; @@ -38,12 +464,12 @@ $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ -SELECT content FROM collections -WHERE id=$1 -; + SELECT content FROM collections + WHERE id=$1 + ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ -SELECT jsonb_agg(content) FROM collections; + SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; diff --git a/sql/002a_queryables.sql b/sql/002a_queryables.sql new file mode 100644 index 00000000..747c9e43 --- /dev/null +++ b/sql/002a_queryables.sql @@ -0,0 +1,124 @@ +CREATE TABLE queryables ( + id bigint GENERATED ALWAYS AS identity PRIMARY KEY, + name text UNIQUE NOT NULL, + collection_ids text[], -- used to determine what partitions to create indexes on + definition jsonb, + property_path text, + property_wrapper text, + property_index_type text +); +CREATE INDEX queryables_name_idx ON queryables (name); +CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); + + +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') +ON CONFLICT DO NOTHING; + + + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + +CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ + SELECT string_agg( + quote_literal(v), + '->' + ) FROM unnest(arr) v; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + + + +CREATE OR REPLACE FUNCTION queryable( + IN dotpath text, + OUT path text, + OUT expression text, + OUT wrapper text +) AS $$ +DECLARE + q RECORD; + path_elements text[]; +BEGIN + IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN + path := dotpath; + expression := dotpath; + wrapper := NULL; + RETURN; + END IF; + SELECT * INTO q FROM queryables WHERE name=dotpath; + IF q.property_wrapper IS NULL THEN + IF q.definition->>'type' = 'number' THEN + wrapper := 'to_float'; + ELSIF q.definition->>'format' = 'date-time' THEN + wrapper := 'to_tstz'; + ELSE + wrapper := 'to_text'; + END IF; + ELSE + wrapper := q.property_wrapper; + END IF; + IF q.property_path IS NOT NULL THEN + path := q.property_path; + ELSE + path_elements := string_to_array(dotpath, '.'); + IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN + path := format('content->%s', array_to_path(path_elements)); + ELSIF path_elements[1] = 'properties' THEN + path := format('content->%s', array_to_path(path_elements)); + ELSE + path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); + END IF; + END IF; + expression := format('%I(%s)', wrapper, path); + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ +DECLARE + queryable RECORD; + q text; +BEGIN + FOR queryable IN + SELECT + queryables.id as qid, + CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, + property_index_type, + expression + FROM + queryables + LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) + JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) + LOOP + q := format( + $q$ + CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); + $q$, + format('%s_%s_idx', queryable.part, queryable.qid), + queryable.part, + COALESCE(queryable.property_index_type, 'to_text'), + queryable.expression + ); + RAISE NOTICE '%',q; + EXECUTE q; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ +DECLARE +BEGIN +PERFORM create_queryable_indexes(); +RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); + +CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); diff --git a/sql/002b_cql.sql b/sql/002b_cql.sql new file mode 100644 index 00000000..16f4dc28 --- /dev/null +++ b/sql/002b_cql.sql @@ -0,0 +1,451 @@ +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate jsonb, + relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) +) RETURNS tstzrange AS $$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate text, + relative_base timestamptz DEFAULT CURRENT_TIMESTAMP +) RETURNS tstzrange AS $$ + SELECT parse_dtrange(to_jsonb(_indate), relative_base); +$$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + ll text := 'datetime'; + lh text := 'end_datetime'; + rrange tstzrange; + rl text; + rh text; + outq text; +BEGIN + rrange := parse_dtrange(args->1); + RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; + op := lower(op); + rl := format('%L::timestamptz', lower(rrange)); + rh := format('%L::timestamptz', upper(rrange)); + outq := CASE op + WHEN 't_before' THEN 'lh < rl' + WHEN 't_after' THEN 'll > rh' + WHEN 't_meets' THEN 'lh = rl' + WHEN 't_metby' THEN 'll = rh' + WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' + WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' + WHEN 't_starts' THEN 'll = rl AND lh < rh' + WHEN 't_startedby' THEN 'll = rl AND lh > rh' + WHEN 't_during' THEN 'll > rl AND lh < rh' + WHEN 't_contains' THEN 'll < rl AND lh > rh' + WHEN 't_finishes' THEN 'll > rl AND lh = rh' + WHEN 't_finishedby' THEN 'll < rl AND lh = rh' + WHEN 't_equals' THEN 'll = rl AND lh = rh' + WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' + WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' + WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' + END; + outq := regexp_replace(outq, '\mll\M', ll); + outq := regexp_replace(outq, '\mlh\M', lh); + outq := regexp_replace(outq, '\mrl\M', rl); + outq := regexp_replace(outq, '\mrh\M', rh); + outq := format('(%s)', outq); + RETURN outq; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + + +CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + geom text; + j jsonb := args->1; +BEGIN + op := lower(op); + RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; + IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN + RAISE EXCEPTION 'Spatial Operator % Not Supported', op; + END IF; + op := regexp_replace(op, '^s_', 'st_'); + IF op = 'intersects' THEN + op := 'st_intersects'; + END IF; + -- Convert geometry to WKB string + IF j ? 'type' AND j ? 'coordinates' THEN + geom := st_geomfromgeojson(j)::text; + ELSIF jsonb_typeof(j) = 'array' THEN + geom := bbox_geom(j)::text; + END IF; + + RETURN format('%s(geometry, %L::geometry)', op, geom); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ +-- Translates anything passed in through the deprecated "query" into equivalent CQL2 +WITH t AS ( + SELECT key as property, value as ops + FROM jsonb_each(q) +), t2 AS ( + SELECT property, (jsonb_each(ops)).* + FROM t WHERE jsonb_typeof(ops) = 'object' + UNION ALL + SELECT property, 'eq', ops + FROM t WHERE jsonb_typeof(ops) != 'object' +) +SELECT + jsonb_strip_nulls(jsonb_build_object( + 'op', 'and', + 'args', jsonb_agg( + jsonb_build_object( + 'op', key, + 'args', jsonb_build_array( + jsonb_build_object('property',property), + value + ) + ) + ) + ) +) as qcql FROM t2 +; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ +DECLARE + args jsonb; + ret jsonb; +BEGIN + RAISE NOTICE 'CQL1_TO_CQL2: %', j; + IF j ? 'filter' THEN + RETURN cql1_to_cql2(j->'filter'); + END IF; + IF j ? 'property' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'array' THEN + SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; + RETURN args; + END IF; + IF jsonb_typeof(j) = 'number' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'string' THEN + RETURN j; + END IF; + + IF jsonb_typeof(j) = 'object' THEN + SELECT jsonb_build_object( + 'op', key, + 'args', cql1_to_cql2(value) + ) INTO ret + FROM jsonb_each(j) + WHERE j IS NOT NULL; + RETURN ret; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT; + +CREATE TABLE cql2_ops ( + op text PRIMARY KEY, + template text, + types text[] +); +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('in', '%s = ANY (%s)', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), + ('isnull', '%s IS NULL', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template; +; + + +CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ +#variable_conflict use_variable +DECLARE + args jsonb := j->'args'; + arg jsonb; + op text := lower(j->>'op'); + cql2op RECORD; + literal text; +BEGIN + IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'CQL2_QUERY: %', j; + IF j ? 'filter' THEN + RETURN cql2_query(j->'filter'); + END IF; + + IF j ? 'upper' THEN + RETURN format('upper(%s)', cql2_query(j->'upper')); + END IF; + + IF j ? 'lower' THEN + RETURN format('lower(%s)', cql2_query(j->'lower')); + END IF; + + -- Temporal Query + IF op ilike 't_%' or op = 'anyinteracts' THEN + RETURN temporal_op_query(op, args); + END IF; + + -- If property is a timestamp convert it to text to use with + -- general operators + IF j ? 'timestamp' THEN + RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); + END IF; + IF j ? 'interval' THEN + RAISE EXCEPTION 'Please use temporal operators when using intervals.'; + RETURN NONE; + END IF; + + -- Spatial Query + IF op ilike 's_%' or op = 'intersects' THEN + RETURN spatial_op_query(op, args); + END IF; + + + IF op = 'in' THEN + RETURN format( + '%s = ANY (%L)', + cql2_query(args->0), + to_text_array(args->1) + ); + END IF; + + + + IF op = 'between' THEN + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + RETURN format( + '%s BETWEEN %s and %s', + cql2_query(args->0, wrapper), + cql2_query(args->1->0, wrapper), + cql2_query(args->1->1, wrapper) + ); + END IF; + + -- Make sure that args is an array and run cql2_query on + -- each element of the array + RAISE NOTICE 'ARGS PRE: %', args; + IF j ? 'args' THEN + IF jsonb_typeof(args) != 'array' THEN + args := jsonb_build_array(args); + END IF; + + SELECT (queryable(a->>'property')).wrapper INTO wrapper + FROM jsonb_array_elements(args) a + WHERE a ? 'property' LIMIT 1; + + SELECT jsonb_agg(cql2_query(a, wrapper)) + INTO args + FROM jsonb_array_elements(args) a; + END IF; + RAISE NOTICE 'ARGS: %', args; + + IF op IN ('and', 'or') THEN + RETURN + format( + '(%s)', + array_to_string(to_text_array(args), format(' %s ', upper(op))) + ); + END IF; + + -- Look up template from cql2_ops + IF j ? 'op' THEN + SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; + IF FOUND THEN + -- If specific index set in queryables for a property cast other arguments to that type + RETURN format( + cql2op.template, + VARIADIC (to_text_array(args)) + ); + ELSE + RAISE EXCEPTION 'Operator % Not Supported.', op; + END IF; + END IF; + + + IF j ? 'property' THEN + RETURN (queryable(j->>'property')).expression; + END IF; + + IF wrapper IS NOT NULL THEN + EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; + RAISE NOTICE '% % %',wrapper, j, literal; + RETURN format('%I(%L)', wrapper, j); + END IF; + + RETURN quote_literal(to_text(j)); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION paging_dtrange( + j jsonb +) RETURNS tstzrange AS $$ +DECLARE + op text; + filter jsonb := j->'filter'; + dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); + sdate timestamptz := '-infinity'::timestamptz; + edate timestamptz := 'infinity'::timestamptz; + jpitem jsonb; +BEGIN + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP + op := lower(jpitem->>'op'); + dtrange := parse_dtrange(jpitem->'args'->1); + IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN + sdate := greatest(sdate,'-infinity'); + edate := least(edate, upper(dtrange)); + ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN + edate := least(edate, 'infinity'); + sdate := greatest(sdate, lower(dtrange)); + ELSIF op IN ('=', 'eq') THEN + edate := least(edate, upper(dtrange)); + sdate := greatest(sdate, lower(dtrange)); + END IF; + RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; + END LOOP; + END IF; + IF sdate > edate THEN + RETURN 'empty'::tstzrange; + END IF; + RETURN tstzrange(sdate,edate, '[]'); +END; +$$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION paging_collections( + IN j jsonb +) RETURNS text[] AS $$ +DECLARE + filter jsonb := j->'filter'; + jpitem jsonb; + op text; + args jsonb; + arg jsonb; + collections text[]; +BEGIN + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP + RAISE NOTICE 'JPITEM: %', jpitem; + op := jpitem->>'op'; + args := jpitem->'args'; + IF op IN ('=', 'eq', 'in') THEN + FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP + IF jsonb_typeof(arg) IN ('string', 'array') THEN + RAISE NOTICE 'arg: %, collections: %', arg, collections; + IF collections IS NULL OR collections = '{}'::text[] THEN + collections := to_text_array(arg); + ELSE + collections := array_intersection(collections, to_text_array(arg)); + END IF; + END IF; + END LOOP; + END IF; + END LOOP; + END IF; + IF collections = '{}'::text[] THEN + RETURN NULL; + END IF; + RETURN collections; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; diff --git a/sql/003_items.sql b/sql/003_items.sql index a3eb142e..ba8ed53e 100644 --- a/sql/003_items.sql +++ b/sql/003_items.sql @@ -1,421 +1,330 @@ -SET SEARCH_PATH TO pgstac, public; - CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, - collection_id text NOT NULL, + collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, - properties jsonb NOT NULL, content JSONB NOT NULL ) -PARTITION BY RANGE (datetime) +PARTITION BY LIST (collection) ; -CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ - with recursive extract_all as - ( - select - ARRAY[key]::text[] as path, - ARRAY[key]::text[] as fullpath, - value - FROM jsonb_each(content->'properties') - union all - select - CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, - path || coalesce(obj_key, (arr_key- 1)::text), - coalesce(obj_value, arr_value) - from extract_all - left join lateral - jsonb_each(case jsonb_typeof(value) when 'object' then value end) - as o(obj_key, obj_value) - on jsonb_typeof(value) = 'object' - left join lateral - jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) - with ordinality as a(arr_value, arr_key) - on jsonb_typeof(value) = 'array' - where obj_key is not null or arr_key is not null - ) - , paths AS ( - select - array_to_string(path, '.') as path, - value - FROM extract_all - WHERE - jsonb_typeof(value) NOT IN ('array','object') - ), grouped AS ( - SELECT path, jsonb_agg(distinct value) vals FROM paths group by path - ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped - ; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; - -CREATE INDEX "datetime_idx" ON items (datetime); -CREATE INDEX "end_datetime_idx" ON items (end_datetime); -CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); -CREATE INDEX "collection_idx" ON items (collection_id); +CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); -CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); -ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; +CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; -CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ -DECLARE - p text; -BEGIN - FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP - EXECUTE format('ANALYZE %I;', p); - END LOOP; -END; -$$ LANGUAGE PLPGSQL; +ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; + + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ + SELECT + jsonb_object_agg( + key, + CASE + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_slim(i.value, c.value) + ELSE i.value + END + ) + FROM + jsonb_each(_item) as i + LEFT JOIN + jsonb_each(_collection) as c + USING (key) + WHERE + i.value IS DISTINCT FROM c.value + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ - SELECT to_char($1, '"items_p"IYYY"w"IW'); +CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ + SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ - SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); -$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ + SELECT + content->>'id' as id, + stac_geom(content) as geometry, + content->>'collection' as collection, + stac_datetime(content) as datetime, + stac_end_datetime(content) as end_datetime, + content_slim(content) as content + ; +$$ LANGUAGE SQL STABLE; -CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ - SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); -$$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE - err_context text; + includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); + excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); BEGIN - EXECUTE format( - $f$ - CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items - FOR VALUES FROM (%2$L) TO (%3$L); - CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); - $f$, - partition, - partition_start, - partition_end, - concat(partition, '_id_pk') - ); -EXCEPTION - WHEN duplicate_table THEN - RAISE NOTICE 'Partition % already exists.', partition; - WHEN others THEN - GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; - RAISE INFO 'Error Name:%',SQLERRM; - RAISE INFO 'Error State:%', SQLSTATE; - RAISE INFO 'Error Context:%', err_context; + IF f IS NULL THEN + RETURN NULL; + ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN + RETURN TRUE; + ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN + RETURN FALSE; + ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN + RETURN FALSE; + END IF; + RETURN TRUE; END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; +$$ LANGUAGE PLPGSQL IMMUTABLE; + -CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ +CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ DECLARE - partition text := items_partition_name(ts); - partition_start timestamptz; - partition_end timestamptz; + includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); + excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); BEGIN - IF items_partition_exists(partition) THEN - RETURN partition; + RAISE NOTICE '% % %', k, val, kf; + + include := TRUE; + IF k = 'properties' AND NOT excludes ? 'properties' THEN + excludes := excludes || '["properties"]'; + include := TRUE; + RAISE NOTICE 'Prop include %', include; + ELSIF + jsonb_array_length(excludes)>0 AND excludes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND NOT includes ? k THEN + include := FALSE; + ELSIF + jsonb_array_length(includes)>0 AND includes ? k THEN + includes := '[]'::jsonb; + RAISE NOTICE 'KF: %', kf; END IF; - partition_start := date_trunc('week', ts); - partition_end := partition_start + '1 week'::interval; - PERFORM items_partition_create_worker(partition, partition_start, partition_end); - RAISE NOTICE 'partition: %', partition; - RETURN partition; + kf := jsonb_build_object('includes', includes, 'excludes', excludes); + RETURN; END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ -WITH t AS ( - SELECT - generate_series( - date_trunc('week',st), - date_trunc('week', et), - '1 week'::interval - ) w -) -SELECT items_partition_create(w) FROM t; -$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ + WITH t AS (SELECT * FROM jsonb_each(a)) + SELECT jsonb_object_agg(key, value) FROM t + WHERE value ? 'href'; +$$ LANGUAGE SQL IMMUTABLE STRICT; -CREATE UNLOGGED TABLE items_staging ( - content JSONB NOT NULL -); - -CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ -DECLARE - p record; - _partitions text[]; -BEGIN - CREATE TEMP TABLE new_partitions ON COMMIT DROP AS +CREATE OR REPLACE FUNCTION content_hydrate( + _item jsonb, + _collection jsonb, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ SELECT - items_partition_name(stac_datetime(content)) as partition, - date_trunc('week', min(stac_datetime(content))) as partition_start - FROM newdata - GROUP BY 1; - - -- set statslastupdated in cache to be old enough cache always regenerated - - SELECT array_agg(partition) INTO _partitions FROM new_partitions; - UPDATE search_wheres - SET - statslastupdated = NULL - WHERE _where IN ( - SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions - FOR UPDATE SKIP LOCKED - ) + jsonb_strip_nulls(jsonb_object_agg( + key, + CASE + WHEN key = 'properties' AND include_field('properties', fields) THEN + i.value + WHEN key = 'properties' THEN + content_hydrate(i.value, c.value, kf) + WHEN + c.value IS NULL AND key != 'properties' + THEN i.value + WHEN + key = 'assets' + AND + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN strip_assets(content_hydrate(i.value, c.value, kf)) + WHEN + jsonb_typeof(c.value) = 'object' + AND + jsonb_typeof(i.value) = 'object' + THEN content_hydrate(i.value, c.value, kf) + ELSE coalesce(i.value, c.value) + END + )) + FROM + jsonb_each(coalesce(_item,'{}'::jsonb)) as i + FULL JOIN + jsonb_each(coalesce(_collection,'{}'::jsonb)) as c + USING (key) + JOIN LATERAL ( + SELECT kf, include FROM key_filter(key, i.value, fields) + ) as k ON (include) ; - FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions - LOOP - RAISE NOTICE 'Getting partition % ready.', p.partition; - IF NOT items_partition_exists(p.partition) THEN - RAISE NOTICE 'Creating partition %.', p.partition; - PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); - END IF; - PERFORM drop_partition_constraints(p.partition); - END LOOP; - INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) - SELECT - content->>'id', - stac_geom(content), - content->>'collection', - stac_datetime(content), - stac_end_datetime(content), - properties_idx(content), - content - FROM newdata - ; - DELETE FROM items_staging; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - FOR p IN SELECT new_partitions.partition FROM new_partitions - LOOP - RAISE NOTICE 'Setting constraints for partition %.', p.partition; - PERFORM partition_checks(p.partition); - END LOOP; - DROP TABLE IF EXISTS new_partitions; - RETURN NULL; -END; -$$ LANGUAGE PLPGSQL; +CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; + content jsonb; + base_item jsonb := _collection.base_item; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry)::jsonb; + END IF; + IF include_field('bbox', fields) THEN + bbox := geom_bbox(_item.geometry)::jsonb; + END IF; + output := content_hydrate( + jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'bbox',bbox, + 'collection', _item.collection + ) || _item.content, + _collection.base_item, + fields + ); + + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; -CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata - FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); +CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ + SELECT content_hydrate( + _item, + (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), + fields + ); +$$ LANGUAGE SQL STABLE; +CREATE UNLOGGED TABLE items_staging ( + content JSONB NOT NULL +); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); +CREATE UNLOGGED TABLE items_staging_upsert ( + content JSONB NOT NULL +); -CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ +CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; + ts timestamptz := clock_timestamp(); BEGIN - CREATE TEMP TABLE new_partitions ON COMMIT DROP AS - SELECT - items_partition_name(stac_datetime(content)) as partition, - date_trunc('week', min(stac_datetime(content))) as partition_start - FROM newdata - GROUP BY 1; - - -- set statslastupdated in cache to be old enough cache always regenerated - - SELECT array_agg(partition) INTO _partitions FROM new_partitions; - UPDATE search_wheres - SET - statslastupdated = NULL - WHERE _where IN ( - SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions - FOR UPDATE SKIP LOCKED - ) + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + WITH ranges AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr + FROM newdata n + ), p AS ( + SELECT + collection, + lower(dtr) as datetime, + upper(dtr) as end_datetime, + (partition_name( + collection, + lower(dtr) + )).partition_name as name + FROM ranges + ) + INSERT INTO partitions (collection, datetime_range, end_datetime_range) + SELECT + collection, + tstzrange(min(datetime), max(datetime), '[]') as datetime_range, + tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range + FROM p + GROUP BY collection, name + ON CONFLICT (name) DO UPDATE SET + datetime_range = EXCLUDED.datetime_range, + end_datetime_range = EXCLUDED.end_datetime_range ; - FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions - LOOP - RAISE NOTICE 'Getting partition % ready.', p.partition; - IF NOT items_partition_exists(p.partition) THEN - RAISE NOTICE 'Creating partition %.', p.partition; - PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); - END IF; - PERFORM drop_partition_constraints(p.partition); - END LOOP; - - INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata; + DELETE FROM items_staging; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items SELECT - content->>'id', - stac_geom(content), - content->>'collection', - stac_datetime(content), - stac_end_datetime(content), - properties_idx(content), - content + (content_dehydrate(content)).* FROM newdata - ON CONFLICT DO NOTHING - ; - DELETE FROM items_staging; - + ON CONFLICT DO NOTHING; + DELETE FROM items_staging_ignore; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + WITH staging_formatted AS ( + SELECT (content_dehydrate(content)).* FROM newdata + ), deletes AS ( + DELETE FROM items i USING staging_formatted s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO items + SELECT s.* FROM + staging_formatted s + JOIN deletes d + USING (id, collection); + DELETE FROM items_staging_upsert; + END IF; + RAISE NOTICE 'Done. %', clock_timestamp() - ts; - FOR p IN SELECT new_partitions.partition FROM new_partitions - LOOP - RAISE NOTICE 'Setting constraints for partition %.', p.partition; - PERFORM partition_checks(p.partition); - END LOOP; - DROP TABLE IF EXISTS new_partitions; RETURN NULL; + END; $$ LANGUAGE PLPGSQL; -CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata - FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); +CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); -CREATE UNLOGGED TABLE items_staging_upsert ( - content JSONB NOT NULL -); +CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); -CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ -DECLARE - p record; - _partitions text[]; -BEGIN - CREATE TEMP TABLE new_partitions ON COMMIT DROP AS - SELECT - items_partition_name(stac_datetime(content)) as partition, - date_trunc('week', min(stac_datetime(content))) as partition_start - FROM newdata - GROUP BY 1; - - -- set statslastupdated in cache to be old enough cache always regenerated - - SELECT array_agg(partition) INTO _partitions FROM new_partitions; - UPDATE search_wheres - SET - statslastupdated = NULL - WHERE _where IN ( - SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions - FOR UPDATE SKIP LOCKED - ) - ; +CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); - FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions - LOOP - RAISE NOTICE 'Getting partition % ready.', p.partition; - IF NOT items_partition_exists(p.partition) THEN - RAISE NOTICE 'Creating partition %.', p.partition; - PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); - END IF; - PERFORM drop_partition_constraints(p.partition); - END LOOP; - - INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) - SELECT - content->>'id', - stac_geom(content), - content->>'collection', - stac_datetime(content), - stac_end_datetime(content), - properties_idx(content), - content - FROM newdata - ON CONFLICT (datetime, id) DO UPDATE SET - content = EXCLUDED.content - WHERE items.content IS DISTINCT FROM EXCLUDED.content - ; - DELETE FROM items_staging; - - - FOR p IN SELECT new_partitions.partition FROM new_partitions - LOOP - RAISE NOTICE 'Setting constraints for partition %.', p.partition; - PERFORM partition_checks(p.partition); - END LOOP; - DROP TABLE IF EXISTS new_partitions; - RETURN NULL; -END; -$$ LANGUAGE PLPGSQL; -CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata - FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); -CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ -DECLARE -BEGIN - NEW.id := NEW.content->>'id'; - NEW.datetime := stac_datetime(NEW.content); - NEW.end_datetime := stac_end_datetime(NEW.content); - NEW.collection_id := NEW.content->>'collection'; - NEW.geometry := stac_geom(NEW.content); - NEW.properties := properties_idx(NEW.content); - IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN - RETURN NULL; - END IF; - RETURN NEW; -END; -$$ LANGUAGE PLPGSQL; -CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items - FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); - -/* -View to get a table of available items partitions -with date ranges -*/ -DROP VIEW IF EXISTS all_items_partitions CASCADE; -CREATE VIEW all_items_partitions AS -WITH base AS -(SELECT - c.oid::pg_catalog.regclass::text as partition, - pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, - regexp_matches( - pg_catalog.pg_get_expr(c.relpartbound, c.oid), - E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' - ) as t, - reltuples::bigint as est_cnt -FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i -WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) -SELECT partition, tstzrange( - t[1]::timestamptz, - t[2]::timestamptz -), t[1]::timestamptz as pstart, - t[2]::timestamptz as pend, est_cnt -FROM base -ORDER BY 2 desc; - -CREATE OR REPLACE VIEW items_partitions AS -SELECT * FROM all_items_partitions WHERE est_cnt>0; - -CREATE OR REPLACE FUNCTION item_by_id(_id text) RETURNS items AS +CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN - SELECT * INTO i FROM items WHERE id=_id LIMIT 1; + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; -$$ LANGUAGE PLPGSQL; +$$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; -CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ - SELECT content FROM items WHERE id=_id; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; +CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; -CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN - DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; + DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; +$$ LANGUAGE PLPGSQL STABLE; +--/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; -CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE + old items %ROWTYPE; out items%ROWTYPE; BEGIN - UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; + SELECT delete_item(content->>'id', content->>'collection'); + SELECT create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; @@ -435,14 +344,14 @@ $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ -SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb -FROM items WHERE collection_id=$1; -; + SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb + FROM items WHERE collection=$1; + ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ -SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) -FROM items WHERE collection_id=$1; + SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) + FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; @@ -460,4 +369,4 @@ UPDATE collections SET ) ) ; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; +$$ LANGUAGE SQL; diff --git a/sql/004_search.sql b/sql/004_search.sql index 69c97979..c72d90cd 100644 --- a/sql/004_search.sql +++ b/sql/004_search.sql @@ -1,791 +1,274 @@ - -SET SEARCH_PATH TO pgstac, public; - -CREATE OR REPLACE FUNCTION items_path( - IN dotpath text, - OUT field text, - OUT path text, - OUT path_txt text, - OUT jsonpath text, - OUT eq text -) RETURNS RECORD AS $$ -DECLARE -path_elements text[]; -last_element text; -BEGIN -dotpath := replace(trim(dotpath), 'properties.', ''); - -IF dotpath = '' THEN - RETURN; -END IF; - -path_elements := string_to_array(dotpath, '.'); -jsonpath := NULL; - -IF path_elements[1] IN ('id','geometry','datetime') THEN - field := path_elements[1]; - path_elements := path_elements[2:]; -ELSIF path_elements[1] = 'collection' THEN - field := 'collection_id'; - path_elements := path_elements[2:]; -ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN - field := 'content'; -ELSE - field := 'content'; - path_elements := '{properties}'::text[] || path_elements; -END IF; -IF cardinality(path_elements)<1 THEN - path := field; - path_txt := field; - jsonpath := '$'; - eq := NULL; - RETURN; -END IF; - - -last_element := path_elements[cardinality(path_elements)]; -path_elements := path_elements[1:cardinality(path_elements)-1]; -jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); -path_elements := array_map_literal(path_elements); -path := format($F$ properties->%s $F$, quote_literal(dotpath)); -path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); -eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); - -RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; -RETURN; -END; -$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - - -CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ -WITH t AS ( - SELECT CASE - WHEN _indate ? 'timestamp' THEN - ARRAY[_indate->>'timestamp', 'infinity'] - WHEN _indate ? 'interval' THEN - textarr(_indate->'interval') - WHEN jsonb_typeof(_indate) = 'array' THEN - textarr(_indate) - ELSE - regexp_split_to_array( - btrim(_indate::text,'"'), - '/' - ) - END AS arr -) -, t1 AS ( - SELECT - CASE - WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz - ELSE arr[1]::timestamptz - END AS st, - CASE - WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz - WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz - ELSE arr[2]::timestamptz - END AS et - FROM t -) +CREATE VIEW partition_steps AS SELECT - tstzrange(st,et) -FROM t1; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - - -CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ -SELECT CASE jsonb_array_length(_bbox) - WHEN 4 THEN - ST_SetSRID(ST_MakeEnvelope( - (_bbox->>0)::float, - (_bbox->>1)::float, - (_bbox->>2)::float, - (_bbox->>3)::float - ),4326) - WHEN 6 THEN - ST_SetSRID(ST_3DMakeBox( - ST_MakePoint( - (_bbox->>0)::float, - (_bbox->>1)::float, - (_bbox->>2)::float - ), - ST_MakePoint( - (_bbox->>3)::float, - (_bbox->>4)::float, - (_bbox->>5)::float - ) - ),4326) - ELSE null END; + name, + date_trunc('month',lower(datetime_range)) as sdate, + date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate + FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange + ORDER BY datetime_range ASC ; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ -SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN - jsonb_build_object( - 'and', - jsonb_build_array( - existing->'filter', - newfilters - ) - ) -ELSE - newfilters -END; -$$ LANGUAGE SQL; - - --- ADDs base filters (ids, collections, datetime, bbox, intersects) that are --- added outside of the filter/query in the stac request -CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ +CREATE OR REPLACE FUNCTION chunker( + IN _where text, + OUT s timestamptz, + OUT e timestamptz +) RETURNS SETOF RECORD AS $$ DECLARE -newprop jsonb; -newprops jsonb := '[]'::jsonb; + explain jsonb; BEGIN -IF j ? 'ids' THEN - newprop := jsonb_build_object( - 'in', - jsonb_build_array( - '{"property":"id"}'::jsonb, - j->'ids' - ) - ); - newprops := jsonb_insert(newprops, '{1}', newprop); -END IF; -IF j ? 'collections' THEN - newprop := jsonb_build_object( - 'in', - jsonb_build_array( - '{"property":"collection"}'::jsonb, - j->'collections' - ) - ); - newprops := jsonb_insert(newprops, '{1}', newprop); -END IF; - -IF j ? 'datetime' THEN - newprop := format( - '{"anyinteracts":[{"property":"datetime"}, %s]}', - j->'datetime' - ); - newprops := jsonb_insert(newprops, '{1}', newprop); -END IF; - -IF j ? 'bbox' THEN - newprop := format( - '{"intersects":[{"property":"geometry"}, %s]}', - j->'bbox' - ); - newprops := jsonb_insert(newprops, '{1}', newprop); -END IF; - -IF j ? 'intersects' THEN - newprop := format( - '{"intersects":[{"property":"geometry"}, %s]}', - j->'intersects' - ); - newprops := jsonb_insert(newprops, '{1}', newprop); -END IF; - -RAISE NOTICE 'newprops: %', newprops; - -IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN - return jsonb_set( - j, - '{filter}', - cql_and_append(j, jsonb_build_object('and', newprops)) - ) - '{ids,collections,datetime,bbox,intersects}'::text[]; -END IF; + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; -return j; + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; - -CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ --- Translates anything passed in through the deprecated "query" into equivalent CQL -WITH t AS ( - SELECT key as property, value as ops - FROM jsonb_each(j->'query') -), t2 AS ( - SELECT property, (jsonb_each(ops)).* - FROM t WHERE jsonb_typeof(ops) = 'object' - UNION ALL - SELECT property, 'eq', ops - FROM t WHERE jsonb_typeof(ops) != 'object' -), t3 AS ( -SELECT - jsonb_strip_nulls(jsonb_build_object( - 'and', - jsonb_agg( - jsonb_build_object( - key, - jsonb_build_array( - jsonb_build_object('property',property), - value - ) - ) - ) - )) as qcql FROM t2 -) -SELECT - CASE WHEN qcql IS NOT NULL THEN - jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' - ELSE j - END -FROM t3 -; -$$ LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION base_stac_query(j jsonb) RETURNS text AS $$ +CREATE OR REPLACE FUNCTION partition_queries( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL +) RETURNS SETOF text AS $$ DECLARE -_where text := ''; -dtrange tstzrange; -geom geometry; + query text; + sdate timestamptz; + edate timestamptz; BEGIN - - IF j ? 'ids' THEN - _where := format('%s AND id = ANY (%L) ', _where, textarr(j->'ids')); - END IF; - IF j ? 'collections' THEN - _where := format('%s AND collection_id = ANY (%L) ', _where, textarr(j->'collections')); - END IF; - - IF j ? 'datetime' THEN - dtrange := parse_dtrange(j->'datetime'); - _where := format('%s AND datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', - _where, - upper(dtrange), - lower(dtrange) - ); - END IF; - - IF j ? 'bbox' THEN - geom := bbox_geom(j->'bbox'); - _where := format('%s AND geometry && %L', +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, _where, - geom + _orderby ); - END IF; - - IF j ? 'intersects' THEN - geom := st_geomfromgeojson(j->>'intersects'); - _where := format('%s AND st_intersects(geometry, %L)', + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, _where, - geom + _orderby ); - END IF; - - RETURN _where; -END; -$$ LANGUAGE PLPGSQL; - - -CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ -DECLARE -ll text := 'datetime'; -lh text := 'end_datetime'; -rrange tstzrange; -rl text; -rh text; -outq text; -BEGIN -rrange := parse_dtrange(args->1); -RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; -op := lower(op); -rl := format('%L::timestamptz', lower(rrange)); -rh := format('%L::timestamptz', upper(rrange)); -outq := CASE op - WHEN 't_before' THEN 'lh < rl' - WHEN 't_after' THEN 'll > rh' - WHEN 't_meets' THEN 'lh = rl' - WHEN 't_metby' THEN 'll = rh' - WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' - WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' - WHEN 't_starts' THEN 'll = rl AND lh < rh' - WHEN 't_startedby' THEN 'll = rl AND lh > rh' - WHEN 't_during' THEN 'll > rl AND lh < rh' - WHEN 't_contains' THEN 'll < rl AND lh > rh' - WHEN 't_finishes' THEN 'll > rl AND lh = rh' - WHEN 't_finishedby' THEN 'll < rl AND lh = rh' - WHEN 't_equals' THEN 'll = rl AND lh = rh' - WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' - WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' - WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' -END; -outq := regexp_replace(outq, '\mll\M', ll); -outq := regexp_replace(outq, '\mlh\M', lh); -outq := regexp_replace(outq, '\mrl\M', rl); -outq := regexp_replace(outq, '\mrh\M', rh); -outq := format('(%s)', outq); -RETURN outq; -END; -$$ LANGUAGE PLPGSQL; + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + $q$, _where, _orderby + ); -CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ -DECLARE -geom text; -j jsonb := args->1; -BEGIN -op := lower(op); -RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; -IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN - RAISE EXCEPTION 'Spatial Operator % Not Supported', op; -END IF; -op := regexp_replace(op, '^s_', 'st_'); -IF op = 'intersects' THEN - op := 'st_intersects'; -END IF; --- Convert geometry to WKB string -IF j ? 'type' AND j ? 'coordinates' THEN - geom := st_geomfromgeojson(j)::text; -ELSIF jsonb_typeof(j) = 'array' THEN - geom := bbox_geom(j)::text; + RETURN NEXT query; + RETURN; END IF; -RETURN format('%s(geometry, %L::geometry)', op, geom); +RETURN; END; -$$ LANGUAGE PLPGSQL; - +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -/* cql_query_op -- Parses a CQL query operation, recursing when necessary - IN jsonb -- a subelement from a valid stac query - IN text -- the operator being used on elements passed in - RETURNS a SQL fragment to be used in a WHERE clause -*/ -CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ +CREATE OR REPLACE FUNCTION partition_query_view( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN _limit int DEFAULT 10 +) RETURNS text AS $$ + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total LIMIT %s + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ), + _limit + )) + ELSE NULL + END FROM p; +$$ LANGUAGE SQL IMMUTABLE; + + + + +CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE -jtype text := jsonb_typeof(j); -op text := lower(_op); -ops jsonb := - '{ - "eq": "%s = %s", - "lt": "%s < %s", - "lte": "%s <= %s", - "gt": "%s > %s", - "gte": "%s >= %s", - "le": "%s <= %s", - "ge": "%s >= %s", - "=": "%s = %s", - "<": "%s < %s", - "<=": "%s <= %s", - ">": "%s > %s", - ">=": "%s >= %s", - "like": "%s LIKE %s", - "ilike": "%s ILIKE %s", - "+": "%s + %s", - "-": "%s - %s", - "*": "%s * %s", - "/": "%s / %s", - "in": "%s = ANY (%s)", - "not": "NOT (%s)", - "between": "%s BETWEEN %s AND %s", - "lower":" lower(%s)", - "upper":" upper(%s)", - "isnull": "%s IS NULL" - }'::jsonb; -ret text; -args text[] := NULL; - + where_segments text[]; + _where text; + dtrange tstzrange; + collections text[]; + geom geometry; + sdate timestamptz; + edate timestamptz; + filterlang text; + filter jsonb := j->'filter'; BEGIN -RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; - --- for in, convert value, list to array syntax to match other ops -IF op = 'in' and j ? 'value' and j ? 'list' THEN - j := jsonb_build_array( j->'value', j->'list'); - jtype := 'array'; - RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; -END IF; - -IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN - j := jsonb_build_array( j->'value', j->'lower', j->'upper'); - jtype := 'array'; - RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; -END IF; - -IF op = 'not' AND jtype = 'object' THEN - j := jsonb_build_array( j ); - jtype := 'array'; - RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; -END IF; - --- Set Lower Case on Both Arguments When Case Insensitive Flag Set -IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN - IF (j->>2)::boolean THEN - RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); + IF j ? 'ids' THEN + where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; -END IF; --- Special Case when comparing a property in a jsonb field to a string or number using eq --- Allows to leverage GIN index on jsonb fields -IF op = 'eq' THEN - IF j->0 ? 'property' - AND jsonb_typeof(j->1) IN ('number','string') - AND (items_path(j->0->>'property')).eq IS NOT NULL - THEN - RETURN format((items_path(j->0->>'property')).eq, j->1); + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; -END IF; - - - -IF op ilike 't_%' or op = 'anyinteracts' THEN - RETURN temporal_op_query(op, j); -END IF; - -IF op ilike 's_%' or op = 'intersects' THEN - RETURN spatial_op_query(op, j); -END IF; + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); -IF jtype = 'object' THEN - RAISE NOTICE 'parsing object'; - IF j ? 'property' THEN - -- Convert the property to be used as an identifier - return (items_path(j->>'property')).path_txt; - ELSIF _op IS NULL THEN - -- Iterate to convert elements in an object where the operator has not been set - -- Combining with AND - SELECT - array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') - INTO ret - FROM jsonb_each(j) e; - RETURN ret; + where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', + edate, + sdate + ); END IF; -END IF; - -IF jtype = 'string' THEN - RETURN quote_literal(j->>0); -END IF; - -IF jtype ='number' THEN - RETURN (j->>0)::numeric; -END IF; -IF jtype = 'array' AND op IS NULL THEN - RAISE NOTICE 'Parsing array into array arg. j: %', j; - SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; - RETURN ret; -END IF; - - --- If the type of the passed json is an array --- Calculate the arguments that will be passed to functions/operators -IF jtype = 'array' THEN - RAISE NOTICE 'Parsing array into args. j: %', j; - -- If any argument is numeric, cast any text arguments to numeric - IF j @? '$[*] ? (@.type() == "number")' THEN - SELECT INTO args - array_agg(concat('(',cql_query_op(e),')::numeric')) - FROM jsonb_array_elements(j) e; - ELSE - SELECT INTO args - array_agg(cql_query_op(e)) - FROM jsonb_array_elements(j) e; + geom := stac_geom(j); + IF geom IS NOT NULL THEN + where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; - --RETURN args; -END IF; -RAISE NOTICE 'ARGS after array cleaning: %', args; - -IF op IS NULL THEN - RETURN args::text[]; -END IF; - -IF args IS NULL OR cardinality(args) < 1 THEN - RAISE NOTICE 'No Args'; - RETURN ''; -END IF; -IF op IN ('and','or') THEN - SELECT - CONCAT( - '(', - array_to_string(args, UPPER(CONCAT(' ',op,' '))), - ')' - ) INTO ret - FROM jsonb_array_elements(j) e; - RETURN ret; -END IF; - --- If the op is in the ops json then run using the template in the json -IF ops ? op THEN - RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); - - RETURN format(concat('(',ops->>op,')'), VARIADIC args); -END IF; - -RETURN j->>0; - -END; -$$ LANGUAGE PLPGSQL; - - -/* cql_query_op -- Parses a CQL query operation, recursing when necessary - IN jsonb -- a subelement from a valid stac query - IN text -- the operator being used on elements passed in - RETURNS a SQL fragment to be used in a WHERE clause -*/ -DROP FUNCTION IF EXISTS cql2_query; -CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ -DECLARE -args jsonb := j->'args'; -jtype text := jsonb_typeof(j->'args'); -op text := lower(j->>'op'); -arg jsonb; -argtext text; -argstext text[] := '{}'::text[]; -inobj jsonb; -_numeric text := ''; -ops jsonb := - '{ - "eq": "%s = %s", - "lt": "%s < %s", - "lte": "%s <= %s", - "gt": "%s > %s", - "gte": "%s >= %s", - "le": "%s <= %s", - "ge": "%s >= %s", - "=": "%s = %s", - "<": "%s < %s", - "<=": "%s <= %s", - ">": "%s > %s", - ">=": "%s >= %s", - "like": "%s LIKE %s", - "ilike": "%s ILIKE %s", - "+": "%s + %s", - "-": "%s - %s", - "*": "%s * %s", - "/": "%s / %s", - "in": "%s = ANY (%s)", - "not": "NOT (%s)", - "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", - "lower":" lower(%s)", - "upper":" upper(%s)", - "isnull": "%s IS NULL" - }'::jsonb; -ret text; - -BEGIN -RAISE NOTICE 'j: %s', j; -IF j ? 'filter' THEN - RETURN cql2_query(j->'filter'); -END IF; - -IF j ? 'upper' THEN -RAISE NOTICE 'upper %s',jsonb_build_object( - 'op', 'upper', - 'args', jsonb_build_array( j-> 'upper') - ) ; - RETURN cql2_query( - jsonb_build_object( - 'op', 'upper', - 'args', jsonb_build_array( j-> 'upper') - ) + filterlang := COALESCE( + j->>'filter-lang', + get_setting('default-filter-lang', j->'conf') ); -END IF; - -IF j ? 'lower' THEN - RETURN cql2_query( - jsonb_build_object( - 'op', 'lower', - 'args', jsonb_build_array( j-> 'lower') - ) - ); -END IF; - -IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN - args := jsonb_build_array(args); -END IF; --- END Cases where no further nesting is expected -IF j ? 'op' THEN - -- Special case to use JSONB index for equality - IF op IN ('eq', '=') - AND args->0 ? 'property' - AND jsonb_typeof(args->1) IN ('number', 'string') - AND (items_path(args->0->>'property')).eq IS NOT NULL - THEN - RETURN format((items_path(args->0->>'property')).eq, args->1); + IF NOT filter @? '$.**.op' THEN + filterlang := 'cql-json'; END IF; - -- Temporal Query - IF op ilike 't_%' or op = 'anyinteracts' THEN - RETURN temporal_op_query(op, args); + IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN + RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; - -- Spatial Query - IF op ilike 's_%' or op = 'intersects' THEN - RETURN spatial_op_query(op, args); + IF j ? 'query' AND j ? 'filter' THEN + RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; - -- In Query - separate into separate eq statements so that we can use eq jsonb optimization - IF op = 'in' THEN - RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; - SELECT INTO inobj - jsonb_agg( - jsonb_build_object( - 'op', 'eq', - 'args', jsonb_build_array( args->0 , v) - ) - ) - FROM jsonb_array_elements( args->1) v; - RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); + IF j ? 'query' THEN + filter := query_to_cql2(j->'query'); + ELSIF filterlang = 'cql-json' THEN + filter := cql1_to_cql2(filter); END IF; -END IF; - -IF j ? 'property' THEN - RETURN (items_path(j->>'property')).path_txt; -END IF; - -IF j ? 'timestamp' THEN - RETURN quote_literal(j->>'timestamp'); -END IF; - -RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; -IF jsonb_typeof(j) = 'number' THEN - RETURN format('%L::numeric', j->>0); -END IF; - -IF jsonb_typeof(j) = 'string' THEN - RETURN quote_literal(j->>0); -END IF; - -IF jsonb_typeof(j) = 'array' THEN - IF j @? '$[*] ? (@.type() == "number")' THEN - RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); - ELSE - RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); + RAISE NOTICE 'FILTER: %', filter; + where_segments := where_segments || cql2_query(filter); + IF cardinality(where_segments) < 1 THEN + RETURN ' TRUE '; END IF; -END IF; -RAISE NOTICE 'ARGS after array cleaning: %', args; - -RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; -IF j ? 'args' THEN - FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP - argtext := cql2_query(arg, recursion + 1); - RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; - argstext := argstext || argtext; - END LOOP; -END IF; -RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; - -IF op IN ('and', 'or') THEN - RAISE NOTICE 'inand op: %, argstext: %', op, argstext; - SELECT - concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') - INTO ret - FROM unnest(argstext) e; - RETURN ret; -END IF; + _where := array_to_string(array_remove(where_segments, NULL), ' AND '); -IF ops ? op THEN - IF argstext[2] ~* 'numeric' THEN - argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; + IF _where IS NULL OR BTRIM(_where) = '' THEN + RETURN ' TRUE '; END IF; - RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); -END IF; - -RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; + RETURN _where; -RETURN NULL; END; -$$ LANGUAGE PLPGSQL; - - - -CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ -DECLARE -filterlang text; -search jsonb := _search; -base_where text; -_where text; -BEGIN - -RAISE NOTICE 'SEARCH CQL Final: %', search; -filterlang := COALESCE( - search->>'filter-lang', - get_setting('default-filter-lang', _search->'conf') -); - -base_where := base_stac_query(search); - -IF filterlang = 'cql-json' THEN - search := query_to_cqlfilter(search); - -- search := add_filters_to_cql(search); - _where := cql_query_op(search->'filter'); -ELSE - _where := cql2_query(search->'filter'); -END IF; - -IF trim(_where) = '' THEN - _where := NULL; -END IF; -_where := coalesce(_where, ' TRUE '); -RETURN format('( %s ) %s ', _where, base_where); -END; -$$ LANGUAGE PLPGSQL; +$$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ -WITH t AS ( - SELECT COALESCE(upper(_dir), 'ASC') as d -) SELECT - CASE - WHEN NOT reverse THEN d - WHEN d = 'ASC' THEN 'DESC' - WHEN d = 'DESC' THEN 'ASC' - END -FROM t; + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN NOT reverse THEN d + WHEN d = 'ASC' THEN 'DESC' + WHEN d = 'DESC' THEN 'ASC' + END + FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ -WITH t AS ( - SELECT COALESCE(upper(_dir), 'ASC') as d -) SELECT - CASE - WHEN d = 'ASC' AND prev THEN '<=' - WHEN d = 'DESC' AND prev THEN '>=' - WHEN d = 'ASC' THEN '>=' - WHEN d = 'DESC' THEN '<=' - END -FROM t; + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN d = 'ASC' AND prev THEN '<=' + WHEN d = 'DESC' AND prev THEN '>=' + WHEN d = 'ASC' THEN '>=' + WHEN d = 'DESC' THEN '<=' + END + FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ -WITH t AS ( - SELECT - replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s - FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' -) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; -$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ -WITH sortby AS ( - SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort -), withid AS ( - SELECT CASE - WHEN sort @? '$[*] ? (@.field == "id")' THEN sort - ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb - END as sort - FROM sortby -), withid_rows AS ( - SELECT jsonb_array_elements(sort) as value FROM withid -),sorts AS ( - SELECT - coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, - parse_sort_dir(value->>'direction', reverse) as dir - FROM withid_rows -) -SELECT array_to_string( - array_agg(concat(key, ' ', dir)), - ', ' -) FROM sorts; + WITH sortby AS ( + SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort + ), withid AS ( + SELECT CASE + WHEN sort @? '$[*] ? (@.field == "id")' THEN sort + ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb + END as sort + FROM sortby + ), withid_rows AS ( + SELECT jsonb_array_elements(sort) as value FROM withid + ),sorts AS ( + SELECT + coalesce( + -- field_orderby((items_path(value->>'field')).path_txt), + (queryable(value->>'field')).expression + ) as key, + parse_sort_dir(value->>'direction', reverse) as dir + FROM withid_rows + ) + SELECT array_to_string( + array_agg(concat(key, ' ', dir)), + ', ' + ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ -SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; + SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; @@ -802,6 +285,7 @@ DECLARE output text; token_where text; BEGIN + RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND @@ -815,9 +299,10 @@ BEGIN END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); - SELECT to_jsonb(items) INTO token_rec FROM item_by_id(token_id) as items; + SELECT to_jsonb(items) INTO token_rec + FROM items WHERE id=token_id; END IF; - RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; + RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, @@ -829,7 +314,7 @@ BEGIN -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT - (items_path(value->>'field')).path, + (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) @@ -938,18 +423,27 @@ DECLARE partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost float := context_estimated_cost(conf); + _estimated_count int := context_estimated_count(conf); BEGIN - SELECT * INTO sw FROM search_wheres WHERE _where=inwhere_hash FOR UPDATE; + IF _context = 'off' THEN + sw._where := inwhere; + return sw; + END IF; + + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN - RAISE NOTICE 'Checking if update is needed.'; + RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; - RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; - RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; + RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; + RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL - OR (now() - sw.statslastupdated) > context_stats_ttl(conf) + OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; @@ -964,49 +458,36 @@ BEGIN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount - WHERE _where = inwhere_hash + WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; + -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; - WITH t AS ( - SELECT j->>0 as p FROM - jsonb_path_query( - explain_json, - 'strict $.**."Relation Name" ? (@ != null)' - ) j - ), ordered AS ( - SELECT p FROM t ORDER BY p DESC - -- SELECT p FROM t JOIN items_partitions - -- ON (t.p = items_partitions.partition) - -- ORDER BY pstart DESC - ) - SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; - RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; - - sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); - sw.partitions := partitions; + + RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; + RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF - context(conf) = 'on' + _context = 'on' OR - ( context(conf) = 'auto' AND + ( _context = 'auto' AND ( - sw.estimated_count < context_estimated_count(conf) - OR - sw.estimated_cost < context_estimated_cost(conf) + sw.estimated_count < _estimated_count + AND + sw.estimated_cost < _estimated_cost ) ) THEN @@ -1037,7 +518,6 @@ BEGIN estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, - partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; @@ -1046,18 +526,6 @@ END; $$ LANGUAGE PLPGSQL ; -CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ -DECLARE -cnt bigint; -BEGIN -EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; -RETURN cnt; -END; -$$ LANGUAGE PLPGSQL; - - - - DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( @@ -1071,41 +539,39 @@ DECLARE t timestamptz; i interval; BEGIN -SELECT * INTO search FROM searches -WHERE hash=search_hash(_search, _metadata) FOR UPDATE; + SELECT * INTO search FROM searches + WHERE hash=search_hash(_search, _metadata) FOR UPDATE; --- Calculate the where clause if not already calculated -IF search._where IS NULL THEN - search._where := cql_to_where(_search); -END IF; + -- Calculate the where clause if not already calculated + IF search._where IS NULL THEN + search._where := stac_search_to_where(_search); + END IF; --- Calculate the order by clause if not already calculated -IF search.orderby IS NULL THEN - search.orderby := sort_sqlorderby(_search); -END IF; + -- Calculate the order by clause if not already calculated + IF search.orderby IS NULL THEN + search.orderby := sort_sqlorderby(_search); + END IF; -PERFORM where_stats(search._where, updatestats, _search->'conf'); - -search.lastused := now(); -search.usecount := coalesce(search.usecount, 0) + 1; -INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) -VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) -ON CONFLICT (hash) DO -UPDATE SET - _where = EXCLUDED._where, - orderby = EXCLUDED.orderby, - lastused = EXCLUDED.lastused, - usecount = EXCLUDED.usecount, - metadata = EXCLUDED.metadata -RETURNING * INTO search -; -RETURN search; + PERFORM where_stats(search._where, updatestats, _search->'conf'); + + search.lastused := now(); + search.usecount := coalesce(search.usecount, 0) + 1; + INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) + VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) + ON CONFLICT (hash) DO + UPDATE SET + _where = EXCLUDED._where, + orderby = EXCLUDED.orderby, + lastused = EXCLUDED.lastused, + usecount = EXCLUDED.usecount, + metadata = EXCLUDED.metadata + RETURNING * INTO search + ; + RETURN search; END; $$ LANGUAGE PLPGSQL; - - CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; @@ -1119,8 +585,8 @@ DECLARE curs refcursor; cntr int := 0; iter_record items%ROWTYPE; - first_record items%ROWTYPE; - last_record items%ROWTYPE; + first_record jsonb; + last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; @@ -1147,9 +613,17 @@ CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN - FOR id IN SELECT jsonb_array_elements_text(_search->'ids') LOOP - INSERT INTO results (content) SELECT content FROM item_by_id(id) WHERE content IS NOT NULL; - END LOOP; + INSERT INTO results + SELECT content_hydrate(items, _search->'fields') + FROM items WHERE + items.id = ANY(to_text_array(_search->'ids')) + AND + CASE WHEN _search ? 'collections' THEN + items.collection = ANY(to_text_array(_search->'collections')) + ELSE TRUE + END + ORDER BY items.datetime desc, items.id desc + ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); @@ -1158,8 +632,6 @@ ELSE search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); - - IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); @@ -1172,9 +644,6 @@ ELSE RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); - - - FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); @@ -1186,14 +655,12 @@ ELSE FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; - last_record := iter_record; + last_record := content_hydrate(iter_record, _search->'fields'); IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN - INSERT INTO results (content) VALUES (last_record.content); - -- out_records := out_records || last_record.content; - + INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; @@ -1201,7 +668,7 @@ ELSE END IF; END LOOP; CLOSE curs; - RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); + RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; @@ -1226,7 +693,7 @@ IF _search ? 'token' THEN concat_ws( ' AND ', _where, - trim(get_token_filter(_search, to_jsonb(first_record))) + trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; @@ -1238,7 +705,6 @@ IF _search ? 'token' THEN END IF; has_prev := COALESCE(has_prev, FALSE); -RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; @@ -1246,21 +712,6 @@ IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; - --- include/exclude any fields following fields extension -IF _search ? 'fields' THEN - IF _search->'fields' ? 'exclude' THEN - excludes=textarr(_search->'fields'->'exclude'); - END IF; - IF _search->'fields' ? 'include' THEN - includes=textarr(_search->'fields'->'include'); - IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN - includes = includes || '{id}'; - END IF; - END IF; - SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; -END IF; - IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, @@ -1284,6 +735,42 @@ collection := jsonb_build_object( RETURN collection; END; -$$ LANGUAGE PLPGSQL -SET jit TO off -; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + +CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ +DECLARE + curs refcursor; + searches searches%ROWTYPE; + _where text; + _orderby text; + q text; + +BEGIN + searches := search_query(_search); + _where := searches._where; + _orderby := searches.orderby; + + OPEN curs FOR + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ) + )) + ELSE NULL + END FROM p; + RETURN curs; +END; +$$ LANGUAGE PLPGSQL; diff --git a/sql/006_tilesearch.sql b/sql/006_tilesearch.sql index d0c9a26a..9ae7dc6a 100644 --- a/sql/006_tilesearch.sql +++ b/sql/006_tilesearch.sql @@ -17,7 +17,7 @@ DECLARE _where text; query text; iter_record items%ROWTYPE; - out_records jsonb[] := '{}'::jsonb[]; + out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; @@ -31,6 +31,9 @@ DECLARE includes text[]; BEGIN + DROP TABLE IF EXISTS pgstac_results; + CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; + -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; @@ -45,26 +48,11 @@ BEGIN tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); - IF fields IS NOT NULL THEN - IF fields ? 'fields' THEN - fields := fields->'fields'; - END IF; - IF fields ? 'exclude' THEN - excludes=textarr(fields->'exclude'); - END IF; - IF fields ? 'include' THEN - includes=textarr(fields->'include'); - IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN - includes = includes || '{id}'; - END IF; - END IF; - END IF; - RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; - curs = create_cursor(query); + OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; @@ -88,12 +76,9 @@ BEGIN RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; + RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); + INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); - IF fields IS NOT NULL THEN - out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); - ELSE - out_records := out_records || iter_record.content; - END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit @@ -106,13 +91,16 @@ BEGIN scancounter := scancounter + 1; END LOOP; + CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; + SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; + RETURN jsonb_build_object( 'type', 'FeatureCollection', - 'features', array_to_json(out_records)::jsonb + 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; diff --git a/sql/998_permissions.sql b/sql/998_permissions.sql new file mode 100644 index 00000000..13d8696f --- /dev/null +++ b/sql/998_permissions.sql @@ -0,0 +1,13 @@ +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; diff --git a/sql/999_version.sql b/sql/999_version.sql index b5b92b95..6bae54c7 100644 --- a/sql/999_version.sql +++ b/sql/999_version.sql @@ -1 +1 @@ -SELECT set_version('0.4.5'); +SELECT set_version('0.5.0'); diff --git a/test/pgtap.sql b/test/pgtap.sql index 550fc380..e33dcab1 100644 --- a/test/pgtap.sql +++ b/test/pgtap.sql @@ -19,7 +19,7 @@ SET SEARCH_PATH TO pgstac, pgtap, public; SET CLIENT_MIN_MESSAGES TO 'warning'; -- Plan the tests. -SELECT plan(127); +SELECT plan(97); --SELECT * FROM no_plan(); -- Run the tests. diff --git a/test/pgtap/001_core.sql b/test/pgtap/001_core.sql index e063a019..221e84f9 100644 --- a/test/pgtap/001_core.sql +++ b/test/pgtap/001_core.sql @@ -7,12 +7,9 @@ SELECT has_extension('postgis'); SELECT has_table('pgstac'::name, 'migrations'::name); -SELECT has_function('pgstac'::name, 'textarr', ARRAY['jsonb']); +SELECT has_function('pgstac'::name, 'to_text_array', ARRAY['jsonb']); SELECT results_eq( - $$ SELECT textarr('["a","b","c"]'::jsonb) $$, + $$ SELECT to_text_array('["a","b","c"]'::jsonb) $$, $$ SELECT '{a,b,c}'::text[] $$, - 'textarr returns text[] from jsonb array' + 'to_text_array returns text[] from jsonb array' ); - - -SELECT has_function('pgstac'::name, 'estimated_count', ARRAY['text']); diff --git a/test/pgtap/001a_jsonutils.sql b/test/pgtap/001a_jsonutils.sql index 280bb79b..0bb0c674 100644 --- a/test/pgtap/001a_jsonutils.sql +++ b/test/pgtap/001a_jsonutils.sql @@ -1,26 +1,7 @@ -SELECT has_function('pgstac'::name, 'textarr', ARRAY['jsonb']); -SELECT has_function('pgstac'::name, 'jsonb_paths', ARRAY['jsonb']); -SELECT has_function('pgstac'::name, 'jsonb_obj_paths', ARRAY['jsonb']); -SELECT has_function('pgstac'::name, 'jsonb_val_paths', ARRAY['jsonb']); -SELECT has_function('pgstac'::name, 'path_includes', ARRAY['text[]', 'text[]']); -SELECT has_function('pgstac'::name, 'path_excludes', ARRAY['text[]', 'text[]']); -SELECT has_function('pgstac'::name, 'jsonb_obj_paths_filtered', ARRAY['jsonb','text[]','text[]']); -SELECT has_function('pgstac'::name, 'filter_jsonb', ARRAY['jsonb','text[]','text[]']); - +SELECT has_function('pgstac'::name, 'to_text_array', ARRAY['jsonb']); SELECT results_eq( - $$ SELECT textarr('["a","b","c"]'::jsonb) $$, + $$ SELECT to_text_array('["a","b","c"]'::jsonb) $$, $$ SELECT '{a,b,c}'::text[] $$, 'textarr returns text[] from jsonb array' ); - -SELECT results_eq( - $$ SELECT filter_jsonb('{"a":1,"b":2,"c":3}'::jsonb, includes=>'{a,c}') $$, - $$ SELECT '{"a":1,"c":3}'::jsonb $$, - 'filter_jsonb includes work' -); -SELECT results_eq( - $$ SELECT filter_jsonb('{"a":1,"b":2,"c":3}'::jsonb, includes=>NULL, excludes=>'{a,c}') $$, - $$ SELECT '{"b":2}'::jsonb $$, - 'filter_jsonb excludes work' -); diff --git a/test/pgtap/001b_cursorutils.sql b/test/pgtap/001b_cursorutils.sql index c94a4fc5..e69de29b 100644 --- a/test/pgtap/001b_cursorutils.sql +++ b/test/pgtap/001b_cursorutils.sql @@ -1,5 +0,0 @@ ---create_cursor -SELECT has_function('pgstac'::name, 'create_cursor', ARRAY['text']); - ---partition_cursor -SELECT has_function('pgstac'::name, 'partition_cursor', ARRAY['text', 'text']); diff --git a/test/pgtap/001s_stacutils.sql b/test/pgtap/001s_stacutils.sql index 98e6287a..591cd4ad 100644 --- a/test/pgtap/001s_stacutils.sql +++ b/test/pgtap/001s_stacutils.sql @@ -2,4 +2,3 @@ SELECT has_function('pgstac'::name, 'stac_geom', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_datetime', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_end_datetime', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_daterange', ARRAY['jsonb']); -SELECT has_function('pgstac'::name, 'properties_idx', ARRAY['jsonb']); diff --git a/test/pgtap/002_collections.sql b/test/pgtap/002_collections.sql index 9feebce3..fff43d66 100644 --- a/test/pgtap/002_collections.sql +++ b/test/pgtap/002_collections.sql @@ -1,5 +1,5 @@ SELECT has_table('pgstac'::name, 'collections'::name); -SELECT col_is_pk('pgstac'::name, 'collections'::name, 'id', 'collections has primary key'); +SELECT col_is_pk('pgstac'::name, 'collections'::name, 'key', 'collections has primary key'); SELECT has_function('pgstac'::name, 'create_collection', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'update_collection', ARRAY['jsonb']); diff --git a/test/pgtap/003_items.sql b/test/pgtap/003_items.sql index f416e852..84c42f30 100644 --- a/test/pgtap/003_items.sql +++ b/test/pgtap/003_items.sql @@ -2,17 +2,12 @@ SELECT has_table('pgstac'::name, 'items'::name); SELECT is_indexed('pgstac'::name, 'items'::name, 'geometry'); -SELECT is_indexed('pgstac'::name, 'items'::name, 'collection_id'); -SELECT is_indexed('pgstac'::name, 'items'::name, 'datetime'); -SELECT is_indexed('pgstac'::name, 'items'::name, 'end_datetime'); -SELECT is_indexed('pgstac'::name, 'items'::name, 'properties'); -SELECT is_indexed('pgstac'::name, 'items'::name, 'collection_id'); SELECT is_partitioned('pgstac'::name,'items'::name); -SELECT has_function('pgstac'::name, 'get_item', ARRAY['text']); -SELECT has_function('pgstac'::name, 'delete_item', ARRAY['text']); +SELECT has_function('pgstac'::name, 'get_item', ARRAY['text','text']); +SELECT has_function('pgstac'::name, 'delete_item', ARRAY['text','text']); SELECT has_function('pgstac'::name, 'create_item', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'update_item', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'upsert_item', ARRAY['jsonb']); @@ -20,14 +15,6 @@ SELECT has_function('pgstac'::name, 'create_items', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'upsert_items', ARRAY['jsonb']); - -SELECT has_function('pgstac'::name, 'analyze_empty_partitions', NULL); -SELECT has_function('pgstac'::name, 'items_update_triggerfunc', NULL); - - -SELECT has_view('pgstac'::name, 'all_items_partitions'::name, 'all_items_partitions view exists'); -SELECT has_view('pgstac'::name, 'items_partitions'::name, 'items_partitions view exists'); - -- tools to update collection extents based on extents in items SELECT has_function('pgstac'::name, 'collection_bbox', ARRAY['text']); SELECT has_function('pgstac'::name, 'collection_temporal_extent', ARRAY['text']); diff --git a/test/pgtap/004_search.sql b/test/pgtap/004_search.sql index 4fa2b260..c8e5bd40 100644 --- a/test/pgtap/004_search.sql +++ b/test/pgtap/004_search.sql @@ -6,13 +6,13 @@ DELETE FROM collections WHERE id = 'pgstac-test-collection'; SET pgstac.context TO 'on'; SET pgstac."default-filter-lang" TO 'cql-json'; -SELECT has_function('pgstac'::name, 'parse_dtrange', ARRAY['jsonb']); +SELECT has_function('pgstac'::name, 'parse_dtrange', ARRAY['jsonb','timestamptz']); -SELECT results_eq($$ SELECT parse_dtrange('["2020-01-01","2021-01-01"]') $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'daterange passed as array range'); +SELECT results_eq($$ SELECT parse_dtrange('["2020-01-01","2021-01-01"]'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'daterange passed as array range'); -SELECT results_eq($$ SELECT parse_dtrange('"2020-01-01/2021-01-01"') $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'date range passed as string range'); +SELECT results_eq($$ SELECT parse_dtrange('"2020-01-01/2021-01-01"'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'date range passed as string range'); SELECT has_function('pgstac'::name, 'bbox_geom', ARRAY['jsonb']); @@ -24,69 +24,22 @@ SELECT results_eq($$ SELECT bbox_geom('[0,1,2,3]') $$, $$ SELECT 'SRID=4326;POLY SELECT results_eq($$ SELECT bbox_geom('[0,1,2,3,4,5]'::jsonb) $$, $$ SELECT '010F0000A0E610000006000000010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000000000000000000104000000000000000400000000000000840000000000000104000000000000000400000000000000840000000000000F03F00000000000000400000000000000000000000000000F03F0000000000000040010300008001000000050000000000000000000000000000000000F03F00000000000014400000000000000840000000000000F03F00000000000014400000000000000840000000000000104000000000000014400000000000000000000000000000104000000000000014400000000000000000000000000000F03F0000000000001440010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000000000000000000F03F00000000000014400000000000000000000000000000104000000000000014400000000000000000000000000000104000000000000000400000000000000000000000000000F03F0000000000000040010300008001000000050000000000000000000840000000000000F03F00000000000000400000000000000840000000000000104000000000000000400000000000000840000000000000104000000000000014400000000000000840000000000000F03F00000000000014400000000000000840000000000000F03F0000000000000040010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000840000000000000F03F00000000000000400000000000000840000000000000F03F00000000000014400000000000000000000000000000F03F00000000000014400000000000000000000000000000F03F000000000000004001030000800100000005000000000000000000000000000000000010400000000000000040000000000000000000000000000010400000000000001440000000000000084000000000000010400000000000001440000000000000084000000000000010400000000000000040000000000000000000000000000010400000000000000040'::geometry $$, '3d bbox'); -SELECT has_function('pgstac'::name, 'add_filters_to_cql', ARRAY['jsonb']); - -SELECT results_eq($$ - SELECT add_filters_to_cql('{"ids":["a","b"]}'::jsonb); - $$,$$ - SELECT '{"filter":{"and": [{"in": [{"property": "id"}, ["a", "b"]]}]}}'::jsonb; - $$, - 'Test that id gets added to cql filter when cql filter does not exist' -); - -SELECT results_eq($$ - SELECT add_filters_to_cql('{"ids":["a","b"],"filter":{"and":[{"eq":[1,1]}]}}'::jsonb); - $$,$$ - SELECT '{"filter":{"and": [{"and": [{"eq": [1, 1]}]}, {"and": [{"in": [{"property": "id"}, ["a", "b"]]}]}]}}'::jsonb; - $$, - 'Test that id gets added to cql filter when cql filter does exist' -); - -SELECT results_eq($$ - SELECT add_filters_to_cql('{"collections":["a","b"]}'::jsonb); - $$,$$ - SELECT '{"filter":{"and": [{"in": [{"property": "collection"}, ["a", "b"]]}]}}'::jsonb; - $$, - 'Test that collections gets added to cql filter when cql filter does not exist' -); - -SELECT results_eq($$ - SELECT add_filters_to_cql('{"collection":["a","b"]}'::jsonb); - $$,$$ - SELECT '{"collection": ["a", "b"]}'::jsonb; - $$, - 'Test that collection are not added to cql filter' -); - - -SELECT has_function('pgstac'::name, 'cql_and_append', ARRAY['jsonb','jsonb']); - -SELECT has_function('pgstac'::name, 'query_to_cqlfilter', ARRAY['jsonb']); - -SELECT results_eq($$ - SELECT query_to_cqlfilter('{"query":{"a":{"gt":0,"lte":10},"b":"test"}}'); - $$,$$ - SELECT '{"filter":{"and": [{"gt": [{"property": "a"}, 0]}, {"lte": [{"property": "a"}, 10]}, {"eq": [{"property": "b"}, "test"]}]}}'::jsonb; - $$, - 'Test that query_to_cqlfilter appropriately converts old style query items to cql filters' -); - SELECT has_function('pgstac'::name, 'sort_sqlorderby', ARRAY['jsonb','boolean']); SELECT results_eq($$ - SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloudcover","direction":"asc"}]}'::jsonb); + SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb); $$,$$ - SELECT 'datetime DESC, properties->''eo:cloudcover'' ASC, id DESC'; + SELECT 'datetime DESC, to_int(content->''properties''->''eo:cloud_cover'') ASC, id DESC'; $$, 'Test creation of sort sql' ); SELECT results_eq($$ - SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloudcover","direction":"asc"}]}'::jsonb, true); + SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb, true); $$,$$ - SELECT 'datetime ASC, properties->''eo:cloudcover'' DESC, id ASC'; + SELECT 'datetime ASC, to_int(content->''properties''->''eo:cloud_cover'') DESC, id ASC'; $$, 'Test creation of reverse sort sql' ); @@ -168,7 +121,7 @@ SELECT results_eq($$ SELECT results_eq($$ select s from search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}') s; $$,$$ - select '{"next": null, "prev": null, "type": "FeatureCollection", "context": {"limit": 10, "matched": 2, "returned": 2}, "features": [{"id": "pgstac-test-item-0097"},{"id": "pgstac-test-item-0003"}]}'::jsonb + select '{"next": null, "prev": null, "type": "FeatureCollection", "context": {"limit": 10, "matched": 2, "returned": 2}, "features": [{"id": "pgstac-test-item-0003"},{"id": "pgstac-test-item-0097"}]}'::jsonb $$, 'Test ids search multi' ); @@ -227,7 +180,7 @@ SELECT results_eq($$ SELECT results_eq($$ - SELECT BTRIM(cql_to_where($q$ + SELECT BTRIM(stac_search_to_where($q$ { "intersects": { @@ -242,7 +195,7 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( TRUE ) AND st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340') + st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340') $r$,E' \n'); $$, 'Make sure that intersects returns valid query' ); @@ -263,7 +216,7 @@ SELECT results_eq($$ SELECT results_eq($$ select s from search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}') s; $$,$$ - select '{"next": null, "prev": null, "type": "FeatureCollection", "context": {"limit": 10, "matched": 2, "returned": 2}, "features": [{"id": "pgstac-test-item-0097"},{"id": "pgstac-test-item-0003"}]}'::jsonb + select '{"next": null, "prev": null, "type": "FeatureCollection", "context": {"limit": 10, "matched": 2, "returned": 2}, "features": [{"id": "pgstac-test-item-0003"},{"id": "pgstac-test-item-0097"}]}'::jsonb $$, 'Test ids search multi' ); @@ -294,7 +247,7 @@ SELECT results_eq($$ ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter": { "op" : "and", @@ -313,14 +266,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( (id = 'LC08_L1TP_060247_20180905_20180912_01_T1_L1TP') and (collection_id = 'landsat8_l1tp') ) + (id = 'LC08_L1TP_060247_20180905_20180912_01_T1_L1TP' AND collection = 'landsat8_l1tp') $r$,E' \n'); $$, 'Test Example 1' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -370,14 +323,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( (collection_id = 'landsat8_l1tp') and ( properties->>'eo:cloud_cover' <= '10') and (datetime >= '2021-04-08T04:39:23Z') and st_intersects(geometry, '0103000020E6100000010000000B000000894160E5D0CA4540ED9E3C2CD4E253C0849ECDAACFCD4540B37BF2B050DF53C038F8C264AAC8454076E09C11A5DD53C0F5DBD78173CE454085EB51B81ED953C08126C286A7CF4540789CA223B9D453C0C0EC9E3C2CD4454063EE5A423ED453C004560E2DB2E5454001DE02098AC753C063EE5A423EE84540C442AD69DEC953C02FDD240681ED454034A2B437F8CA53C08048BF7D1DE0454037894160E5E853C0894160E5D0CA4540ED9E3C2CD4E253C0'::geometry) ) + (collection = 'landsat8_l1tp' AND to_int(content->'properties'->'eo:cloud_cover') <= to_int('"10"') AND datetime >= '2021-04-08 04:39:23+00'::timestamptz AND st_intersects(geometry, '0103000020E6100000010000000B000000894160E5D0CA4540ED9E3C2CD4E253C0849ECDAACFCD4540B37BF2B050DF53C038F8C264AAC8454076E09C11A5DD53C0F5DBD78173CE454085EB51B81ED953C08126C286A7CF4540789CA223B9D453C0C0EC9E3C2CD4454063EE5A423ED453C004560E2DB2E5454001DE02098AC753C063EE5A423EE84540C442AD69DEC953C02FDD240681ED454034A2B437F8CA53C08048BF7D1DE0454037894160E5E853C0894160E5D0CA4540ED9E3C2CD4E253C0'::geometry)) $r$,E' \n'); $$, 'Test Example 2' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -397,7 +350,7 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( (( properties->>'sentinel:data_coverage' )::numeric > '50'::numeric) and (( properties->>'eo:cloud_cover' )::numeric < '10'::numeric) ) + (to_text(content->'properties'->'sentinel:data_coverage') > to_text('50') AND to_int(content->'properties'->'eo:cloud_cover') < to_int('10')) $r$,E' \n'); $$, 'Test Example 3' ); @@ -405,7 +358,7 @@ SELECT results_eq($$ SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -425,7 +378,7 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( (( properties->>'sentinel:data_coverage' )::numeric > '50'::numeric) or (( properties->>'eo:cloud_cover' )::numeric < '10'::numeric) ) + (to_text(content->'properties'->'sentinel:data_coverage') > to_text('50') OR to_int(content->'properties'->'eo:cloud_cover') < to_int('10')) $r$,E' \n'); $$, 'Test Example 4' ); @@ -433,7 +386,7 @@ SELECT results_eq($$ SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -447,14 +400,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( properties->>'prop1' = properties->>'prop2' ) + to_text(content->'properties'->'prop1') = to_text(content->'properties'->'prop2') $r$,E' \n'); $$, 'Test Example 5' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -476,7 +429,7 @@ SELECT results_eq($$ SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -505,7 +458,7 @@ SELECT results_eq($$ SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter": { "op": "or" , @@ -544,14 +497,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340'::geometry) or st_intersects(geometry, '0103000020E61000000100000005000000448B6CE7FBC553C014D044D8F064434060E5D022DBC153C014D044D8F064434060E5D022DBC153C0DE718A8EE46A4340448B6CE7FBC553C0DE718A8EE46A4340448B6CE7FBC553C014D044D8F0644340'::geometry) ) + (st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340'::geometry) OR st_intersects(geometry, '0103000020E61000000100000005000000448B6CE7FBC553C014D044D8F064434060E5D022DBC153C014D044D8F064434060E5D022DBC153C0DE718A8EE46A4340448B6CE7FBC553C0DE718A8EE46A4340448B6CE7FBC553C014D044D8F0644340'::geometry)) $r$,E' \n'); $$, 'Test Example 8' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -584,14 +537,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( (( properties->>'sentinel:data_coverage' )::numeric >= '50'::numeric) or (( properties->>'landsat:coverage_percent' )::numeric >= '50'::numeric) or ( ( properties->>'sentinel:data_coverage' IS NULL) and ( properties->>'landsat:coverage_percent' IS NULL) ) ) + (to_text(content->'properties'->'sentinel:data_coverage') >= to_text('50') OR to_text(content->'properties'->'landsat:coverage_percent') >= to_text('50') OR (to_text(content->'properties'->'sentinel:data_coverage') IS NULL AND to_text(content->'properties'->'landsat:coverage_percent') IS NULL)) $r$,E' \n'); $$, 'Test Example 9' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -605,14 +558,14 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - (( properties->>'eo:cloud_cover' )::numeric BETWEEN ('{0,50}'::numeric[])[1] AND ('{0,50}'::numeric[])[2]) + to_int(content->'properties'->'eo:cloud_cover') BETWEEN to_int('0') and to_int('50') $r$,E' \n'); $$, 'Test Example 10' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { @@ -626,47 +579,47 @@ SELECT results_eq($$ $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - ( properties->>'mission' LIKE 'sentinel%') + to_text(content->'properties'->'mission') LIKE to_text('"sentinel%"') $r$,E' \n'); $$, 'Test Example 11' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"upper": { "property": "mission" }}, - "sentinel" + {"upper": "sentinel"} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - (( upper( properties->>'mission' )) = 'sentinel') + upper(to_text(content->'properties'->'mission')) = upper('sentinel') $r$,E' \n'); $$, 'Test upper' ); SELECT results_eq($$ - SELECT BTRIM(cql2_query($q$ + SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"lower": { "property": "mission" }}, - "sentinel" + {"lower": "sentinel"} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ - (( lower( properties->>'mission' )) = 'sentinel') + lower(to_text(content->'properties'->'mission')) = lower('sentinel') $r$,E' \n'); $$, 'Test lower' ); diff --git a/test/testdata/collections.json b/test/testdata/collections.json new file mode 100644 index 00000000..d0a16298 --- /dev/null +++ b/test/testdata/collections.json @@ -0,0 +1,92 @@ +{ + "id": "pgstac-test-collection", + "stac_version": "1.0.0-beta.2", + "description": "The National Agriculture Imagery Program (NAIP) acquires aerial imagery\nduring the agricultural growing seasons in the continental U.S.\n\nNAIP projects are contracted each year based upon available funding and the\nFSA imagery acquisition cycle. Beginning in 2003, NAIP was acquired on\na 5-year cycle. 2008 was a transition year, and a three-year cycle began\nin 2009.\n\nNAIP imagery is acquired at a one-meter ground sample distance (GSD) with a\nhorizontal accuracy that matches within six meters of photo-identifiable\nground control points, which are used during image inspection.\n\nOlder images were collected using 3 bands (Red, Green, and Blue: RGB), but\nnewer imagery is usually collected with an additional near-infrared band\n(RGBN).", + "links": [ + { + "rel": "root", + "href": "/collection.json", + "type": "application/json" + }, + { + "rel": "self", + "href": "/collection.json", + "type": "application/json" + } + ], + "stac_extensions": [], + "title": "NAIP: National Agriculture Imagery Program", + "extent": { + "spatial": { + "bbox": [ + [ + -124.784, + 24.744, + -66.951, + 49.346 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2011-01-01T00:00:00Z", + "2019-01-01T00:00:00Z" + ] + ] + } + }, + "license": "PDDL-1.0", + "providers": [ + { + "name": "USDA Farm Service Agency", + "roles": [ + "producer", + "licensor" + ], + "url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/" + } + ], + "item_assets": { + "image": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ], + "title": "RGBIR COG tile", + "eo:bands": [ + { + "name": "Red", + "common_name": "red" + }, + { + "name": "Green", + "common_name": "green" + }, + { + "name": "Blue", + "common_name": "blue" + }, + { + "name": "NIR", + "common_name": "nir", + "description": "near-infrared" + } + ] + }, + "metadata": { + "type": "text/plain", + "roles": [ + "metadata" + ], + "title": "FGDC Metdata" + }, + "thumbnail": { + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail" + } + } +}