diff --git a/Dockerfile b/Dockerfile index 2a4da5d4b1..9e0ddc7d9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN python -m venv "$VIRTUAL_ENV" ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install --quiet pip==22.0.4 && \ pip install --quiet pip-tools -COPY ./dependencies/pip/external_services.txt "/tmp/pip_dependencies.txt" +COPY ./dependencies/pip/requirements.txt "/tmp/pip_dependencies.txt" RUN pip-sync "/tmp/pip_dependencies.txt" 1>/dev/null diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 5320249109..e70de424ac 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -12,10 +12,12 @@ # via -r dependencies/pip/requirements.in -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==5.1.0 +amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu +asgiref==3.5.0 + # via django asttokens==2.0.5 # via stack-data async-timeout==4.0.2 @@ -38,11 +40,11 @@ billiard==3.6.4.0 # via # -r dependencies/pip/requirements.in # celery -boto3==1.21.35 +boto3==1.22.2 # via # django-amazon-ses # django-storages -botocore==1.24.35 +botocore==1.25.2 # via # boto3 # s3transfer @@ -54,6 +56,7 @@ certifi==2021.10.8 # via # msrest # requests + # sentry-sdk cffi==1.15.0 # via # bcrypt @@ -80,7 +83,7 @@ coverage[toml]==6.3.2 # pytest-cov coveralls==3.3.1 # via -r dependencies/pip/dev_requirements.in -cryptography==36.0.2 +cryptography==37.0.1 # via # azure-storage-blob # jwcrypto @@ -104,7 +107,7 @@ dict2xml==1.7.1 # via -r dependencies/pip/requirements.in dj-static==0.0.6 # via -r dependencies/pip/requirements.in -django==2.2.27 +django==3.2.15 # via # -r dependencies/pip/requirements.in # django-amazon-ses @@ -157,7 +160,7 @@ django-markitup==4.0.0 # via -r dependencies/pip/requirements.in django-mptt==0.13.4 # via -r dependencies/pip/requirements.in -django-oauth-toolkit==1.7.1 +django-oauth-toolkit==2.0.0 # via -r dependencies/pip/requirements.in django-picklefield==3.0.1 # via django-constance @@ -171,7 +174,7 @@ django-registration-redux==2.10 # via -r dependencies/pip/requirements.in django-request-cache==1.2 # via -r dependencies/pip/requirements.in -django-reversion==3.0.1 +django-reversion==5.0.0 # via -r dependencies/pip/requirements.in django-storages[azure,boto3]==1.12.3 # via -r dependencies/pip/requirements.in @@ -229,7 +232,7 @@ jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==4.4.0 # via formpack -jwcrypto==1.0 +jwcrypto==1.2 # via django-oauth-toolkit kombu==5.2.4 # via @@ -268,7 +271,7 @@ packaging==21.3 # mongomock # pytest # redis -paramiko==2.10.3 +paramiko==2.10.4 # via fabric parso==0.8.3 # via jedi @@ -290,7 +293,7 @@ prompt-toolkit==3.0.29 # via # click-repl # ipython -psycopg2==2.8.6 +psycopg2==2.9.3 # via -r dependencies/pip/requirements.in ptyprocess==0.7.0 # via pexpect @@ -304,7 +307,7 @@ pyasn1==0.4.8 # ndg-httpsclient pycparser==2.21 # via cffi -pygments==2.11.2 +pygments==2.12.0 # via # -r dependencies/pip/requirements.in # ipython @@ -320,13 +323,13 @@ pyopenssl==22.0.0 # ndg-httpsclient pyotp==2.6.0 # via django-trench -pyparsing==3.0.7 +pyparsing==3.0.8 # via packaging pyquery==1.4.3 # via formpack pyrsistent==0.18.1 # via jsonschema -pytest==7.1.1 +pytest==7.1.2 # via # -r dependencies/pip/dev_requirements.in # pytest-cov @@ -381,6 +384,8 @@ s3transfer==0.5.2 # via boto3 sentinels==1.0.0 # via mongomock +sentry-sdk==1.5.12 + # via -r dependencies/pip/requirements.in shortuuid==1.0.8 # via -r dependencies/pip/requirements.in six==1.16.0 @@ -394,7 +399,7 @@ six==1.16.0 # paramiko # pathlib2 # python-dateutil -smsapi-client==2.5.0 +smsapi-client==2.6.0 # via django-trench sqlparse==0.4.2 # via @@ -419,7 +424,7 @@ traitlets==5.1.1 # via # ipython # matplotlib-inline -twilio==7.8.0 +twilio==7.8.2 # via django-trench typing-extensions==4.2.0 # via azure-core @@ -430,6 +435,7 @@ urllib3==1.26.9 # botocore # requests # responses + # sentry-sdk uwsgi==2.0.20 # via -r dependencies/pip/requirements.in vine==5.0.0 diff --git a/dependencies/pip/external_services.in b/dependencies/pip/external_services.in deleted file mode 100644 index bf9be01d46..0000000000 --- a/dependencies/pip/external_services.in +++ /dev/null @@ -1,5 +0,0 @@ -# File for use with `pip-compile`; see https://github.com/nvie/pip-tools - --r requirements.in - -sentry-sdk diff --git a/dependencies/pip/external_services.txt b/dependencies/pip/external_services.txt deleted file mode 100644 index 0e3b8adfb5..0000000000 --- a/dependencies/pip/external_services.txt +++ /dev/null @@ -1,371 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# pip-compile dependencies/pip/external_services.in -# --e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/formpack.git@7c5cf0596da54546b67cbb7fd4c486f2a7761c4e#egg=formpack - # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect - # via -r dependencies/pip/requirements.in -amqp==5.1.0 - # via - # -r dependencies/pip/requirements.in - # kombu -async-timeout==4.0.2 - # via redis -attrs==21.4.0 - # via jsonschema -azure-core==1.23.1 - # via azure-storage-blob -azure-storage-blob==12.11.0 - # via django-storages -begins==0.9 - # via formpack -billiard==3.6.4.0 - # via - # -r dependencies/pip/requirements.in - # celery -boto3==1.21.35 - # via - # django-amazon-ses - # django-storages -botocore==1.24.35 - # via - # boto3 - # s3transfer -celery[redis]==5.2.6 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat -certifi==2021.10.8 - # via - # msrest - # requests - # sentry-sdk -cffi==1.15.0 - # via cryptography -charset-normalizer==2.0.12 - # via requests -click==8.1.2 - # via - # celery - # click-didyoumean - # click-plugins - # click-repl -click-didyoumean==0.3.0 - # via celery -click-plugins==1.1.1 - # via celery -click-repl==0.2.0 - # via celery -cryptography==36.0.2 - # via - # azure-storage-blob - # jwcrypto - # pyopenssl -cssselect==1.1.0 - # via pyquery -deepmerge==1.0.1 - # via -r dependencies/pip/requirements.in -defusedxml==0.7.1 - # via - # djangorestframework-xml - # pyxform -deprecated==1.2.13 - # via - # jwcrypto - # redis -dict2xml==1.7.1 - # via -r dependencies/pip/requirements.in -dj-static==0.0.6 - # via -r dependencies/pip/requirements.in -django==2.2.27 - # via - # -r dependencies/pip/requirements.in - # django-amazon-ses - # django-braces - # django-celery-beat - # django-cors-headers - # django-csp - # django-debug-toolbar - # django-extensions - # django-filter - # django-js-asset - # django-markdownx - # django-oauth-toolkit - # django-picklefield - # django-redis - # django-request-cache - # django-reversion - # django-storages - # django-taggit - # django-timezone-field - # djangorestframework - # jsonfield -django-amazon-ses==4.0.1 - # via -r dependencies/pip/requirements.in -django-braces==1.15.0 - # via -r dependencies/pip/requirements.in -django-celery-beat==2.2.1 - # via -r dependencies/pip/requirements.in -django-constance[database]==2.8.0 - # via -r dependencies/pip/requirements.in -django-cors-headers==3.11.0 - # via -r dependencies/pip/requirements.in -django-csp==3.7 - # via -r dependencies/pip/requirements.in -django-debug-toolbar==3.2.4 - # via -r dependencies/pip/requirements.in -django-environ==0.8.1 - # via -r dependencies/pip/requirements.in -django-extensions==3.1.5 - # via -r dependencies/pip/requirements.in -django-filter==21.1 - # via -r dependencies/pip/requirements.in -django-js-asset==2.0.0 - # via django-mptt -django-loginas==0.3.10 - # via -r dependencies/pip/requirements.in -django-markdownx==3.0.1 - # via -r dependencies/pip/requirements.in -django-markitup==4.0.0 - # via -r dependencies/pip/requirements.in -django-mptt==0.13.4 - # via -r dependencies/pip/requirements.in -django-oauth-toolkit==1.7.1 - # via -r dependencies/pip/requirements.in -django-picklefield==3.0.1 - # via django-constance -django-private-storage==3.0 - # via -r dependencies/pip/requirements.in -django-redis==5.2.0 - # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.2 - # via -r dependencies/pip/requirements.in -django-registration-redux==2.10 - # via -r dependencies/pip/requirements.in -django-request-cache==1.2 - # via -r dependencies/pip/requirements.in -django-reversion==3.0.1 - # via -r dependencies/pip/requirements.in -django-storages[azure,boto3]==1.12.3 - # via -r dependencies/pip/requirements.in -django-taggit==2.1.0 - # via -r dependencies/pip/requirements.in -django-timezone-field==4.2.3 - # via django-celery-beat -django-trench==0.2.3 - # via -r dependencies/pip/requirements.in -django-userforeignkey==0.4.0 - # via django-request-cache -django-webpack-loader==1.5.0 - # via -r dependencies/pip/requirements.in -djangorestframework==3.13.1 - # via - # -r dependencies/pip/requirements.in - # drf-extensions -djangorestframework-xml==2.0.0 - # via -r dependencies/pip/requirements.in -docutils==0.18.1 - # via statistics -drf-extensions==0.7.1 - # via -r dependencies/pip/requirements.in -et-xmlfile==1.1.0 - # via openpyxl -future==0.18.2 - # via -r dependencies/pip/requirements.in -geojson-rewind==1.0.2 - # via - # -r dependencies/pip/requirements.in - # formpack -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via path-py -isodate==0.6.1 - # via msrest -jmespath==1.0.0 - # via - # boto3 - # botocore -jsonfield==3.1.0 - # via -r dependencies/pip/requirements.in -jsonschema==4.4.0 - # via formpack -jwcrypto==1.0 - # via django-oauth-toolkit -kombu==5.2.4 - # via - # -r dependencies/pip/requirements.in - # celery -lxml==4.8.0 - # via - # -r dependencies/pip/requirements.in - # formpack - # pyquery -markdown==3.3.6 - # via - # -r dependencies/pip/requirements.in - # django-markdownx -msrest==0.6.21 - # via azure-storage-blob -ndg-httpsclient==0.5.1 - # via -r dependencies/pip/requirements.in -oauthlib==3.2.0 - # via - # -r dependencies/pip/requirements.in - # django-oauth-toolkit - # requests-oauthlib -openpyxl==3.0.9 - # via - # -r dependencies/pip/requirements.in - # pyxform -packaging==21.3 - # via redis -path-py==12.0.2 - # via formpack -pillow==9.1.0 - # via django-markdownx -prompt-toolkit==3.0.29 - # via click-repl -psycopg2==2.8.6 - # via -r dependencies/pip/requirements.in -pyasn1==0.4.8 - # via - # -r dependencies/pip/requirements.in - # ndg-httpsclient -pycparser==2.21 - # via cffi -pygments==2.11.2 - # via -r dependencies/pip/requirements.in -pyjwt==2.3.0 - # via twilio -pymongo==3.12.3 - # via -r dependencies/pip/requirements.in -pyopenssl==22.0.0 - # via - # -r dependencies/pip/requirements.in - # ndg-httpsclient -pyotp==2.6.0 - # via django-trench -pyparsing==3.0.7 - # via packaging -pyquery==1.4.3 - # via formpack -pyrsistent==0.18.1 - # via jsonschema -python-crontab==2.6.0 - # via django-celery-beat -python-dateutil==2.8.2 - # via - # -r dependencies/pip/requirements.in - # botocore - # python-crontab -pytz==2022.1 - # via - # celery - # django - # django-timezone-field - # djangorestframework - # twilio -pyxform==1.9.0 - # via - # -r dependencies/pip/requirements.in - # formpack -redis==4.2.2 - # via - # celery - # django-redis - # django-redis-sessions -requests==2.27.1 - # via - # -r dependencies/pip/requirements.in - # azure-core - # django-oauth-toolkit - # msrest - # requests-oauthlib - # responses - # smsapi-client - # twilio - # yubico-client -requests-oauthlib==1.3.1 - # via msrest -responses==0.20.0 - # via -r dependencies/pip/requirements.in -s3transfer==0.5.2 - # via boto3 -sentry-sdk==1.5.8 - # via -r dependencies/pip/external_services.in -shortuuid==1.0.8 - # via -r dependencies/pip/requirements.in -six==1.16.0 - # via - # azure-core - # click-repl - # isodate - # python-dateutil -smsapi-client==2.5.0 - # via django-trench -sqlparse==0.4.2 - # via - # -r dependencies/pip/requirements.in - # django - # django-debug-toolbar -static3==0.7.0 - # via - # -r dependencies/pip/requirements.in - # dj-static -statistics==1.0.3.5 - # via formpack -tabulate==0.8.9 - # via -r dependencies/pip/requirements.in -twilio==7.8.0 - # via django-trench -typing-extensions==4.2.0 - # via azure-core -unicodecsv==0.14.1 - # via -r dependencies/pip/requirements.in -urllib3==1.26.9 - # via - # botocore - # requests - # responses - # sentry-sdk -uwsgi==2.0.20 - # via -r dependencies/pip/requirements.in -vine==5.0.0 - # via - # amqp - # celery - # kombu -wcwidth==0.2.5 - # via prompt-toolkit -werkzeug==2.0.3 - # via -r dependencies/pip/requirements.in -wrapt==1.14.0 - # via deprecated -xlrd==2.0.1 - # via - # -r dependencies/pip/requirements.in - # pyxform - # xlutils -xlsxwriter==3.0.3 - # via - # -r dependencies/pip/requirements.in - # formpack -xlutils==2.0.0 - # via -r dependencies/pip/requirements.in -xlwt==1.3.0 - # via - # -r dependencies/pip/requirements.in - # xlutils -yubico-client==1.13.0 - # via django-trench -zipp==3.8.0 - # via importlib-metadata -backports-zoneinfo==0.2.1; python_version < '3.9' diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 720ee4b84d..7c7bceac5d 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -13,7 +13,7 @@ -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # Regular PyPI packages -Django>=2.2,<2.3 +Django>=3.2,<3.3 Markdown Pygments amqp @@ -43,7 +43,7 @@ django-markdownx django-markitup django-mptt -django-reversion<3.0.2 # Migration issue with 3.0.2 +django-reversion django-taggit django-private-storage djangorestframework @@ -59,7 +59,7 @@ lxml oauthlib openpyxl #py-gfm # Incompatible with markdown 3.x -psycopg2>=2.8,<2.9 +psycopg2 pymongo==3.12.3 python-dateutil pyxform==1.9.0 @@ -88,5 +88,8 @@ deepmerge # MFA django-trench==0.2.3 +# Sentry +sentry-sdk + # Python 3.8 support backports.zoneinfo; python_version < '3.9' diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index ecfa7cf48c..17d0e892d0 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -12,10 +12,12 @@ # via -r dependencies/pip/requirements.in -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==5.1.0 +amqp==5.1.1 # via # -r dependencies/pip/requirements.in # kombu +asgiref==3.5.0 + # via django async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -30,11 +32,11 @@ billiard==3.6.4.0 # via # -r dependencies/pip/requirements.in # celery -boto3==1.21.35 +boto3==1.22.2 # via # django-amazon-ses # django-storages -botocore==1.24.35 +botocore==1.25.2 # via # boto3 # s3transfer @@ -46,6 +48,7 @@ certifi==2021.10.8 # via # msrest # requests + # sentry-sdk cffi==1.15.0 # via cryptography charset-normalizer==2.0.12 @@ -62,7 +65,7 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery -cryptography==36.0.2 +cryptography==37.0.1 # via # azure-storage-blob # jwcrypto @@ -83,7 +86,7 @@ dict2xml==1.7.1 # via -r dependencies/pip/requirements.in dj-static==0.0.6 # via -r dependencies/pip/requirements.in -django==2.2.27 +django==3.2.15 # via # -r dependencies/pip/requirements.in # django-amazon-ses @@ -136,7 +139,7 @@ django-markitup==4.0.0 # via -r dependencies/pip/requirements.in django-mptt==0.13.4 # via -r dependencies/pip/requirements.in -django-oauth-toolkit==1.7.1 +django-oauth-toolkit==2.0.0 # via -r dependencies/pip/requirements.in django-picklefield==3.0.1 # via django-constance @@ -150,7 +153,7 @@ django-registration-redux==2.10 # via -r dependencies/pip/requirements.in django-request-cache==1.2 # via -r dependencies/pip/requirements.in -django-reversion==3.0.1 +django-reversion==5.0.0 # via -r dependencies/pip/requirements.in django-storages[azure,boto3]==1.12.3 # via -r dependencies/pip/requirements.in @@ -194,7 +197,7 @@ jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==4.4.0 # via formpack -jwcrypto==1.0 +jwcrypto==1.2 # via django-oauth-toolkit kombu==5.2.4 # via @@ -232,7 +235,7 @@ pillow==9.1.0 # via django-markdownx prompt-toolkit==3.0.29 # via click-repl -psycopg2==2.8.6 +psycopg2==2.9.3 # via -r dependencies/pip/requirements.in pyasn1==0.4.8 # via @@ -240,7 +243,7 @@ pyasn1==0.4.8 # ndg-httpsclient pycparser==2.21 # via cffi -pygments==2.11.2 +pygments==2.12.0 # via -r dependencies/pip/requirements.in pyjwt==2.3.0 # via twilio @@ -252,7 +255,7 @@ pyopenssl==22.0.0 # ndg-httpsclient pyotp==2.6.0 # via django-trench -pyparsing==3.0.7 +pyparsing==3.0.8 # via packaging pyquery==1.4.3 # via formpack @@ -298,6 +301,8 @@ responses==0.20.0 # via -r dependencies/pip/requirements.in s3transfer==0.5.2 # via boto3 +sentry-sdk==1.5.12 + # via -r dependencies/pip/requirements.in shortuuid==1.0.8 # via -r dependencies/pip/requirements.in six==1.16.0 @@ -306,7 +311,7 @@ six==1.16.0 # click-repl # isodate # python-dateutil -smsapi-client==2.5.0 +smsapi-client==2.6.0 # via django-trench sqlparse==0.4.2 # via @@ -321,7 +326,7 @@ statistics==1.0.3.5 # via formpack tabulate==0.8.9 # via -r dependencies/pip/requirements.in -twilio==7.8.0 +twilio==7.8.2 # via django-trench typing-extensions==4.2.0 # via azure-core @@ -332,6 +337,7 @@ urllib3==1.26.9 # botocore # requests # responses + # sentry-sdk uwsgi==2.0.20 # via -r dependencies/pip/requirements.in vine==5.0.0 diff --git a/docker/init.bash b/docker/init.bash index 6c4e0656df..b05f5fd3b3 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -17,11 +17,11 @@ fi KPI_WEB_SERVER="${KPI_WEB_SERVER:-uWSGI}" if [[ "${KPI_WEB_SERVER,,}" == 'uwsgi' ]]; then # `diff` returns exit code 1 if it finds a difference between the files - if ! diff -q "${KPI_SRC_DIR}/dependencies/pip/external_services.txt" "/srv/tmp/pip_dependencies.txt" + if ! diff -q "${KPI_SRC_DIR}/dependencies/pip/requirements.txt" "/srv/tmp/pip_dependencies.txt" then echo "Syncing production pip dependencies..." - pip-sync dependencies/pip/external_services.txt 1>/dev/null - cp "dependencies/pip/external_services.txt" "/srv/tmp/pip_dependencies.txt" + pip-sync dependencies/pip/requirements.txt 1>/dev/null + cp "dependencies/pip/requirements.txt" "/srv/tmp/pip_dependencies.txt" fi else if ! diff -q "${KPI_SRC_DIR}/dependencies/pip/dev_requirements.txt" "/srv/tmp/pip_dependencies.txt" diff --git a/hub/migrations/0005_perusersetting.py b/hub/migrations/0005_perusersetting.py index f875a8b103..a228fa9077 100644 --- a/hub/migrations/0005_perusersetting.py +++ b/hub/migrations/0005_perusersetting.py @@ -1,5 +1,4 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models @@ -14,7 +13,7 @@ class Migration(migrations.Migration): name='PerUserSetting', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('user_queries', JSONBField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')), + ('user_queries', models.JSONField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')), ('name', models.CharField(unique=True, max_length=255)), ('value_when_matched', models.CharField(max_length=2048, blank=True)), ('value_when_not_matched', models.CharField(max_length=2048, blank=True)), diff --git a/hub/migrations/0007_alter_jsonfield_to_jsonbfield.py b/hub/migrations/0007_alter_jsonfield_to_jsonbfield.py index 64ea147845..e77d7034d7 100644 --- a/hub/migrations/0007_alter_jsonfield_to_jsonbfield.py +++ b/hub/migrations/0007_alter_jsonfield_to_jsonbfield.py @@ -1,6 +1,5 @@ # Generated by Django 2.2.7 on 2020-02-25 20:11 -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models @@ -14,11 +13,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='extrauserdetail', name='data', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='perusersetting', name='user_queries', - field=JSONBField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__iendswith": "@kobotoolbox.org"}, {"email__iendswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.'), + field=models.JSONField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__iendswith": "@kobotoolbox.org"}, {"email__iendswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.'), ), ] diff --git a/hub/models.py b/hub/models.py index 27ec00a050..450df848ca 100644 --- a/hub/models.py +++ b/hub/models.py @@ -1,7 +1,6 @@ # coding: utf-8 from django.conf import settings from django.contrib.auth.models import User -from django.contrib.postgres.fields import JSONField as JSONBField from django.core.exceptions import FieldError, ValidationError from django.urls import reverse from django.db import models @@ -63,7 +62,7 @@ class PerUserSetting(models.Model): A configuration setting that has different values depending on whether not a user matches certain criteria """ - user_queries = JSONBField( + user_queries = models.JSONField( help_text='A JSON representation of a *list* of Django queries, ' 'e.g. `[{"email__iendswith": "@kobotoolbox.org"}, ' '{"email__iendswith": "@kbtdev.org"}]`. ' @@ -115,7 +114,7 @@ class ExtraUserDetail(models.Model): user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='extra_details', on_delete=models.CASCADE) - data = JSONBField(default=dict) + data = models.JSONField(default=dict) def __str__(self): return '{}\'s data: {}'.format(self.user.__str__(), repr(self.data)) diff --git a/kobo/apps/help/migrations/0001_initial.py b/kobo/apps/help/migrations/0001_initial.py index 2e23a5d22c..b70d1fc66e 100644 --- a/kobo/apps/help/migrations/0001_initial.py +++ b/kobo/apps/help/migrations/0001_initial.py @@ -2,7 +2,6 @@ import datetime from django.conf import settings -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models import markdownx.models import private_storage.storage.files @@ -42,7 +41,7 @@ class Migration(migrations.Migration): name='InAppMessageUserInteractions', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('interactions', JSONBField(default=dict)), + ('interactions', models.JSONField(default=dict)), ('message', models.ForeignKey(to='help.InAppMessage', on_delete=models.CASCADE)), ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, diff --git a/kobo/apps/help/models.py b/kobo/apps/help/models.py index 7b4f35da47..e6f1aa710f 100644 --- a/kobo/apps/help/models.py +++ b/kobo/apps/help/models.py @@ -2,7 +2,6 @@ # 😇 import datetime -from django.contrib.postgres.fields import JSONField as JSONBField from django.conf import settings from django.db import models from django.utils.module_loading import import_string @@ -78,7 +77,7 @@ def __str__(self): class InAppMessageUserInteractions(models.Model): message = models.ForeignKey(InAppMessage, on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - interactions = JSONBField(default=dict) + interactions = models.JSONField(default=dict) class Meta: unique_together = ('message', 'user') diff --git a/kobo/apps/hook/migrations/0001_tables_creation.py b/kobo/apps/hook/migrations/0001_tables_creation.py index 5cbcc0898d..01f5e9ad86 100644 --- a/kobo/apps/hook/migrations/0001_tables_creation.py +++ b/kobo/apps/hook/migrations/0001_tables_creation.py @@ -1,5 +1,4 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models import django.utils.timezone @@ -23,7 +22,7 @@ class Migration(migrations.Migration): ('active', models.BooleanField(default=True)), ('export_type', models.CharField(default='json', max_length=10, choices=[('xml', 'xml'), ('json', 'json')])), ('auth_level', models.CharField(default='no_auth', max_length=10, choices=[('no_auth', 'no_auth'), ('basic_auth', 'basic_auth')])), - ('settings', JSONBField(default=dict)), + ('settings', models.JSONField(default=dict)), ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('date_modified', models.DateTimeField(default=django.utils.timezone.now)), ('asset', models.ForeignKey(related_name='hooks', to='kpi.Asset', on_delete=models.CASCADE)), diff --git a/kobo/apps/hook/models/hook.py b/kobo/apps/hook/models/hook.py index 613a9da6c4..46f4b5420c 100644 --- a/kobo/apps/hook/models/hook.py +++ b/kobo/apps/hook/models/hook.py @@ -2,7 +2,6 @@ from importlib import import_module from django.contrib.postgres.fields import ArrayField -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.utils import timezone @@ -39,7 +38,7 @@ class Hook(models.Model): active = models.BooleanField(default=True) export_type = models.CharField(choices=EXPORT_TYPE_CHOICES, default=JSON, max_length=10) auth_level = models.CharField(choices=AUTHENTICATION_LEVEL_CHOICES, default=NO_AUTH, max_length=10) - settings = JSONBField(default=dict) + settings = models.JSONField(default=dict) date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) email_notification = models.BooleanField(default=True) diff --git a/kobo/settings/base.py b/kobo/settings/base.py index bfb342ef0c..55cbd25e8b 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -491,6 +491,7 @@ def __init__(self, *args, **kwargs): 'django.template.context_processors.media', 'django.template.context_processors.static', 'django.template.context_processors.tz', + 'django.template.context_processors.request', 'django.contrib.messages.context_processors.messages', # Additional processors 'kpi.context_processors.external_service_tokens', @@ -956,3 +957,6 @@ def __init__(self, *args, **kwargs): MFA_SUPPORTED_AUTH_CLASSES = [ 'kpi.authentication.TokenAuthentication', ] + +# Django 3.2 required settings +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/kpi/deployment_backends/kc_access/shadow_models.py b/kpi/deployment_backends/kc_access/shadow_models.py index ad3c1c70c4..741d0318e4 100644 --- a/kpi/deployment_backends/kc_access/shadow_models.py +++ b/kpi/deployment_backends/kc_access/shadow_models.py @@ -5,7 +5,6 @@ from django.conf import settings from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.postgres.fields import JSONField as JSONBField from django.core import checks from django.core.exceptions import FieldDoesNotExist from django.core.files.base import ContentFile @@ -468,7 +467,7 @@ class Meta(ShadowModel.Meta): created_by = models.ForeignKey(KobocatUser, null=True, blank=True, on_delete=models.CASCADE) num_of_submissions = models.IntegerField(default=0) - metadata = JSONBField(default=dict, blank=True) + metadata = models.JSONField(default=dict, blank=True) # We need to cast `is_active` to an (positive small) integer because KoBoCAT # is using `LazyBooleanField` which is an integer behind the scene. # We do not want to port this class to KPI only for one line of code. diff --git a/kpi/deployment_backends/kc_access/storage.py b/kpi/deployment_backends/kc_access/storage.py index d2a9525c8b..c9d965a2dc 100644 --- a/kpi/deployment_backends/kc_access/storage.py +++ b/kpi/deployment_backends/kc_access/storage.py @@ -39,7 +39,8 @@ def __init__( class KobocatS3Boto3Storage(S3Boto3Storage): + def __init__(self, **settings): # This allows KoboCat to have a different bucket name, which is not recommended - settings["bucket_name"] = django_settings.KOBOCAT_AWS_STORAGE_BUCKET_NAME - super().__init__(**settings) \ No newline at end of file + settings['bucket_name'] = django_settings.KOBOCAT_AWS_STORAGE_BUCKET_NAME + super().__init__(**settings) diff --git a/kpi/deployment_backends/kobocat_backend.py b/kpi/deployment_backends/kobocat_backend.py index 1ff7b4954d..e116d44be7 100644 --- a/kpi/deployment_backends/kobocat_backend.py +++ b/kpi/deployment_backends/kobocat_backend.py @@ -72,7 +72,7 @@ class KobocatDeploymentBackend(BaseDeploymentBackend): """ Used to deploy a project into KoBoCAT. Stores the project identifiers in the - `self.asset._deployment_data` JSONBField (referred as "deployment data") + `self.asset._deployment_data` models.JSONField (referred as "deployment data") """ PROTECTED_XML_FIELDS = [ diff --git a/kpi/fields/lazy_default_jsonb.py b/kpi/fields/lazy_default_jsonb.py index c70d98ae19..baf92848b0 100644 --- a/kpi/fields/lazy_default_jsonb.py +++ b/kpi/fields/lazy_default_jsonb.py @@ -1,11 +1,11 @@ # coding: utf-8 from collections.abc import Callable +from django.db.models import JSONField from django.core.exceptions import FieldError -from django.contrib.postgres.fields import JSONField as JSONBField -class LazyDefaultJSONBField(JSONBField): +class LazyDefaultJSONBField(JSONField): """ Allows specifying a default value for a new field without having to rewrite every row in the corresponding table when migrating the database. @@ -42,7 +42,7 @@ def deconstruct(self): def from_db_value(self, value, *args, **kwargs): if value is None: return self._get_lazy_default() - return value + return super().from_db_value(value, *args, **kwargs) def pre_save(self, model_instance, add): value = getattr(model_instance, self.attname) diff --git a/kpi/migrations/0004_default_permissions_1910.py b/kpi/migrations/0004_default_permissions_1910.py index 66d0d2a70a..6e793fa5eb 100644 --- a/kpi/migrations/0004_default_permissions_1910.py +++ b/kpi/migrations/0004_default_permissions_1910.py @@ -9,9 +9,9 @@ def default_permissions_to_existing_users(apps, schema_editor): # The permissions objects might not have been created yet. See # https://code.djangoproject.com/ticket/23422. This is a workaround. for app_config in apps.get_app_configs(): - old_models_module = app_config.models_module - if app_config.models_module is None: - app_config.models_module = True # HACK HACK HACK + old_models_module = getattr(app_config, 'models_module', None) + if old_models_module is None: + app_config.models_module = True # HACK HACK HACK create_permissions( app_config=app_config, verbosity=0, @@ -46,11 +46,13 @@ def default_permissions_to_existing_users(apps, schema_editor): sys.stdout.write(progress_message) sys.stdout.flush() + def do_nothing(*args, **kwargs): ''' A no-op for reverse migration. Django 1.8 has RunPython.noop(), but 1.7 does not. ''' pass + class Migration(migrations.Migration): dependencies = [ diff --git a/kpi/migrations/0015_assetversion.py b/kpi/migrations/0015_assetversion.py index 348d0d868e..9d8aa215d1 100644 --- a/kpi/migrations/0015_assetversion.py +++ b/kpi/migrations/0015_assetversion.py @@ -1,6 +1,5 @@ # coding: utf-8 from django.conf import settings -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models from django.utils import timezone from jsonfield.fields import JSONField @@ -42,7 +41,7 @@ def noop(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('reversion', '0002_auto_20141216_1509'), + ('reversion', '0001_squashed_0004_auto_20160611_1202'), ('kpi', '0014_discoverable_subscribable_collections'), ] @@ -54,9 +53,9 @@ class Migration(migrations.Migration): ('uid', kpi.fields.KpiUidField(uid_prefix='v')), ('name', models.CharField(max_length=255, null=True)), ('date_modified', models.DateTimeField(default=timezone.now)), - ('version_content', JSONBField()), - ('deployed_content', JSONBField(null=True)), - ('_deployment_data', JSONBField(default=False)), + ('version_content', models.JSONField()), + ('deployed_content', models.JSONField(null=True)), + ('_deployment_data', models.JSONField(default=False)), ('deployed', models.BooleanField(default=False)), ('_reversion_version', models.OneToOneField(null=True, on_delete=models.SET_NULL, to='reversion.Version')), @@ -75,7 +74,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='asset', name='report_styles', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.RenameField( model_name='assetsnapshot', diff --git a/kpi/migrations/0016_asset_settings.py b/kpi/migrations/0016_asset_settings.py index 29b64d0881..9cf8caabd7 100644 --- a/kpi/migrations/0016_asset_settings.py +++ b/kpi/migrations/0016_asset_settings.py @@ -1,6 +1,5 @@ # coding: utf-8 -from django.db import migrations -from django.contrib.postgres.fields import JSONField as JSONBField +from django.db import migrations, models class Migration(migrations.Migration): @@ -13,6 +12,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='asset', name='settings', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), ] diff --git a/kpi/migrations/0017_assetversion_uid_aliases_20170608.py b/kpi/migrations/0017_assetversion_uid_aliases_20170608.py index a6538d7412..846c096f77 100644 --- a/kpi/migrations/0017_assetversion_uid_aliases_20170608.py +++ b/kpi/migrations/0017_assetversion_uid_aliases_20170608.py @@ -1,6 +1,5 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -13,6 +12,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='assetversion', name='uid_aliases', - field=JSONBField(null=True), + field=models.JSONField(null=True), ), ] diff --git a/kpi/migrations/0019_add_report_custom_field.py b/kpi/migrations/0019_add_report_custom_field.py index a5b5059200..0f354b5c09 100644 --- a/kpi/migrations/0019_add_report_custom_field.py +++ b/kpi/migrations/0019_add_report_custom_field.py @@ -1,6 +1,5 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -13,6 +12,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='asset', name='report_custom', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), ] diff --git a/kpi/migrations/0022_assetfile.py b/kpi/migrations/0022_assetfile.py index 43ceec07f2..fc8455477a 100644 --- a/kpi/migrations/0022_assetfile.py +++ b/kpi/migrations/0022_assetfile.py @@ -3,7 +3,6 @@ import private_storage.fields import private_storage.storage.files from django.conf import settings -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import migrations, models import kpi.fields @@ -28,7 +27,7 @@ class Migration(migrations.Migration): ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('content', private_storage.fields.PrivateFileField(storage=private_storage.storage.files.PrivateFileSystemStorage(), max_length=380, upload_to=kpi.models.asset_file.upload_to)), - ('metadata', JSONBField(default=dict)), + ('metadata', models.JSONField(default=dict)), ], ), # Why did `manage.py makemigrations` create these as separate operations? diff --git a/kpi/migrations/0023_partial_permissions.py b/kpi/migrations/0023_partial_permissions.py index 8b05ad6fe6..f9392fb2ee 100644 --- a/kpi/migrations/0023_partial_permissions.py +++ b/kpi/migrations/0023_partial_permissions.py @@ -1,5 +1,5 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField + from django.conf import settings from django.db import migrations, models import django.utils.timezone @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name='AssetUserPartialPermission', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('permissions', JSONBField(default=dict)), + ('permissions', models.JSONField(default=dict)), ('date_created', models.DateTimeField(default=django.utils.timezone.now)), ('date_modified', models.DateTimeField(default=django.utils.timezone.now)), ], diff --git a/kpi/migrations/0024_alter_jsonfield_to_jsonbfield.py b/kpi/migrations/0024_alter_jsonfield_to_jsonbfield.py index cea741da52..6bbef6dd40 100644 --- a/kpi/migrations/0024_alter_jsonfield_to_jsonbfield.py +++ b/kpi/migrations/0024_alter_jsonfield_to_jsonbfield.py @@ -1,14 +1,6 @@ # Generated by Django 2.2.7 on 2020-02-25 19:49 -import json -from django.contrib.postgres.fields.jsonb import JSONField as JSONBField from django.db import migrations, models -import kpi.fields.kpi_uid -import kpi.models.asset_file -import kpi.models.import_export_task -import private_storage.fields -import private_storage.storage.s3boto3 - JSON_NULL_CHAR = '\\u0000' PYTHON_NULL_CHAR = '\0' @@ -104,51 +96,51 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='asset', name='_deployment_data', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='asset', name='content', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='asset', name='summary', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='assetsnapshot', name='details', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='assetsnapshot', name='source', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='assetversion', name='_deployment_data', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='exporttask', name='data', - field=JSONBField(), + field=models.JSONField(), ), migrations.AlterField( model_name='exporttask', name='messages', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='importtask', name='data', - field=JSONBField(), + field=models.JSONField(), ), migrations.AlterField( model_name='importtask', name='messages', - field=JSONBField(default=dict), + field=models.JSONField(default=dict), ), ] diff --git a/kpi/migrations/0032_asset_export_settings_model.py b/kpi/migrations/0032_asset_export_settings_model.py index f831cd5e72..c2148c48ee 100644 --- a/kpi/migrations/0032_asset_export_settings_model.py +++ b/kpi/migrations/0032_asset_export_settings_model.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.7 on 2021-03-04 20:53 -import django.contrib.postgres.fields.jsonb -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + import kpi.fields.kpi_uid @@ -24,7 +24,7 @@ class Migration(migrations.Migration): ('uid', kpi.fields.kpi_uid.KpiUidField(uid_prefix='es')), ('date_modified', models.DateTimeField()), ('name', models.CharField(blank=True, default='', max_length=255)), - ('export_settings', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('export_settings', models.JSONField(default=dict)), ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='asset_export_settings', to='kpi.Asset')), ], options={ diff --git a/kpi/migrations/0040_synchronous_export.py b/kpi/migrations/0040_synchronous_export.py index 5575e8435a..ba9789557b 100644 --- a/kpi/migrations/0040_synchronous_export.py +++ b/kpi/migrations/0040_synchronous_export.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.7 on 2022-03-09 02:18 from django.conf import settings -import django.contrib.postgres.fields.jsonb from django.db import migrations, models import django.db.models.deletion import kpi.fields.kpi_uid @@ -23,8 +22,8 @@ class Migration(migrations.Migration): name='SynchronousExport', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data', django.contrib.postgres.fields.jsonb.JSONField()), - ('messages', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ('data', models.JSONField()), + ('messages', models.JSONField(default=dict)), ('status', models.CharField(choices=[('created', 'created'), ('processing', 'processing'), ('error', 'error'), ('complete', 'complete')], default='created', max_length=32)), ('date_created', models.DateTimeField(auto_now_add=True)), ('uid', kpi.fields.kpi_uid.KpiUidField(uid_prefix='e')), diff --git a/kpi/models/asset.py b/kpi/models/asset.py index 9ca1796c0e..2161c334b1 100644 --- a/kpi/models/asset.py +++ b/kpi/models/asset.py @@ -7,7 +7,6 @@ from django.conf import settings from django.contrib.auth.models import Permission -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.db import transaction from django.db.models import Exists, OuterRef, Prefetch, Q @@ -136,10 +135,10 @@ class Asset(ObjectPermissionMixin, name = models.CharField(max_length=255, blank=True, default='') date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - content = JSONBField(default=dict) - summary = JSONBField(default=dict) - report_styles = JSONBField(default=dict) - report_custom = JSONBField(default=dict) + content = models.JSONField(default=dict) + summary = models.JSONField(default=dict) + report_styles = models.JSONField(default=dict) + report_custom = models.JSONField(default=dict) map_styles = LazyDefaultJSONBField(default=dict) map_custom = LazyDefaultJSONBField(default=dict) asset_type = models.CharField( @@ -150,12 +149,12 @@ class Asset(ObjectPermissionMixin, on_delete=models.CASCADE) uid = KpiUidField(uid_prefix='a') tags = TaggableManager(manager=KpiTaggableManager) - settings = JSONBField(default=dict) + settings = models.JSONField(default=dict) # `_deployment_data` must **NOT** be touched directly by anything except # the `deployment` property provided by `DeployableMixin`. # ToDo Move the field to another table with one-to-one relationship - _deployment_data = JSONBField(default=dict) + _deployment_data = models.JSONField(default=dict) # JSON with subset of fields to share # { diff --git a/kpi/models/asset_export_settings.py b/kpi/models/asset_export_settings.py index 4b664a28f7..22293efe64 100644 --- a/kpi/models/asset_export_settings.py +++ b/kpi/models/asset_export_settings.py @@ -1,5 +1,4 @@ # coding: utf-8 -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.utils import timezone @@ -16,7 +15,7 @@ class AssetExportSettings(models.Model): on_delete=models.CASCADE) date_modified = models.DateTimeField() name = models.CharField(max_length=255, blank=True, default='') - export_settings = JSONBField(default=dict) + export_settings = models.JSONField(default=dict) def save(self, *args, **kwargs): self.date_modified = timezone.now() @@ -28,4 +27,3 @@ class Meta: def __str__(self): return f'{self.name} ({self.uid})' - diff --git a/kpi/models/asset_file.py b/kpi/models/asset_file.py index c34f013bd2..df3ef8b936 100644 --- a/kpi/models/asset_file.py +++ b/kpi/models/asset_file.py @@ -3,7 +3,6 @@ from mimetypes import guess_type from typing import Optional -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.utils import timezone from private_storage.fields import PrivateFileField @@ -102,7 +101,7 @@ class AssetFile(models.Model, AbstractFormMedia): description = models.CharField(max_length=255) date_created = models.DateTimeField(default=timezone.now) content = PrivateFileField(upload_to=upload_to, max_length=380, null=True) - metadata = JSONBField(default=dict) + metadata = models.JSONField(default=dict) date_deleted = models.DateTimeField(null=True, default=None) date_modified = models.DateTimeField(default=timezone.now) synced_with_backend = models.BooleanField(default=False) diff --git a/kpi/models/asset_snapshot.py b/kpi/models/asset_snapshot.py index 8d8e5f0203..18ab3f7e5f 100644 --- a/kpi/models/asset_snapshot.py +++ b/kpi/models/asset_snapshot.py @@ -2,7 +2,6 @@ # 😬 import copy -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from rest_framework.reverse import reverse @@ -63,8 +62,8 @@ class AssetSnapshot( Remove above lines when PR is merged """ xml = models.TextField() - source = JSONBField(default=dict) - details = JSONBField(default=dict) + source = models.JSONField(default=dict) + details = models.JSONField(default=dict) owner = models.ForeignKey('auth.User', related_name='asset_snapshots', null=True, on_delete=models.CASCADE) asset = models.ForeignKey('Asset', null=True, on_delete=models.CASCADE) diff --git a/kpi/models/asset_user_partial_permission.py b/kpi/models/asset_user_partial_permission.py index f3c6d3e5df..07ff0097cb 100644 --- a/kpi/models/asset_user_partial_permission.py +++ b/kpi/models/asset_user_partial_permission.py @@ -1,7 +1,6 @@ # coding: utf-8 from collections import defaultdict -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.utils import timezone @@ -38,7 +37,7 @@ class Meta: on_delete=models.CASCADE) user = models.ForeignKey('auth.User', related_name='user_partial_permissions', on_delete=models.CASCADE) - permissions = JSONBField(default=dict) + permissions = models.JSONField(default=dict) date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) diff --git a/kpi/models/asset_version.py b/kpi/models/asset_version.py index 81f33f7a8e..fe17325172 100644 --- a/kpi/models/asset_version.py +++ b/kpi/models/asset_version.py @@ -2,7 +2,6 @@ import datetime import json -from django.contrib.postgres.fields import JSONField as JSONBField from django.db import models from django.utils import timezone from formpack.utils.expand_content import expand_content @@ -28,10 +27,10 @@ class AssetVersion(models.Model): null=True, on_delete=models.SET_NULL, ) - version_content = JSONBField() - uid_aliases = JSONBField(null=True) - deployed_content = JSONBField(null=True) - _deployment_data = JSONBField(default=dict) + version_content = models.JSONField() + uid_aliases = models.JSONField(null=True) + deployed_content = models.JSONField(null=True) + _deployment_data = models.JSONField(default=dict) deployed = models.BooleanField(default=False) class Meta: diff --git a/kpi/models/import_export_task.py b/kpi/models/import_export_task.py index b3de9a46da..4e4e357435 100644 --- a/kpi/models/import_export_task.py +++ b/kpi/models/import_export_task.py @@ -17,7 +17,6 @@ import constance import requests from django.conf import settings -from django.contrib.postgres.fields import JSONField as JSONBField from django.core.files.base import ContentFile from django.db import models, transaction from django.urls import reverse @@ -98,8 +97,8 @@ class Meta: ) user = models.ForeignKey('auth.User', on_delete=models.CASCADE) - data = JSONBField() - messages = JSONBField(default=dict) + data = models.JSONField() + messages = models.JSONField(default=dict) status = models.CharField(choices=STATUS_CHOICES, max_length=32, default=CREATED) date_created = models.DateTimeField(auto_now_add=True) diff --git a/kpi/tests/api/v1/test_api_assets.py b/kpi/tests/api/v1/test_api_assets.py index ab0cab011b..dade3bf601 100644 --- a/kpi/tests/api/v1/test_api_assets.py +++ b/kpi/tests/api/v1/test_api_assets.py @@ -1,6 +1,7 @@ # coding: utf-8 import json import unittest +from urllib.parse import unquote_plus from django.contrib.auth.models import User from django.urls import reverse @@ -10,14 +11,15 @@ from rest_framework.authtoken.models import Token from kpi.constants import ASSET_TYPE_COLLECTION -from kpi.models import Asset -from kpi.models import ExportTask +from kpi.models import Asset, ExportTask +from kpi.models.import_export_task import export_upload_to from kpi.serializers.v1.asset import AssetListSerializer # importing module instead of the class, avoid running the tests twice from kpi.tests.api.v2 import test_api_assets from kpi.tests.base_test_case import BaseTestCase from kpi.tests.kpi_test_case import KpiTestCase + EMPTY_SURVEY = {'survey': [], 'schema': SCHEMA_VERSION, 'settings': {}} @@ -329,7 +331,12 @@ def test_owner_with_token_auth_can_access_export(self): def test_owner_can_create_and_delete_export(self): detail_response = self.test_owner_can_create_export() result_response = self.client.get(detail_response.data['result']) - file_name = str(result_response._closable_objects[0].name) + file_name = unquote_plus( + result_response.headers['Content-Disposition'].replace( + "inline; filename*=utf-8''", '' + ) + ) + file_path = export_upload_to(self, file_name) detail_url = reverse('exporttask-detail', kwargs={ 'uid': detail_response.data['uid'] @@ -337,7 +344,7 @@ def test_owner_can_create_and_delete_export(self): # checking if file exists before attempting to delete file_exists_before_delete = ExportTask.result.field.storage.exists( - name=file_name + name=file_path ) assert file_exists_before_delete @@ -347,7 +354,7 @@ def test_owner_can_create_and_delete_export(self): # checking if file still exists after attempting to delete it file_exists_after_delete = ExportTask.result.field.storage.exists( - name=file_name + name=file_path ) assert not file_exists_after_delete diff --git a/kpi/utils/jsonbfield_helper.py b/kpi/utils/jsonbfield_helper.py index f8a00ba751..767ee0c9dd 100644 --- a/kpi/utils/jsonbfield_helper.py +++ b/kpi/utils/jsonbfield_helper.py @@ -7,7 +7,7 @@ class ReplaceValues(Func): """ - Updates several properties at once of a JSONBField without overwriting the + Updates several properties at once of a models.JSONField without overwriting the whole document. Avoids race conditions when document is saved in two different transactions at the same time. (i.e.: `Asset._deployment['status']`)