From d673457526d29a542f0798c9c06ef9708388ec85 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 09:44:46 +0200 Subject: [PATCH 01/16] Add first attempt at getting mssql support working --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d2f4423..2a251e69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,9 @@ jobs: strategy: matrix: - python-version: ["3.7", "3.8"] - django-version: ["2.1.1", "3.1.4"] - database-engine: ["postgres", "mysql"] + python-version: [ "3.7", "3.8" ] + django-version: [ "2.1.1", "3.1.4" ] + database-engine: [ "postgres", "mysql", "mssql"] services: postgres: @@ -37,6 +37,14 @@ jobs: ports: - 3306:3306 + + mssqldb: + image: mcr.microsoft.com/mssql/server:2017-latest + env: + ACCEPT_EULA: y + SA_PASSWORD: Test + + steps: - name: Checkout code uses: actions/checkout@v2 From 620d80ddd09a6c02a74a213c5f138794b1f4dd85 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 09:59:13 +0200 Subject: [PATCH 02/16] quotes around image --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a251e69..825feb54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: mssqldb: - image: mcr.microsoft.com/mssql/server:2017-latest + image: "mcr.microsoft.com/mssql/server:2017-latest" env: ACCEPT_EULA: y SA_PASSWORD: Test From d322be070a52aef9eca36631b22ec2aa45b0df8a Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:01:11 +0200 Subject: [PATCH 03/16] expose mssqldb port --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 825feb54..3e966553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,11 @@ jobs: mssqldb: image: "mcr.microsoft.com/mssql/server:2017-latest" - env: - ACCEPT_EULA: y - SA_PASSWORD: Test - + env: + ACCEPT_EULA: y + SA_PASSWORD: Test + ports: + - 1433:1433 steps: - name: Checkout code From 7b175b264d36d636c9590136c4b9b1804172c8f8 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:18:56 +0200 Subject: [PATCH 04/16] add prepare mssql database script --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e966553..d3a69f81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,11 @@ jobs: mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql --protocol=TCP -h localhost -u root -prootpassword mysql if: matrix.database-engine == 'mysql' + - name: Prepare mssql database + run: | + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' -h localhost -U postgres + if: matrix.database-engine == 'mssql' + - name: Run tests run: | .venv/bin/coverage run --include="binder/*" setup.py test From 4adc5040d65e63f5dd976d880b980dee9f6eea68 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:22:58 +0200 Subject: [PATCH 05/16] remove double options --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3a69f81..03efc2cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: - name: Prepare mssql database run: | - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' -h localhost -U postgres + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' if: matrix.database-engine == 'mssql' - name: Run tests From c49af3b6d252bcb6401db35179752e05f35d0f79 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:26:27 +0200 Subject: [PATCH 06/16] fix capaltilization error in password --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03efc2cc..b65184df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: image: "mcr.microsoft.com/mssql/server:2017-latest" env: ACCEPT_EULA: y - SA_PASSWORD: Test + SA_PASSWORD: test ports: - 1433:1433 @@ -84,7 +84,7 @@ jobs: - name: Prepare mssql database run: | - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' if: matrix.database-engine == 'mssql' - name: Run tests From b4349c9a8da5f29734205455834c209939b379c7 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:27:52 +0200 Subject: [PATCH 07/16] temporarily remove not interesting versions to speed up CI --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b65184df..f61066a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,13 @@ jobs: strategy: matrix: - python-version: [ "3.7", "3.8" ] - django-version: [ "2.1.1", "3.1.4" ] - database-engine: [ "postgres", "mysql", "mssql"] + python-version: [ "3.8" ] + django-version: [ "3.1.4" ] + database-engine: ["mssql" ] +# +# python-version: [ "3.7", "3.8" ] +# django-version: [ "2.1.1", "3.1.4" ] +# database-engine: [ "postgres", "mysql", "mssql"] services: postgres: From e39c1ff7e96db05eb8b232600dc1f116ce77f2ba Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:38:40 +0200 Subject: [PATCH 08/16] make mssql db password stronger --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f61066a7..a212cccd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: image: "mcr.microsoft.com/mssql/server:2017-latest" env: ACCEPT_EULA: y - SA_PASSWORD: test + SA_PASSWORD: '~Test123' ports: - 1433:1433 @@ -88,7 +88,7 @@ jobs: - name: Prepare mssql database run: | - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P test -c 'CREATE DATABASE "binder-test"; go' + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -c 'CREATE DATABASE "binder-test"; go' if: matrix.database-engine == 'mssql' - name: Run tests From 42c9fdeff5ccd74238c091935b5e11245fed6907 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:47:39 +0200 Subject: [PATCH 09/16] update create database script for sqlcmd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a212cccd..6849c55c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: - name: Prepare mssql database run: | - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -c 'CREATE DATABASE "binder-test"; go' + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -Q 'CREATE DATABASE [binder-test];' if: matrix.database-engine == 'mssql' - name: Run tests From 88b087afd8081f048758108fe070bf34de28ccb8 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:53:13 +0200 Subject: [PATCH 10/16] rewrite database engine logic such that it is not a binary 'ismysql' --- .github/workflows/ci.yml | 2 +- setup.py | 2 +- tests/__init__.py | 2 +- tests/test_postgres_fields.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6849c55c..0cd369a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: run: | .venv/bin/coverage run --include="binder/*" setup.py test env: - BINDER_TEST_MYSQL: ${{ matrix.database-engine == 'mysql' && 1 || 0 }} + BINDER_TEST_DATABASE_ENGINE: ${{ matrix.database-engine }} CY_RUNNING_INSIDE_CI: 1 - name: Upload coverage report diff --git a/setup.py b/setup.py index 31a0f90f..350c1229 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'django-hijack >= 2.1.10', ( 'mysqlclient >= 1.3.12' - if os.environ.get('BINDER_TEST_MYSQL', '0') == '1' else + if os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') == 'mysql' else 'psycopg2 >= 2.7' ), "openpyxl >= 3.0.0" diff --git a/tests/__init__.py b/tests/__init__.py index 4dcaf8bd..987f0d69 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,7 +14,7 @@ 'HOST': 'db', 'PORT': 5432, } -elif os.environ.get('BINDER_TEST_MYSQL', '0') == '1': +elif os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') == 'mysql': db_settings = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'binder-test', diff --git a/tests/test_postgres_fields.py b/tests/test_postgres_fields.py index 4713d218..d45fae44 100644 --- a/tests/test_postgres_fields.py +++ b/tests/test_postgres_fields.py @@ -7,7 +7,7 @@ from binder.json import jsonloads from django.contrib.auth.models import User -if os.environ.get('BINDER_TEST_MYSQL', '0') == '0': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') != 'mysql': from .testapp.models import FeedingSchedule, Animal, Zoo # TODO: Currently these only really test filtering. Move to test/filters? From 77ba0ceda75844abe11a764535d7a0dcac3eb194 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 10:56:44 +0200 Subject: [PATCH 11/16] database engine selection wip --- setup.py | 13 ++++++++----- tests/__init__.py | 2 +- tests/test_postgres_fields.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 350c1229..e6244010 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,13 @@ # allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) +test_require_database_engine = { + 'mssql': 'mysqlclient >= 1.3.12', + 'mysql': 'django-pyodbc >= 1.1.3' +}.get('BINDER_TEST_DATABASE_ENGINE', 'psycopg2 >= 2.7') + + + setup( name='django-binder', version='1.5.0', @@ -44,11 +51,7 @@ ], tests_require=[ 'django-hijack >= 2.1.10', - ( - 'mysqlclient >= 1.3.12' - if os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') == 'mysql' else - 'psycopg2 >= 2.7' - ), + test_require_database_engine, "openpyxl >= 3.0.0" ], ) diff --git a/tests/__init__.py b/tests/__init__.py index 987f0d69..20c7b56f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,7 +14,7 @@ 'HOST': 'db', 'PORT': 5432, } -elif os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') == 'mysql': +elif os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mysql': db_settings = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'binder-test', diff --git a/tests/test_postgres_fields.py b/tests/test_postgres_fields.py index d45fae44..6be4b829 100644 --- a/tests/test_postgres_fields.py +++ b/tests/test_postgres_fields.py @@ -7,7 +7,7 @@ from binder.json import jsonloads from django.contrib.auth.models import User -if os.environ.get('BINDER_TEST_DATABASE_ENGINE', ') != 'mysql': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') != 'mysql': from .testapp.models import FeedingSchedule, Animal, Zoo # TODO: Currently these only really test filtering. Move to test/filters? From 21a72510071146d14c1959a73e14d7a77e71bcfb Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 11:13:09 +0200 Subject: [PATCH 12/16] add support for mysql / mssql in local test setup --- README.md | 10 +++++++++- docker-compose.yml | 13 +++++++++++++ tests/__init__.py | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da9d2020..7c5527da 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,19 @@ There are two ways to run the tests: - Run with docker `docker-compose run binder ./setup.py test` - Access the test database directly by with `docker-compose run db psql -h db -U postgres`. - It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker-compose rm`. The database will be created during the setup in `tests/__init__.py`. - + The tests are set up in such a way that there is no need to keep migration files. The setup procedure in `tests/__init__.py` handles the preparation of the database by directly calling some build-in Django commands. To only run a selection of the tests, use the `-s` flag like `./setup.py test -s tests.test_some_specific_test`. +### Running test other database systems +Locally the tests can be run for MySQL or MSSQL by overriding the `BINDER_TEST_DATABASE_ENGINE` environment variable: + +- `docker-compose run -e BINDER_TEST_DATABASE_ENGINE=mysql binder ./setup.py test` for MySQL +- `docker-compose run -e BINDER_TEST_DATABASE_ENGINE=mssql binder ./setup.py test` for MsSQL + + + ## MySQL support MySQL is supported, but only with the goal to replace it with diff --git a/docker-compose.yml b/docker-compose.yml index 5e890a8a..e48d9dcc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,3 +10,16 @@ services: - .:/binder depends_on: - db + mysql: + image: mysql + environment: + MYSQL_ROOT_PASSWORD: rootpassword + ports: + - 3306:3306 + mssqldb: + image: "mcr.microsoft.com/mssql/server:2017-latest" + environment: + ACCEPT_EULA: y + SA_PASSWORD: '~Test123' + ports: + - 1433:1433 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 20c7b56f..3e25664b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,6 +3,7 @@ from django.core.management import call_command import os + if ( os.path.exists('/.dockerenv') and 'CY_RUNNING_INSIDE_CI' not in os.environ From f53e0cafc70a316a48be7543f4322c591b2a56f9 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 11:18:49 +0200 Subject: [PATCH 13/16] fix test only for mssql support --- tests/test_postgres_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_postgres_fields.py b/tests/test_postgres_fields.py index 6be4b829..54e23555 100644 --- a/tests/test_postgres_fields.py +++ b/tests/test_postgres_fields.py @@ -12,7 +12,7 @@ # TODO: Currently these only really test filtering. Move to test/filters? @unittest.skipIf( - os.environ.get('BINDER_TEST_MYSQL', '0') != '0', + os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mysql'], "Only available with PostgreSQL" ) class PostgresFieldsTest(TestCase): From f4db430448f77750f2fbf05b56e3eee1074c7d99 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 14:37:25 +0200 Subject: [PATCH 14/16] working mssql setup locally. --- docker-compose.yml | 10 +++++-- Dockerfile => docker/binder/Dockerfile | 3 ++- docker/mssqldb/Dockerfile | 12 +++++++++ docker/mssqldb/bootstrap.sql | 4 +++ docker/mssqldb/entrypoint.sh | 5 ++++ docker/mssqldb/init | 7 +++++ install-mssql-driver.sh | 14 ++++++++++ project/packages.pip | 4 ++- setup.py | 9 +++---- tests/__init__.py | 37 +++++++++++++++++--------- tests/test_nulls_last.py | 4 +-- tests/test_postgres_fields.py | 4 +-- tests/testapp/models/__init__.py | 2 +- tests/testapp/views/__init__.py | 2 +- 14 files changed, 90 insertions(+), 27 deletions(-) rename Dockerfile => docker/binder/Dockerfile (58%) create mode 100644 docker/mssqldb/Dockerfile create mode 100644 docker/mssqldb/bootstrap.sql create mode 100755 docker/mssqldb/entrypoint.sh create mode 100755 docker/mssqldb/init create mode 100755 install-mssql-driver.sh diff --git a/docker-compose.yml b/docker-compose.yml index e48d9dcc..dcdacd3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,12 +4,16 @@ services: db: image: postgres:11.5 binder: - build: . + build: + context: ./ + dockerfile: docker/binder/Dockerfile command: tail -f /dev/null volumes: - .:/binder depends_on: - db + - mysql + - mssqldb mysql: image: mysql environment: @@ -17,7 +21,9 @@ services: ports: - 3306:3306 mssqldb: - image: "mcr.microsoft.com/mssql/server:2017-latest" + build: + context: ./ + dockerfile: docker/mssqldb/Dockerfile environment: ACCEPT_EULA: y SA_PASSWORD: '~Test123' diff --git a/Dockerfile b/docker/binder/Dockerfile similarity index 58% rename from Dockerfile rename to docker/binder/Dockerfile index 41f16d7d..bc1c2d94 100644 --- a/Dockerfile +++ b/docker/binder/Dockerfile @@ -1,5 +1,6 @@ -FROM python:3 +FROM python:3-buster ENV PYTHONUNBUFFERED 1 RUN mkdir /binder WORKDIR /binder ADD . /binder +RUN ./install-mssql-driver.sh diff --git a/docker/mssqldb/Dockerfile b/docker/mssqldb/Dockerfile new file mode 100644 index 00000000..2e1ed3ab --- /dev/null +++ b/docker/mssqldb/Dockerfile @@ -0,0 +1,12 @@ +FROM mcr.microsoft.com/mssql/server:2017-latest +WORKDIR / + +ENV ACCEPT_EULA Y +ENV SA_PASSWORD "~Test123" + +COPY docker/mssqldb/init /init +COPY docker/mssqldb/bootstrap.sql /bootstrap.sql +COPY docker/mssqldb/entrypoint.sh /entrypoint.sh + +EXPOSE 1433 +CMD /entrypoint.sh \ No newline at end of file diff --git a/docker/mssqldb/bootstrap.sql b/docker/mssqldb/bootstrap.sql new file mode 100644 index 00000000..9b64f644 --- /dev/null +++ b/docker/mssqldb/bootstrap.sql @@ -0,0 +1,4 @@ +USE [master] +GO +CREATE DATABASE [binder-test] +GO \ No newline at end of file diff --git a/docker/mssqldb/entrypoint.sh b/docker/mssqldb/entrypoint.sh new file mode 100755 index 00000000..962beb49 --- /dev/null +++ b/docker/mssqldb/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +#start SQL Server, start the script to create the DB and import the data, start the app +nohup /init & +/opt/mssql/bin/sqlservr diff --git a/docker/mssqldb/init b/docker/mssqldb/init new file mode 100755 index 00000000..1f63253b --- /dev/null +++ b/docker/mssqldb/init @@ -0,0 +1,7 @@ +#!/bin/sh + +#wait for the SQL Server to come up +sleep 20s + +echo "Running CREATE DATABASE" +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -i bootstrap.sql \ No newline at end of file diff --git a/install-mssql-driver.sh b/install-mssql-driver.sh new file mode 100755 index 00000000..05888a87 --- /dev/null +++ b/install-mssql-driver.sh @@ -0,0 +1,14 @@ +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + +#Download appropriate package for the OS version +#Choose only ONE of the following, corresponding to your OS version + +#Debian 10 +curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list + +apt-get update +ACCEPT_EULA=Y apt-get install -y msodbcsql17 +# optional: for bcp and sqlcmd +ACCEPT_EULA=Y apt-get install -y mssql-tools +echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc +apt-get install -y unixodbc-dev \ No newline at end of file diff --git a/project/packages.pip b/project/packages.pip index 88f0be8c..c49e8afd 100644 --- a/project/packages.pip +++ b/project/packages.pip @@ -3,4 +3,6 @@ Pillow django-request-id psycopg2 requests -openpyxl \ No newline at end of file +openpyxl +django-pyodbc-azure +django-pyodbc \ No newline at end of file diff --git a/setup.py b/setup.py index e6244010..1835f5f3 100755 --- a/setup.py +++ b/setup.py @@ -10,10 +10,9 @@ os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) test_require_database_engine = { - 'mssql': 'mysqlclient >= 1.3.12', - 'mysql': 'django-pyodbc >= 1.1.3' -}.get('BINDER_TEST_DATABASE_ENGINE', 'psycopg2 >= 2.7') - + 'mysql': ['mysqlclient >= 1.3.12', 'psycopg2 >= 2.7'], + 'mssql': ['django-pyodbc-azure >= 2.1.0.0', 'django-pyodbc >= 1.1.3', 'django-hijack == 2.1.10', 'psycopg2 >= 2.7'], +}.get(os.environ.get('BINDER_TEST_DATABASE_ENGINE'), ['psycopg2 >= 2.7']) setup( @@ -51,7 +50,7 @@ ], tests_require=[ 'django-hijack >= 2.1.10', - test_require_database_engine, + *test_require_database_engine, "openpyxl >= 3.0.0" ], ) diff --git a/tests/__init__.py b/tests/__init__.py index 3e25664b..72e16360 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,19 +3,12 @@ from django.core.management import call_command import os +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mssql': + os.system("/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -Q 'CREATE DATABASE [binder-test];'") -if ( - os.path.exists('/.dockerenv') and - 'CY_RUNNING_INSIDE_CI' not in os.environ -): - db_settings = { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'postgres', - 'USER': 'postgres', - 'HOST': 'db', - 'PORT': 5432, - } -elif os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mysql': + + +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mysql': db_settings = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'binder-test', @@ -24,6 +17,26 @@ 'USER': 'root', 'PASSWORD': 'rootpassword', } +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mssql': + db_settings = { + 'ENGINE': 'sql_server.pyodbc', + 'NAME': 'binder-test', + 'TIME_ZONE': 'UTC', + 'HOST': 'mssqldb', + 'USER': 'sa', + 'PASSWORD': '~Test123', + 'OPTIONS': { + 'driver': 'ODBC Driver 17 for SQL Server', + }, + } +elif (os.path.exists('/.dockerenv') and 'CY_RUNNING_INSIDE_CI' not in os.environ): + db_settings = { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'postgres', + 'USER': 'postgres', + 'HOST': 'db', + 'PORT': 5432, + } else: db_settings = { 'ENGINE': 'django.db.backends.postgresql_psycopg2', diff --git a/tests/test_nulls_last.py b/tests/test_nulls_last.py index bd459014..e67b81bc 100644 --- a/tests/test_nulls_last.py +++ b/tests/test_nulls_last.py @@ -26,7 +26,7 @@ def test_order_by_nulls_last(self): self._load_test_data() # MySQL has different defaults when no nulls option is selected... - if os.environ.get('BINDER_TEST_MYSQL', '0') != '0': + if os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mssql', 'mysql']: self._assert_order('last_seen', ['4', '5', '1', '2', '3']) self._assert_order('-last_seen', ['3', '2', '1', '4', '5']) else: @@ -46,7 +46,7 @@ def test_order_by_nulls_last_on_annotation(self): self._load_test_data() # MySQL has different defaults when no nulls option is selected... - if os.environ.get('BINDER_TEST_MYSQL', '0') != '0': + if os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mssql', 'mysql']: self._assert_order('last_present', ['4', '5', '1', '2', '3']) self._assert_order('-last_present', ['3', '2', '1', '4', '5']) else: diff --git a/tests/test_postgres_fields.py b/tests/test_postgres_fields.py index 54e23555..b5a9a828 100644 --- a/tests/test_postgres_fields.py +++ b/tests/test_postgres_fields.py @@ -7,12 +7,12 @@ from binder.json import jsonloads from django.contrib.auth.models import User -if os.environ.get('BINDER_TEST_DATABASE_ENGINE') != 'mysql': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .testapp.models import FeedingSchedule, Animal, Zoo # TODO: Currently these only really test filtering. Move to test/filters? @unittest.skipIf( - os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mysql'], + os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mysql', 'mssql'], "Only available with PostgreSQL" ) class PostgresFieldsTest(TestCase): diff --git a/tests/testapp/models/__init__.py b/tests/testapp/models/__init__.py index d83931a2..ae3d5b5d 100644 --- a/tests/testapp/models/__init__.py +++ b/tests/testapp/models/__init__.py @@ -7,7 +7,7 @@ from .contact_person import ContactPerson from .costume import Costume # This is a Postgres-specific model -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .feeding_schedule import FeedingSchedule from .gate import Gate from .nickname import Nickname, NullableNickname diff --git a/tests/testapp/views/__init__.py b/tests/testapp/views/__init__.py index 7dd1f4f0..6f4a0531 100644 --- a/tests/testapp/views/__init__.py +++ b/tests/testapp/views/__init__.py @@ -10,7 +10,7 @@ from .country import CountryView # This has a Postgres-specific model -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .feeding_schedule import FeedingScheduleView from .gate import GateView from .lion import LionView From 2e910054a0d1d9325fe928eb290b505f17b0df71 Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 16:17:46 +0200 Subject: [PATCH 15/16] fix onetoonenullable constrainst failure in mssql: --- .github/workflows/ci.yml | 10 +++------- binder/models.py | 1 + setup.py | 2 ++ tests/__init__.py | 16 ++++++++++++++-- tests/filters/test_time_filters.py | 2 +- tests/test_multi_put.py | 2 ++ tests/testapp/models/animal.py | 1 + tests/testapp/models/zoo.py | 3 ++- 8 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cd369a9..48ba9a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,9 @@ jobs: strategy: matrix: - python-version: [ "3.8" ] - django-version: [ "3.1.4" ] - database-engine: ["mssql" ] -# -# python-version: [ "3.7", "3.8" ] -# django-version: [ "2.1.1", "3.1.4" ] -# database-engine: [ "postgres", "mysql", "mssql"] + python-version: [ "3.7", "3.8" ] + django-version: [ "2.1.1", "3.1.4" ] + database-engine: [ "postgres", "mysql", "mssql"] services: postgres: diff --git a/binder/models.py b/binder/models.py index 3764054b..7c526754 100644 --- a/binder/models.py +++ b/binder/models.py @@ -405,6 +405,7 @@ class Meta: def save(self, *args, **kwargs): self.full_clean() # Never allow saving invalid models! + return super().save(*args, **kwargs) diff --git a/setup.py b/setup.py index 1835f5f3..99184980 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,8 @@ test_require_database_engine = { 'mysql': ['mysqlclient >= 1.3.12', 'psycopg2 >= 2.7'], 'mssql': ['django-pyodbc-azure >= 2.1.0.0', 'django-pyodbc >= 1.1.3', 'django-hijack == 2.1.10', 'psycopg2 >= 2.7'], + # Alternative mssql setup with django 3 setup. For later + # 'mssql': ['mssql-django==1.0rc1', 'psycopg2 >= 2.7'] }.get(os.environ.get('BINDER_TEST_DATABASE_ENGINE'), ['psycopg2 >= 2.7']) diff --git a/tests/__init__.py b/tests/__init__.py index 72e16360..bea87d5d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -45,6 +45,7 @@ 'USER': 'postgres', } + settings.configure(**{ 'DEBUG': True, 'SECRET_KEY': 'testy mctestface', @@ -88,6 +89,12 @@ 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, + 'file': { + 'level': 'DEBUG', + 'class': 'logging.handlers.WatchedFileHandler', + 'filename': 'backend.log', + 'filters': [], + }, }, 'loggers': { # We override only this one to avoid logspam @@ -97,7 +104,13 @@ 'handlers': ['console'], 'level': 'ERROR', }, - } + # 'django.db.backends': { + # 'handlers': ['console', 'file'], + # 'level': 'DEBUG', + # 'propagate': True, + # }, + } + }, 'BINDER_PERMISSION': { 'default': [ @@ -124,7 +137,6 @@ 'admin': [] } }) - setup() # Do the dance to ensure the models are synched to the DB. diff --git a/tests/filters/test_time_filters.py b/tests/filters/test_time_filters.py index 76b923ef..b1e702e4 100644 --- a/tests/filters/test_time_filters.py +++ b/tests/filters/test_time_filters.py @@ -6,7 +6,7 @@ import os # This is not possible in -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mssql', 'mysql']: class TimeFiltersTest(TestCase): def setUp(self): diff --git a/tests/test_multi_put.py b/tests/test_multi_put.py index b664443c..dfa4f90f 100644 --- a/tests/test_multi_put.py +++ b/tests/test_multi_put.py @@ -176,6 +176,8 @@ def test_put_relations_from_referencing_side(self): } response = self.client.put('/animal/', data=json.dumps(with_model_data), content_type='application/json') + print(response.content) + self.assertEqual(response.status_code, 200) returned_data = jsonloads(response.content) diff --git a/tests/testapp/models/animal.py b/tests/testapp/models/animal.py index 85d844e0..d2d22f8e 100644 --- a/tests/testapp/models/animal.py +++ b/tests/testapp/models/animal.py @@ -11,6 +11,7 @@ class Concat(Func): # From the api docs: an animal with a name. class Animal(LoadedValuesMixin, BinderModel): + # id = models.IntegerField(primary_key=True, null=True, blank=True) name = models.TextField(max_length=64) zoo = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='animals', blank=True, null=True) zoo_of_birth = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='+', blank=True, null=True) # might've been born outside captivity diff --git a/tests/testapp/models/zoo.py b/tests/testapp/models/zoo.py index a6b1737c..8d931404 100644 --- a/tests/testapp/models/zoo.py +++ b/tests/testapp/models/zoo.py @@ -17,13 +17,14 @@ def delete_files(sender, instance=None, **kwargs): # From the api docs: a zoo with a name. It also has a founding date, # which is nullable (representing "unknown"). class Zoo(BinderModel): + # id = models.IntegerField(primary_key=True, null=True, blank=True) name = models.TextField() founding_date = models.DateField(null=True, blank=True) floor_plan = models.ImageField(upload_to='floor-plans', null=True, blank=True) # It is important that this m2m relationship is defined on this model, one of the tests depends on this fact contacts = models.ManyToManyField('ContactPerson', blank=True, related_name='zoos') # We assume that a person can only be the director of one zoo, one of the tests depends on this fact - director = models.OneToOneField('ContactPerson', on_delete=models.SET_NULL, blank=True, null=True, related_name='managing_zoo') + director = models.ForeignKey('ContactPerson', on_delete=models.SET_NULL, blank=True, null=True, related_name='managing_zoo') most_popular_animals = models.ManyToManyField('Animal', blank=True, related_name='+') opening_time = models.TimeField(default=datetime.time(9, 0, 0)) From ce0369fa474a6a527ee5aefba830f6edc02ef02f Mon Sep 17 00:00:00 2001 From: stefanmajoor Date: Mon, 26 Jul 2021 16:33:51 +0200 Subject: [PATCH 16/16] Fix problem with sql compiler for multiple withs --- binder/views.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/binder/views.py b/binder/views.py index c12c1760..56e3446c 100644 --- a/binder/views.py +++ b/binder/views.py @@ -315,11 +315,18 @@ class ModelView(View): # } virtual_relations = {} + @property + def database_type(self): + return { + 'mysql': 'mysql', + 'microsoft': 'mssql', + }.get(connections[self.model.objects.db].vendor, 'postgres') + @property def AggStrategy(self): - if connections[self.model.objects.db].vendor == 'mysql': + if self.database_type == 'mysql': return GroupConcat - if connections[self.model.objects.db].vendor == 'microsoft': + if self.database_type == 'mssql': return StringAgg return OrderableArrayAgg @@ -869,6 +876,14 @@ def _follow_related(self, fieldspec): # permission scoping. This will be done when fetching the actual # objects. def _get_with_ids(self, pks, request, include_annotations, with_map, where_map): + if self.database_type == 'mssql': + with_ids = {} + for key, sub_with_map in with_map.items(): + with_ids.update(self._base_get_with_ids(pks, request, include_annotations, {key: sub_with_map}, where_map)) + return with_ids + return self._base_get_with_ids( pks, request, include_annotations, with_map, where_map) + + def _base_get_with_ids(self, pks, request, include_annotations, with_map, where_map): result = {} annotations = {}