From 5e92f256893a73fbc3706a2d3f10f60e6c85e613 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Wed, 14 Aug 2024 15:33:59 +0200 Subject: [PATCH] Feature/user bitmap (#65) * upgrade python version and django 5.0.7 * add docker and related documentation * Update docker-compose.yml * Update admin.py * Create 0037_datasetbitmapposition_userbitmap.py * Update __init__.py * Create datasetBitmapPosition.py * Create userBitmap.py * Update docker-compose.yml * Update admin.py * Create checksystemhealth.py * Create checkuserbitmap.py * commit migrations * Update datasetBitmapPosition.py * Update checkuserbitmap.py * Update userBitmap.py * Update settings.py * Update admin.py * Update userBitmap.py * Update checksystemhealth.py --- .docker/.gitkeep | 0 .gitignore | 5 + Pipfile | 4 +- Pipfile.lock | 86 +++++++------- README.md | 55 +++++++-- docker-compose.yml | 29 +++++ impresso/admin.py | 45 ++++++++ .../management/commands/checksystemhealth.py | 79 +++++++++++++ .../management/commands/checkuserbitmap.py | 70 +++++++++++ .../0037_datasetbitmapposition_userbitmap.py | 36 ++++++ ...datasetbitmapposition_metadata_and_more.py | 23 ++++ ...p_subscriptions_alter_userbitmap_bitmap.py | 23 ++++ ...datasetbitmapposition_metadata_and_more.py | 23 ++++ impresso/models/__init__.py | 3 + impresso/models/datasetBitmapPosition.py | 23 ++++ impresso/models/userBitmap.py | 109 ++++++++++++++++++ impresso/settings.py | 2 +- 17 files changed, 561 insertions(+), 54 deletions(-) create mode 100644 .docker/.gitkeep create mode 100644 docker-compose.yml create mode 100644 impresso/management/commands/checksystemhealth.py create mode 100644 impresso/management/commands/checkuserbitmap.py create mode 100644 impresso/migrations/0037_datasetbitmapposition_userbitmap.py create mode 100644 impresso/migrations/0038_datasetbitmapposition_metadata_and_more.py create mode 100644 impresso/migrations/0039_userbitmap_subscriptions_alter_userbitmap_bitmap.py create mode 100644 impresso/migrations/0040_alter_datasetbitmapposition_metadata_and_more.py create mode 100644 impresso/models/datasetBitmapPosition.py create mode 100644 impresso/models/userBitmap.py diff --git a/.docker/.gitkeep b/.docker/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 02e0111..8eb1e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,8 @@ ___* *.rdb *.json .DS_Store + + +# local docker settings and config +.docker/* +!.docker/.gitkeep \ No newline at end of file diff --git a/Pipfile b/Pipfile index 44a4bff..174da75 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ pip = "*" celery = "*" requests = "*" redis = "*" -django = "==5.0.4" +django = "==5.0.7" pymysql = "*" django-registration = "*" gunicorn = "*" @@ -18,4 +18,4 @@ gunicorn = "*" "flake8" = "*" [requires] -python_version = "3.12.2" +python_version = "3.12.4" diff --git a/Pipfile.lock b/Pipfile.lock index 20838af..b4b0879 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "f988ed11b876746ecb45240553acb1c5392882a889e87d4babf77f7782a6d830" + "sha256": "91e03dce1496254c3f117a47f49953587054b57a8305e52bc6d652fd3c78d146" }, "pipfile-spec": 6, "requires": { - "python_version": "3.12.2" + "python_version": "3.12.4" }, "sources": [ { @@ -51,11 +51,11 @@ }, "certifi": { "hashes": [ - "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", - "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" + "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", + "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" ], "markers": "python_version >= '3.6'", - "version": "==2024.2.2" + "version": "==2024.7.4" }, "charset-normalizer": { "hashes": [ @@ -193,12 +193,12 @@ }, "django": { "hashes": [ - "sha256:4bd01a8c830bb77a8a3b0e7d8b25b887e536ad17a81ba2dce5476135c73312bd", - "sha256:916423499d75d62da7aa038d19aef23d23498d8df229775eb0a6309ee1013775" + "sha256:bd4505cae0b9bd642313e8fb71810893df5dc2ffcacaa67a33af2d5cd61888f2", + "sha256:f216510ace3de5de01329463a315a629f33480e893a9024fc93d8c32c22913da" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==5.0.4" + "version": "==5.0.7" }, "django-registration": { "hashes": [ @@ -236,44 +236,44 @@ }, "packaging": { "hashes": [ - "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" + "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", + "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" ], - "markers": "python_version >= '3.7'", - "version": "==24.0" + "markers": "python_version >= '3.8'", + "version": "==24.1" }, "pip": { "hashes": [ - "sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" + "sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2", + "sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==24.0" + "markers": "python_version >= '3.8'", + "version": "==24.2" }, "prompt-toolkit": { "hashes": [ - "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d", - "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6" + "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10", + "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.0.43" + "version": "==3.0.47" }, "pymysql": { "hashes": [ - "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", - "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7" + "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c", + "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.1.0" + "version": "==1.1.1" }, "python-dateutil": { "hashes": [ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pytz": { @@ -286,37 +286,37 @@ }, "redis": { "hashes": [ - "sha256:7adc2835c7a9b5033b7ad8f8918d09b7344188228809c98df07af226d39dec91", - "sha256:ec31f2ed9675cc54c21ba854cfe0462e6faf1d83c8ce5944709db8a4700b9c61" + "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870", + "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==5.0.4" + "version": "==5.0.8" }, "requests": { "hashes": [ - "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", - "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.31.0" + "markers": "python_version >= '3.8'", + "version": "==2.32.3" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "sqlparse": { "hashes": [ - "sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93", - "sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663" + "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4", + "sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e" ], "markers": "python_version >= '3.8'", - "version": "==0.5.0" + "version": "==0.5.1" }, "tzdata": { "hashes": [ @@ -328,11 +328,11 @@ }, "urllib3": { "hashes": [ - "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d", - "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19" + "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" ], "markers": "python_version >= '3.8'", - "version": "==2.2.1" + "version": "==2.2.2" }, "vine": { "hashes": [ @@ -353,12 +353,12 @@ "develop": { "flake8": { "hashes": [ - "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", - "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3" + "sha256:049d058491e228e03e67b390f311bbf88fce2dbaa8fa673e7aea87b7198b8d38", + "sha256:597477df7860daa5aa0fdd84bf5208a043ab96b8e96ab708770ae0364dd03213" ], "index": "pypi", "markers": "python_full_version >= '3.8.1'", - "version": "==7.0.0" + "version": "==7.1.1" }, "mccabe": { "hashes": [ @@ -370,11 +370,11 @@ }, "pycodestyle": { "hashes": [ - "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", - "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" + "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", + "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521" ], "markers": "python_version >= '3.8'", - "version": "==2.11.1" + "version": "==2.12.1" }, "pyflakes": { "hashes": [ diff --git a/README.md b/README.md index 0f6db41..f1db55e 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,58 @@ # impresso-user-admin A basic django application to manage user-related information contained in [Impresso's Master DB](https://github.com/impresso/impresso-master-db). -We use `pipenv`for development and `docker` for production. Please look at the relevant sections in the documentation. +We use `pipenv` for development together with `docker`. Please look at the relevant sections in the documentation. -To start _django admin_ in development with pipenv: +## Development - ENV=dev pipenv run ./manage.py runserver +Take the time to explore the `.example.env` file and the related `./impresso/settings.py` to understand the settings that can be configured via environment variables for your specific environment. We have configured `dotenv` in `./impresso/base.py` to allow the loading of different `.env` files. For example, you can use `.env` or `.dev.env` for development, and `.prod.env` to test production settings. -or to test tags: +```sh +# our .dev.env file, that connects to the local redis instance +REDIS_HOST=localhost:6379 +IMPRESSO_DB_HOST=localhost +IMPRESSO_DB_PORT=3306 +# Then don't forget to fill all SOLR related settings accordiung to your impresso configuration +IMPRESSO_SOLR_URL=http://localhost:8983/solr/impresso +IMPRESSO_SOLR_USER=your-user-reader-only +IMPRESSO_SOLR_PASSWORD=our-user-reader-only-password +IMPRESSO_SOLR_USER_WRITE=your-user-write-allowed +IMPRESSO_SOLR_PASSWORD_WRITE=your-user-write-allowed-password +IMPRESSO_SOLR_PASSAGES_URL=http://localhost:8983/solr/impresso-tr-passages +``` + +To start the Django admin, you need to have Redis and MySQL running. You can start them by running the command `docker compose up`. Please note that in our YAML file, the ports for Redis and MySQL are exposed to facilitate local development and testing. + +```sh +docker compose up -d --env-file=.dev.env +``` + +Then you can start the development server, e.g. with pipenv and the `dev.env` file: + +```sh +ENV=dev pipenv run ./manage.py runserver +``` + +or with Makefile: + +```sh +ENV=dev make run-dev +``` - ENV=dev make run-dev +To start _celery_ task manager in development with pipenv, in a new terminal: -To start _celery_ task manager in development with pipenv: +```sh +ENV=dev pipenv run celery -A impresso worker -l info +``` + +Of course, you can also use a generic `.env file` on development, in this case you don't need to specify the `ENV` variable: - ENV=dev pipenv run celery -A impresso worker -l info +```sh +docker compose up -d +pipenv run ./manage.py runserver +# and in another terminal, to start the celery worker +pipenv run celery -A impresso worker -l info +``` ### setup with pyenv + pipenv @@ -30,7 +69,7 @@ The last command gives you the version of the local python. If it doesn't meet t use pyenv install command: ``` -pyenv install 3.6.9 +pyenv install 3.12.4 ``` Use pip to install Pipenv: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c6e5147 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +services: + redis: + image: redis:alpine + restart: always + volumes: + - ./.docker/redis:/data + entrypoint: redis-server --appendonly yes + ports: + - 6379:6379 + # mysql: + # image: mariadb:lts + # restart: always + # volumes: + # - ./.docker/mysql:/var/lib/mysql + # ports: + # - ${DB_PORT:-3308}:3306 + # environment: + # MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD:-root} + # MARIADB_DATABASE: ${IMPRESSO_DB_NAME} + # MARIADB_USER: ${IMPRESSO_DB_USER} + # MARIADB_PASSWORD: ${IMPRESSO_DB_PASSWORD} + mysql-tunnel: + image: kroniak/ssh-client + restart: always + volumes: + - ./.docker/config/ssh:/root/.ssh + ports: + - ${IMPRESSO_DB_PORT:-3306}:3306 + command: ssh -N impresso-mysql-tunnel diff --git a/impresso/admin.py b/impresso/admin.py index dc5010a..5647a0d 100644 --- a/impresso/admin.py +++ b/impresso/admin.py @@ -1,3 +1,4 @@ +from django import forms from django.contrib import admin from django.contrib import messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin @@ -8,10 +9,54 @@ from .models import SearchQuery, ContentItem from .models import Collection, CollectableItem, Tag, TaggableItem from .models import Attachment, UploadedImage +from .models import UserBitmap, DatasetBitmapPosition from impresso.tasks import after_user_activation +@admin.register(UserBitmap) +class UserBitmapAdmin(admin.ModelAdmin): + list_display = ("user", "bitmap_display", "user_plan_display", "num_subscriptions") + search_fields = ["user__username", "user__email"] + + def num_subscriptions(self, obj): + return obj.subscriptions.count() + + def bitmap_display(self, obj): + if obj.bitmap is None: + return "" + return bin(int.from_bytes(obj.bitmap, byteorder="big")) + + def user_plan_display(self, obj): + if obj.bitmap is None: + return "-" + bitmap_int = int.from_bytes(obj.bitmap, byteorder="big") + bitmap_length = bitmap_int.bit_length() + # Extract the first 5 bits + bitmap_plan = ( + bitmap_int >> (bitmap_length - UserBitmap.BITMAP_PLAN_MAX_LENGTH) + ) & 0b11111 + if bitmap_plan == UserBitmap.USER_PLAN_GUEST: + return "Guest" + if bitmap_plan == UserBitmap.USER_PLAN_AUTH_USER: + return "Impresso Registered User" + if bitmap_plan == UserBitmap.USER_PLAN_EDUCATIONAL: + return "Student or Teacher - Educational User" + if bitmap_plan == UserBitmap.USER_PLAN_RESEARCHER: + return "Researcher - Academic User" + + return bin(bitmap_plan) + + user_plan_display.short_description = "User Plan" + + +@admin.register(DatasetBitmapPosition) +class DatasetBitmapPositionAdmin(admin.ModelAdmin): + list_display = ("name", "bitmap_position") + search_fields = ["name"] + readonly_fields = ("bitmap_position",) + + @admin.register(Issue) class IssueAdmin(admin.ModelAdmin): list_display = ( diff --git a/impresso/management/commands/checksystemhealth.py b/impresso/management/commands/checksystemhealth.py new file mode 100644 index 0000000..2297d76 --- /dev/null +++ b/impresso/management/commands/checksystemhealth.py @@ -0,0 +1,79 @@ +import re +import requests +from django.core.management.base import BaseCommand +from django.conf import settings +from django.db import connection + +FLS = [ + "id", + "content_length_i", + "snippet_plain", + "bm_explore_s", + "bm_get_tr_s", + "bm_get_img_s", + "meta_journal_s", + "meta_partnerid_s", +] + + +class Command(BaseCommand): + help = "Check SOLR connectivity" + + def handle(self, *args, **options): + self.stdout.write("Checking Database connectivity...") + with connection.cursor() as cursor: + cursor.execute("SELECT DATABASE()") + database_name = cursor.fetchone()[0] + + self.stdout.write( + f"Current Database: \n \033[94m{database_name}\033[0m\n\n" + ) + cursor.execute("SHOW TABLES") + tables = [t[0] for t in cursor.fetchall()] + self.stdout.write(f"Database Tables: \n - {'\n - '.join(tables)}") + + # test solr connectivity usin g settings + self.stdout.write("\nChecking SOLR connectivity...") + solr_url = settings.IMPRESSO_SOLR_URL_SELECT + # check that solr_url is following the regex pattern, without select at the end + # https://:/solr//select + if not re.match(r"^https?://.*\/solr/[^\/]+\/select$", solr_url): + self.stderr.write(f"Invalid SOLR URL: {solr_url}") + return + + params = { + "q": "*:*", + "rows": 2, + "fl": ",".join(FLS), + } + solr_response = requests.get( + solr_url, + auth=settings.IMPRESSO_SOLR_AUTH, + params=params, + ) + solr_status = solr_response.status_code + self.stdout.write(f"SOLR URL: \n - {solr_url}") + self.stdout.write(f"SOLR Status: \n - {solr_status}") + # n of rows in solr + solr_num_rows = solr_response.json()["response"]["numFound"] + self.stdout.write(f"SOLR Num Rows: \n - {solr_num_rows}") + # example result + docs = solr_response.json()["response"]["docs"] + self.stdout.write(f"\n SOLR Example Docs:") + + for doc in docs: + self.stdout.write(f" - \nid:\033[94m{doc.get('id')}\033[0m") + + for field in FLS: + self.stdout.write(f" {field}: {doc.get(field)}") + # ping redis + self.stdout.write("\nChecking Redis connectivity...") + import redis + + redis_host = settings.REDIS_HOST.split(":")[0] + redis_port = settings.REDIS_HOST.split(":")[-1] + redis_conn = redis.Redis(host=redis_host, port=redis_port, db="4") + redis_status = redis_conn.ping() + self.stdout.write(f"Redis Host: \n - {redis_host}") + self.stdout.write(f"Redis Port: \n - {redis_port}") + self.stdout.write(f"Redis Status: \n - {redis_status}") diff --git a/impresso/management/commands/checkuserbitmap.py b/impresso/management/commands/checkuserbitmap.py new file mode 100644 index 0000000..af16d87 --- /dev/null +++ b/impresso/management/commands/checkuserbitmap.py @@ -0,0 +1,70 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth.models import User +from impresso.models import UserBitmap +from impresso.models import DatasetBitmapPosition + + +class Command(BaseCommand): + help = "Test a user bitmap against a content bitmap" + + def add_arguments(self, parser): + parser.add_argument("username", type=str) + parser.add_argument("bitmap", type=str, nargs="?", default=None) + + def handle(self, username, *args, **options): + self.stdout.write(f"Get user with username: {username}") + user = User.objects.get(username=username) + self.stdout.write(f"User: pk={user.id} \033[34m{user.username}\033[0m") + # rpint out user groups + groups = [group.name for group in user.groups.all()] + self.stdout.write(f"User groups: \n \033[34m{'\n '.join(groups)}\033[0m") + # print out its related user bitmap. If no one, just create it. + user_bitmap = user.bitmap.get_up_to_date_bitmap() + user_bitmap_length = user_bitmap.bit_length() + # print user_bitmap binary as sequence of 0 and 1 + self.stdout.write(f"user get_up_to_date_bitmap(): \033[34m{bin(user_bitmap)}\033[0m") + # get the total number of bits + self.stdout.write(f"user bitmap length: \033[34m{user_bitmap_length}\033[0m") + + self.stdout.write( + f"User bitmap plan max length: \033[34m{UserBitmap.BITMAP_PLAN_MAX_LENGTH}\033[0m" + ) + # get user subscriptions + subscriptions = list( + user.bitmap.subscriptions.values("name", "bitmap_position") + ) + # verify that the user subscription positions are correct + self.stdout.write( + f"User subscriptions: \n \033[34m{'\n '.join([s.get('name') for s in subscriptions])}\033[0m" + ) + max_subscription_position = max( + [s["bitmap_position"] for s in subscriptions] + ) + self.stdout.write( + f"Max subscription position: \033[34m{max_subscription_position}\033[0m" + ) + self.stdout.write( + f"adjusted max subscription position with groups position: \033[34m{max_subscription_position + UserBitmap.BITMAP_PLAN_MAX_LENGTH}\033[0m" + ) + + self.stdout.write("Verify other subscriptions til max user bitmap position (the rest should be 0)") + # get all possible subscriptions + all_subscriptions = DatasetBitmapPosition.objects.filter(bitmap_position__lte=max_subscription_position).order_by("bitmap_position") + + # print out all possible subscriptions + for subscription in all_subscriptions: + position = subscription.bitmap_position + 5 + # Calculate the bit position from the end + bit_position = max_subscription_position + 5 - position + # Check if the bit at the specified position is 1 + is_set = (user_bitmap & (1 << bit_position)) != 0 + if is_set: + self.stdout.write( + f"\033[34m {subscription.name} at position: {position} is set: {is_set}\033[0m" + ) + else: + self.stdout.write( + f" {subscription.name} at position: {position} is set: {is_set}" + ) + self.stdout.write("\n---\nDone! \033[31m❤️\033[0m \n---\n") + diff --git a/impresso/migrations/0037_datasetbitmapposition_userbitmap.py b/impresso/migrations/0037_datasetbitmapposition_userbitmap.py new file mode 100644 index 0000000..76dce0f --- /dev/null +++ b/impresso/migrations/0037_datasetbitmapposition_userbitmap.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.7 on 2024-08-06 09:36 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('impresso', '0036_searchquery_hash_alter_job_type'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='DatasetBitmapPosition', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('bitmap_position', models.PositiveIntegerField(unique=True)), + ], + ), + migrations.CreateModel( + name='UserBitmap', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('bitmap', models.BinaryField()), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='bitmap', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'User Bitmap', + 'verbose_name_plural': 'User Bitmaps', + }, + ), + ] diff --git a/impresso/migrations/0038_datasetbitmapposition_metadata_and_more.py b/impresso/migrations/0038_datasetbitmapposition_metadata_and_more.py new file mode 100644 index 0000000..18a2021 --- /dev/null +++ b/impresso/migrations/0038_datasetbitmapposition_metadata_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.7 on 2024-08-07 11:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('impresso', '0037_datasetbitmapposition_userbitmap'), + ] + + operations = [ + migrations.AddField( + model_name='datasetbitmapposition', + name='metadata', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='datasetbitmapposition', + name='bitmap_position', + field=models.PositiveIntegerField(blank=True, null=True, unique=True), + ), + ] diff --git a/impresso/migrations/0039_userbitmap_subscriptions_alter_userbitmap_bitmap.py b/impresso/migrations/0039_userbitmap_subscriptions_alter_userbitmap_bitmap.py new file mode 100644 index 0000000..f3a0e54 --- /dev/null +++ b/impresso/migrations/0039_userbitmap_subscriptions_alter_userbitmap_bitmap.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.7 on 2024-08-08 08:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('impresso', '0038_datasetbitmapposition_metadata_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='userbitmap', + name='subscriptions', + field=models.ManyToManyField(to='impresso.datasetbitmapposition'), + ), + migrations.AlterField( + model_name='userbitmap', + name='bitmap', + field=models.BinaryField(blank=True, editable=True, null=True), + ), + ] diff --git a/impresso/migrations/0040_alter_datasetbitmapposition_metadata_and_more.py b/impresso/migrations/0040_alter_datasetbitmapposition_metadata_and_more.py new file mode 100644 index 0000000..daafb13 --- /dev/null +++ b/impresso/migrations/0040_alter_datasetbitmapposition_metadata_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.7 on 2024-08-08 08:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('impresso', '0039_userbitmap_subscriptions_alter_userbitmap_bitmap'), + ] + + operations = [ + migrations.AlterField( + model_name='datasetbitmapposition', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='userbitmap', + name='bitmap', + field=models.BinaryField(blank=True, null=True), + ), + ] diff --git a/impresso/models/__init__.py b/impresso/models/__init__.py index ad93efc..408ac14 100644 --- a/impresso/models/__init__.py +++ b/impresso/models/__init__.py @@ -14,3 +14,6 @@ from .job import Job from .attachment import Attachment from .uploadedImage import UploadedImage + +from .datasetBitmapPosition import DatasetBitmapPosition +from .userBitmap import UserBitmap diff --git a/impresso/models/datasetBitmapPosition.py b/impresso/models/datasetBitmapPosition.py new file mode 100644 index 0000000..7fbebba --- /dev/null +++ b/impresso/models/datasetBitmapPosition.py @@ -0,0 +1,23 @@ +from django.db import models +from django.db.models import Max + + +class DatasetBitmapPosition(models.Model): + name = models.CharField(max_length=255) + bitmap_position = models.PositiveIntegerField( + unique=True, + null=True, + blank=True, + ) + metadata = models.JSONField(default=dict, blank=True) + + def __str__(self): + return self.name + + def save(self, *args, **kwargs): + if self.bitmap_position is None: + max_position = DatasetBitmapPosition.objects.aggregate( + Max("bitmap_position") + )["bitmap_position__max"] + self.bitmap_position = 0 if max_position is None else max_position + 1 + super().save(*args, **kwargs) diff --git a/impresso/models/userBitmap.py b/impresso/models/userBitmap.py new file mode 100644 index 0000000..e207a42 --- /dev/null +++ b/impresso/models/userBitmap.py @@ -0,0 +1,109 @@ +import logging +from django.db import models +from django.contrib.auth.models import User +from .datasetBitmapPosition import DatasetBitmapPosition +from django.db.models.signals import m2m_changed + +logger = logging.getLogger(__name__) + + +class UserBitmap(models.Model): + user = models.OneToOneField(User, on_delete=models.CASCADE, related_name="bitmap") + bitmap = models.BinaryField(editable=False, null=True, blank=True) + subscriptions = models.ManyToManyField(DatasetBitmapPosition) + # Guest - Unregisted User public NA (True by default) used only for test purposes + # Impresso Registered User impresso Account created, no academic afiliation + # Student or Teacher - Educational User educational Account created, educational academic afiliation + # Researcher - Academic User researcher Account created, research academic afiliation + USER_PLAN_GUEST = 0b10000 + USER_PLAN_AUTH_USER = 0b11000 + USER_PLAN_EDUCATIONAL = 0b11100 + USER_PLAN_RESEARCHER = 0b11110 + + BITMAP_PLAN_MAX_LENGTH = 5 + + def get_up_to_date_bitmap(self): + """ + Get the bitmap using the groups the user is affiliated to and the affiliations to the DatasetBitmapPosition + The four first bits (starting on the left, indices 0-3) are the ones relating to the user plans + Then there is an empy bit (index 4) and the rest of the bits are for the user's subscriptions to the datasets. + The user bitmap relating to user plans is cumulative, hence, any user that is a researcher (bit #3 = 1) has all preceeding + bits also set to 1 : 1111 [archive bits...]. + All users have at least the "guest" bit set to 1 (bit #1): 10000 [archive bits, all 0] + """ + # get all groups the user is affiliated to as flat array, ordered by a-z + groups = [group.name for group in self.user.groups.all()] + if "plan-researcher" in groups: + bitmap = UserBitmap.USER_PLAN_RESEARCHER + elif "plan-educational" in groups: + bitmap = UserBitmap.USER_PLAN_EDUCATIONAL + else: + bitmap = UserBitmap.USER_PLAN_AUTH_USER + # print current bitmap + # print(f"current bitmap: {bitmap:05b}") + # get all user subscriptions + subscriptions = list(self.subscriptions.values("name", "bitmap_position")) + if not subscriptions: + return bitmap + # max bitmap position + max_position = ( + max([x["bitmap_position"] for x in subscriptions]) + + UserBitmap.BITMAP_PLAN_MAX_LENGTH + + 1 + ) + # Shift the initial signature to the left by the max bit position + bitmap = bitmap << max_position - UserBitmap.BITMAP_PLAN_MAX_LENGTH + # print(f"current empty bitmap: {bitmap:05b}") + for subscription in subscriptions: + # Use the bitmap position to set the corresponding bit + position = ( + subscription["bitmap_position"] + UserBitmap.BITMAP_PLAN_MAX_LENGTH + ) + bitmap |= 1 << (max_position - position - 1) + + return bitmap + + def __str__(self): + return f"{self.user.username} Bitmap" + + class Meta: + verbose_name = "User Bitmap" + verbose_name_plural = "User Bitmaps" + + +def update_user_bitmap(sender, instance, action, **kwargs): + if action == "post_add" or action == "post_remove": + logger.info(f"User {instance.user} subscription changed, updating") + user_bitmap = instance.get_up_to_date_bitmap() + bitmap_bytes = user_bitmap.to_bytes( + (user_bitmap.bit_length() + 7) // 8, byteorder="big" + ) + instance.bitmap = bitmap_bytes + instance.save() + logger.info( + f"User {instance.user} subscription changed, bitmap updated to {user_bitmap:05b}" + ) + + +def update_user_bitmap_on_user_groups_changed(sender, instance, action, **kwargs): + if action == "post_add" or action == "post_remove": + user_bitmap, created = UserBitmap.objects.get_or_create(user=instance) + logger.info( + f"User {instance} groups changed. {'Creating new bitmap.' if created else 'Updating bitmap.'}" + ) + bitmap = user_bitmap.get_up_to_date_bitmap() + bitmap_bytes = bitmap.to_bytes((bitmap.bit_length() + 7) // 8, byteorder="big") + user_bitmap.bitmap = bitmap_bytes + user_bitmap.save() + logger.info(f"User {instance} groups changed, bitmap updated to {bitmap:05b}") + + +m2m_changed.connect( + update_user_bitmap, + sender=UserBitmap.subscriptions.through, +) + +m2m_changed.connect( + update_user_bitmap_on_user_groups_changed, + sender=User.groups.through, +) diff --git a/impresso/settings.py b/impresso/settings.py index a74de9f..267b907 100644 --- a/impresso/settings.py +++ b/impresso/settings.py @@ -18,7 +18,7 @@ ALLOWED_HOSTS = [get_env_variable("ALLOWED_HOSTS")] -CSRF_TRUSTED_ORIGINS=get_env_variable("CSRF_TRUSTED_ORIGINS", "").split(',') +CSRF_TRUSTED_ORIGINS = get_env_variable("CSRF_TRUSTED_ORIGINS", "").split(",") # Application definition