OIDC/Auth2 integration

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,20 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+
+**How To Reproduce**
+
+**Payload from client (handle_request)**
+
+**Response from backend**
+
+**Expected behavior**
+
+**Additional context**

Makefile

@@ -144,5 +144,12 @@ run-dev-otel run-bash-otel: | start-dev-otel run-dev-attach-otel
 build-dev:
 	docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev
 
+build-dev-fullstack:
+	DOCKER_BUILDKIT=1 docker build --file=dev/Dockerfile.dev . \
+        --build-arg=REQUIREMENTS_FILE=requirements_development_fullstack.txt \
+		--build-context pipauth=../openslides-auth-service/libraries/pip-auth \
+		--build-context datastore=../openslides-datastore-service \
+		--tag=openslides-backend-dev-fullstack
+
 rebuild-dev:
-	docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev --no-cache
+	docker build --file=dev/Dockerfile.dev . --target development --tag=openslides-backend-dev --no-cache

dev/Dockerfile.dev

@@ -1,13 +1,9 @@
-FROM python:3.10.13-slim-bookworm
+FROM python:3.10.13-slim-bookworm as base
 
 RUN apt-get update && apt-get install --yes make git curl ncat vim bash-completion mime-support gcc libpq-dev libmagic1
 
 WORKDIR /app
 
-COPY requirements/ requirements/
-ARG REQUIREMENTS_FILE=requirements_development.txt
-RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE
-
 COPY dev/.bashrc .
 COPY dev/cleanup.sh .
 
@@ -42,3 +38,17 @@ ENV DEFAULT_FROM_EMAIL [email protected]
 STOPSIGNAL SIGKILL
 ENTRYPOINT ["./entrypoint.sh"]
 CMD exec python -m debugpy --listen 0.0.0.0:5678 openslides_backend
+
+FROM base AS development
+
+COPY requirements/ requirements/
+ARG REQUIREMENTS_FILE=requirements_development.txt
+RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE
+
+FROM base AS development-fullstack
+
+COPY --from=pipauth / /pip-auth
+COPY --from=datastore / /openslides-datastore-service
+COPY requirements/ requirements/
+ARG REQUIREMENTS_FILE=requirements_development.txt
+RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE

dev/dc.local.yml

@@ -7,7 +7,7 @@ services:
                 - REQUIREMENTS_FILE=requirements_development_local.txt
         volumes:
             - ../../openslides-datastore-service/:/datastore-service
-            - ../../openslides-auth-service/auth/libraries/pip-auth/:/authlib
+            - ../../openslides-auth-service/libraries/pip-auth/:/authlib
         environment:
             - PYTHONPATH=/app:/datastore-service:/authlib
             - MYPYPATH=/app:/datastore-service:/authlib
@@ -34,6 +34,3 @@ services:
     vote:
         build:
             context: ../../openslides-vote-service
-    auth:
-        build:
-            context: ../../openslides-auth-service

dev/docker-compose.dev.yml

@@ -27,6 +27,12 @@ services:
             - CACHE_HOST=redis
             - DATABASE_HOST=postgres
             - DATASTORE_LOG_LEVEL=CRITICAL
+            - OPENSLIDES_KEYCLOAK_URL=http://keycloak:8080/idp
+            - OPENSLIDES_AUTH_REALM=os
+            - OPENSLIDES_AUTH_CLIENT_ID=os-ui
+            - OPENSLIDES_TOKEN_ISSUER=http://keycloak:8080/idp/auth/realms/os
+            - OPENSLIDES_KEYCLOAK_ADMIN_USERNAME=admin
+            - OPENSLIDES_KEYCLOAK_ADMIN_PASSWORD=admin
         depends_on:
             - datastore-writer
     datastore-reader:
@@ -58,26 +64,21 @@ services:
         depends_on:
             - postgres
             - redis
-    auth:
-        build:
-            context: "https://github.com/OpenSlides/openslides-auth-service.git#main"
-            dockerfile: "Dockerfile.dev"
-        image: openslides-auth-dev
-        ports:
-            - "9004:9004"
+
+    keycloak:
+#        build:
+#            context: "https://github.com/OpenSlides/openslides-auth-service.git#main/keycloak"
+        image: openslides-keycloak-dev
         environment:
-            - ACTION_HOST=backend
-            - ACTION_PORT=9002
-            - MESSAGE_BUS_HOST=redis
-            - CACHE_HOST=redis
-            - DATASTORE_READER_HOST=datastore-reader
-            - DATASTORE_READER_PORT=9010
-            - DATASTORE_WRITER_HOST=datastore-writer
-            - DATASTORE_WRITER_PORT=9011
-        depends_on:
-            - datastore-reader
-            - datastore-writer
-            - redis
+            - KC_BOOTSTRAP_ADMIN_USERNAME=admin
+            - KC_BOOTSTRAP_ADMIN_PASSWORD=admin
+            - JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"
+            - KEYCLOAK_HOSTNAME=http://keycloak:8080/idp/
+            - KEYCLOAK_HTTP_RELATIVE_PATH=/idp/
+        ports:
+            - 18080:8080
+            - 15005:5005
+
     vote:
         build:
             context: "https://github.com/OpenSlides/openslides-vote-service.git#main"
@@ -96,7 +97,7 @@ services:
         depends_on:
             - datastore-reader
             - redis
-            - auth
+            - keycloak
     postgres:
         image: postgres:15
         environment:

dev/run-tests.sh

@@ -7,7 +7,7 @@ DC="docker compose -f dev/docker-compose.dev.yml"
 $DC up --build --detach
 $DC exec -T backend scripts/wait.sh datastore-writer 9011
 $DC exec -T backend scripts/wait.sh datastore-reader 9010
-$DC exec -T backend scripts/wait.sh auth 9004
+$DC exec -T backend scripts/wait.sh keycloak 8080
 $DC exec -T backend ./entrypoint.sh pytest --cov
 error=$?
 $DC down --volumes

docs/Actions-Overview.md

@@ -113,10 +113,6 @@ A more general format description see in [Action-Service](https://github.com/Ope
 - [motion_comment_section.delete](actions/motion_comment_section.delete.md)
 - [motion_comment_section.sort](actions/motion_comment_section.sort.md)
 - [motion_comment_section.update](actions/motion_comment_section.update.md)
-- [motion_statute_paragraph.create](actions/motion_statute_paragraph.create.md)
-- [motion_statute_paragraph.delete](actions/motion_statute_paragraph.delete.md)
-- [motion_statute_paragraph.sort](actions/motion_statute_paragraph.sort.md)
-- [motion_statute_paragraph.update](actions/motion_statute_paragraph.update.md)
 - [motion_submitter.create](actions/motion_submitter.create.md)
 - [motion_submitter.delete](actions/motion_submitter.delete.md)
 - [motion_submitter.sort](actions/motion_submitter.sort.md)
@@ -207,6 +203,12 @@ A more general format description see in [Action-Service](https://github.com/Ope
 - [theme.delete](actions/theme.delete.md)
 - [theme.update](actions/theme.update.md)
 
+## Gender
+
+- [gender.create](actions/theme.create.md)
+- [gender.delete](actions/theme.delete.md)
+- [gender.update](actions/theme.update.md)
+
 ## Topics
 
 - [topic.create](actions/topic.create.md)
@@ -231,7 +233,6 @@ A more general format description see in [Action-Service](https://github.com/Ope
 - [user.toggle_presence_by_number](actions/user.toggle_presence_by_number.md)
 - [user.update](actions/user.update.md)
 - [user.update_self](actions/user.update_self.md)
-- [user.save_saml_account](actions/user.save_saml_account.md)
 - [meeting_user.create](actions/meeting_user.create.md)
 - [meeting_user.update](actions/meeting_user.update.md)
 - [meeting_user.delete](actions/meeting_user.delete.md)

docs/Presenters-Overview.md

@@ -6,6 +6,7 @@ Available presenters:
 - [get_forwarding_meetings](presenters/get_forwarding_meetings.md)
 - [get_meetings](presenters/get_meetings.md)
 - [get_users](presenters/get_users.md)
+- [get_user_editable](presenters/get_user_editable.md)
 - [get_user_related_models](presenters/get_user_related_models.md)
 - [get_user_scope](presenters/get_user_scope.md)
 - [search_deleted_models](presenters/search_deleted_models.md)

docs/actions/account.import.md

@@ -1,5 +1,5 @@
 ## Payload
-```js
+```
 {
 // required
   id: Id; // action worker id

docs/actions/account.json_upload.md

@@ -2,7 +2,7 @@
 
 Because the data fields are all converted from CSV import file, **they are all of type `string`**. 
 The types noted below are the internal types after conversion in the backend. See [here](preface_special_imports.md#internal-types) for the representation of the types.
-```js
+```
 {
     // required
     data: {
@@ -14,12 +14,12 @@ The types noted below are the internal types after conversion in the backend. Se
         member_number: string, // unique member_number, info: done (used as matching field), new (newly added) or error
         title: string,
         pronoun: string,
-        gender: string, // as defined in organization/genders, info: done or warning
+        gender: string, // info: done or warning
         default_password: string,  // info: generated, done or warning
         is_active: boolean,
         is_physical_person: boolean,
         default_vote_weight: decimal(6),  // info: done or error
-        saml_id: string,  // unique saml_id, info: new, done or error
+        idp_id: string,  // unique idp_id, info: new, done or error
     }[];
 }
 ```
@@ -29,8 +29,8 @@ Besides the usual headers as seen in payload (name and type), there are these di
 
 - General user fields (shared between all user imports):
     - `username`: object with info "generated", "new" or "done", depending on whether the username was generated or not. The username may be overwritten when matching via the `member_nubmer`, then the info will be "new"
-    - `saml_id`: object with info "new" if set for the first time or "done" if changed. "error" will be reported on duplicate "saml_ids.
-    - `default_password`: object with info "generated" or "done", depending on whether the default_password was generated or not. The info "warning" signalizes, that `default_password`, `password` and `can_change_own_password` will be removed by setting `saml_id`, because local login will not be possible anymore.
+    - `idp_id`: object with info "new" if set for the first time or "done" if changed. "error" will be reported on duplicate "idp_ids.
+    - `default_password`: object with info "generated" or "done", depending on whether the default_password was generated or not. The info "warning" signalizes, that `default_password`, `password` and `can_change_own_password` will be removed by setting `idp_id`, because local login will not be possible anymore.
     - `email` must be a valid email
     - `member_number`: object with info "new" or "done", depending on whether the member number will be newly added to an existing user or not. Overwriting a pre-existing value is not permitted and "error" will be used if it is tried. "error" will also be used if the member_number is not unique or matches a different user than the other matching criteria
 - `default_vote_weight` doesn't allow 0 values
@@ -52,16 +52,16 @@ To decide whether to update an existing user with a row or to create a new one,
 
 If nothing is found in this manner, a matching user will be sought analogously to the [`search_users` presenter](search_users.md#logic):
 - If `username` is provided, it is only matched by username. All other data is ignored for the matching. If the username does not exist yet, a new username is created.
-- If `saml_id` is provided, it is only matched by saml_id. All other data is ignored for the matching. If the saml_id does not exist yet, a new user is created. If found add a new column with the Id to the data.
-- If `username` and `saml_id` are not provided, all of `first_name`, `last_name` and `email` must be provided instead. A user matches the row if all three fields are equal. In this case fill the `username` in data from db and a also add a column with the Id to data. If no user is found which matches the data, a new user is created and a username generated.
+- If `idp_id` is provided, it is only matched by idp_id. All other data is ignored for the matching. If the idp_id does not exist yet, a new user is created. If found add a new column with the Id to the data.
+- If `username` and `idp_id` are not provided, all of `first_name`, `last_name` and `email` must be provided instead. A user matches the row if all three fields are equal. In this case fill the `username` in data from db and a also add a column with the Id to data. If no user is found which matches the data, a new user is created and a username generated.
 
 If the user is found, a new column with the Id is added to the data and to the object in the main matching field by which the import of the row will be calculated:
 - `member_number` if the user was matched via `member_number`
 - else `username`
 
 One of these cases must be true. If fewer fields are given than necessary (e.g. `first_name` is missing), no matching to existing users is done at all. Instead, a new user is created and a username generated. If both `first_name` and `last_name` are missing, the row is invalid since no username can be generated.
 
-If `saml_id` is given, there may be no password, default_password or can_change_own_password for local user access set.
+If `idp_id` is given, there may be no password, default_password or can_change_own_password for local user access set.
 
 ## Permission
 Organization management level `can_manage_users`

docs/actions/agenda_item.create.md

@@ -13,7 +13,6 @@
     duration: number; // in seconds
     weight: number;
     tag_ids: Id[];
-    moderator_notes: HTML;
 }
 ```
 
@@ -23,5 +22,4 @@ item or the content object cannot have an agenda item (see available collections
 `models.yml`). `tag_ids` must be from the same meeting.
 
 ## Permissions
-The request user needs `agenda_item.can_manage_moderator_notes` to set `moderator_notes` and
-`agenda_item.can_manage` for all other fields.
+The request user needs `agenda_item.can_manage`.

docs/actions/agenda_item.update.md

@@ -12,7 +12,6 @@
     duration: number; // in minutes
     weight: number;
     tag_ids: Id[];
-    moderator_notes: HTML;
 }
 ```
 
@@ -21,5 +20,5 @@ Updates the agenda item. `tag_ids` must be from the same meeting.
 The `type` attribute of one `agenda_item` must be one of [`common`, `internal`, `hidden`].
 
 ## Permissions
-The request user needs `agenda_item.can_manage_moderator_notes` to set `moderator_notes` and
-`agenda_item.can_manage` for all other fields.
+The request user needs `agenda_item.can_manage`.
+

docs/actions/assignment.update.md

@@ -20,5 +20,7 @@
 ## Action
 Updates an assignment.
 
+If phase is newly set to `voting`, the candidates of the assignment are put in the assignments `list_of_speakers` if they are not already.
+
 ## Permissions
 The user needs `assignment.can_manage`.

docs/actions/assignment_candidate.delete.md

@@ -4,7 +4,7 @@
 ```
 
 ## Action
-Deletes an assignment candidate for the assignment. It is forbidden to remove a candidate from a finished assignment.
+Deletes an assignment candidate for the assignment. It is forbidden to remove a candidate from a finished assignment if the action is called externally.
 
 ## Permissions
 If the `assignment_candidate/user_id` is equal to the request user id, the user needs `assignment.can_nominate_self`, else the user needs `assignment.can_nominate_other`.

docs/actions/committee.json_upload.md

@@ -74,7 +74,7 @@ The data, enriched with building some field values and a first new column "state
 
 ### User matching
 
-The users given in `managers` and `meeting_admins` will be matched only by username. The `saml_id` will not be used for the search.
+The users given in `managers` and `meeting_admins` will be matched only by username. The `idp_id` will not be used for the search.
 
 ## Permission
 

docs/actions/gender.create.md

@@ -0,0 +1,13 @@
+## Payload
+```
+{
+// Required
+    name: string;
+}
+```
+
+## Action
+Creates the gender if a gender with the same name doesn't exist.
+
+## Permissions
+The user needs to have the organization management level `can_manage_organization`.

docs/actions/gender.delete.md

@@ -0,0 +1,13 @@
+## Payload
+```
+{
+//Required
+    id: Id;
+}
+```
+
+## Action
+Deletes the gender if it is not one of the four default genders.
+
+## Permissions
+The user needs to have the organization management level `can_manage_organization`.

docs/actions/gender.update.md

@@ -0,0 +1,15 @@
+## Payload
+```
+{
+// Required
+    id: Id;
+// Group A
+    name: string;
+}
+```
+
+## Action
+Updates the gender if a gender with that name does not exist and is not one of the four default genders.
+
+## Permissions
+- Group A: The user needs the OML `can_manage_organization`

docs/actions/group.update.md

@@ -17,9 +17,9 @@ Updates the group. Permissions are restricted to the following enum: https://git
 If the group is the meetings anonymous group, the name may not be changed and the permissions have to be in the following whitelist:
 - agenda_item.can_see,
 - agenda_item.can_see_internal,
-- agenda_item.can_see_moderator_notes,
 - assignment.can_see,
 - list_of_speakers.can_see,
+- list_of_speakers.can_see_moderator_notes,
 - mediafile.can_see,
 - meeting.can_see_autopilot,
 - meeting.can_see_frontpage,

docs/actions/list_of_speakers.update.md

@@ -6,11 +6,13 @@
 
 // Optional
     closed: boolean;
+    moderator_notes: HTML;
 }
 ```
 
 ## Action
 Updates a list of speakers.
 
 ## Permissions
-The request user needs `list_of_speakers.can_manage`.
+The request user needs `list_of_speakers.can_manage_moderator_notes` to set `moderator_notes` and
+`list_of_speakers.can_manage` for all other fields.

docs/actions/meeting.create.md

@@ -27,7 +27,7 @@ When creating a meeting the following objects have to be created, too:
 - Groups: `Default`, `Admin`, `Delegates`, `Staff`, `Committees`. The first one is set as `meeting/default_group_id`, the second one as `meeting/admin_group_id`. The permissions can be found in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)).
 - Projector: One projector named `"Default projector"` must be created and set as `meeting/reference_projector_id`.
 - All default projectors (`meeting/default_projector_*_ids`, see `models.yml`) must be set to the one projector
-- Motion workflow and states: Create one workflow `"simple workflow"` which is set as `meeting/motions_default_workflow_id`, `meeting/motions_default_amendment_workflow_id` and `meeting/motions_default_statute_amendment_workflow_id`. Create four states (analog as in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)).
+- Motion workflow and states: Create one workflow `"simple workflow"` which is set as `meeting/motions_default_workflow_id` and `meeting/motions_default_amendment_workflow_id`. Create four states (analog as in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)).
 - Two countdowns are created and set as `meeting/list_of_speakers_countdown` (name: "List of speakers countdown") and `meeting/voting_countdown` (name: "Voting countdown").
 
 If `user_ids` are given, it must be checked that it is a subset of `committee/user_ids`. Each user is added to the meeting by being added to the default group.