[DO NOT MERGE]: Diff for scalableminds vs LINC webknossos deployments

.circleci/config.yml

@@ -3,23 +3,15 @@ jobs:
   build_test_deploy:
     machine:
       image: ubuntu-2004:202111-02
-      docker_layer_caching: false
+      docker_layer_caching: true
     resource_class: large
     environment:
       USER_NAME: circleci
       USER_UID: 1000
       USER_GID: 1000
       TZ: Europe/Berlin
     steps:
-      - run:
-          name: "Clone Repository"
-          command: |
-            if [ -n "$CIRCLE_BRANCH" ]; then
-              git clone --depth 5 "$CIRCLE_REPOSITORY_URL" --branch "$CIRCLE_BRANCH" --no-tags .
-            fi
-            if [ -n "$CIRCLE_TAG" ]; then
-              git clone --depth 5 "$CIRCLE_REPOSITORY_URL" --branch "$CIRCLE_TAG" .
-            fi
+      - checkout
       - run:
           name: "Custom environment variables"
           command: |
@@ -35,28 +27,48 @@ jobs:
       - run:
           name: Build webknossos-dev docker image
           command: |
-            docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
+            docker pull lincbrain/webknossos-dev:$NORMALIZED_BRANCH || true
             DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base
       - run:
           name: Prepare dependency folders
-          command: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+          command: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier ~/.cache/yarn
+      - restore_cache:
+          name: Restore target cache
+          keys:
+            - target-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}
+            - target-cache-{{ checksum ".circleci/cache_version" }}-master
+      - restore_cache:
+          name: Restore sbt cache
+          keys:
+            - sbt-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "project/Dependencies.scala" }}
+            - sbt-cache-{{ checksum ".circleci/cache_version" }}-
+      - restore_cache:
+          name: Restore yarn cache
+          keys:
+            - yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }}
+            - yarn-cache-{{ checksum ".circleci/cache_version" }}-
       - run:
           name: Install frontend dependencies
-          command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
+          command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --frozen-lockfile
       - run:
           name: Assert unique evolution numbers
           command: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
-      - run:
-          name: Assert schema.sql and evolutions are equal
-          command: |
-            docker-compose up -d postgres
-            sleep 3
-            docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
-      - run:
-          name: Build frontend documentation
-          command: |
-            WK_VERSION=${CIRCLE_TAG:-${CIRCLE_BUILD_NUM:-dev}}
-            docker-compose run base yarn run docs --project-version $WK_VERSION
+      - restore_cache:
+          name: Restore webpack cache
+          keys:
+            - webpack-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}
+            - webpack-cache-{{ checksum ".circleci/cache_version" }}-master
+#      - run:
+#          name: Assert schema.sql and evolutions are equal
+#          command: |
+#            docker-compose up -d postgres
+#            sleep 3
+#            docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
+#      - run:
+#          name: Build frontend documentation
+#          command: |
+#            WK_VERSION=${CIRCLE_TAG:-${CIRCLE_BUILD_NUM:-dev}}
+#            docker-compose run base yarn run docs --project-version $WK_VERSION
       - run:
           name: Build webknossos (webpack)
           command: |
@@ -77,15 +89,35 @@ jobs:
           name: Build webknossos-tracingstore (sbt)
           command: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
 
-      - run:
-          name: Checksum App Dirs
-          command: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
+      - save_cache:
+          name: Save target cache
+          key: target-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - "project/target"
+            - "target"
+
+      - save_cache:
+          name: Save sbt cache
+          key: sbt-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "project/Dependencies.scala" }}
+          paths:
+            - "~/.ivy2"
+            - "~/.cache/coursier"
+
+      - save_cache:
+          name: Save yarn cache
+          key: yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }}
+          paths:
+            - "~/.cache/yarn"
+
+      - save_cache:
+          name: Save webpack cache
+          key: webpack-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - "node_modules/.cache/webpack"
 
       - run:
           name: Build webknossos docker image
-          command: |
-            docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
-            DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos
+          command: docker-compose build --pull webknossos
 
       - run:
           name: Build webknossos-datastore docker image
@@ -95,70 +127,84 @@ jobs:
           name: Build webknossos-tracingstore docker image
           command: docker-compose build --pull webknossos-tracingstore
 
-      - run:
-          name: Lint frontend code and check formatting
-          command: |
-            .circleci/not-on-master.sh docker-compose run base bash -c "yarn run check-frontend"
-      - run:
-          name: Check for cyclic dependencies in front-end
-          command: |
-            .circleci/not-on-master.sh docker-compose run base yarn check-cyclic-dependencies
-      - run:
-          name: Run frontend tests
-          command: |
-            .circleci/not-on-master.sh docker-compose run base yarn test-verbose
-      - run:
-          name: Lint backend code and check formatting
-          command: |
-            .circleci/not-on-master.sh docker-compose run backend-lint-format
-      - run:
-          name: Run backend tests
-          command: |
-            .circleci/not-on-master.sh docker-compose run backend-tests
-      - run:
-          name: Run end-to-end tests
-          command: |
-            for i in {1..3}; do # retry
-              .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
-            done
-            (exit $s)
-      - run:
-          name: Validate frontend types
-          command: |
-            .circleci/not-on-master.sh docker-compose run base yarn typecheck
-      - run:
-          name: Start webknossos, datastore and tracingstore
-          command: docker-compose up -d webknossos && docker-compose up -d webknossos-datastore && docker-compose up -d webknossos-tracingstore
-
-      - run:
-          name: Run webknossos smoke test
-          command: |
-            for i in {1..20}; do # retry
-              curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$?
-              sleep 5
-            done
-            (exit $s)
-
-      - run:
-          name: Run webknossos-datastore smoke test
-          command: |
-            for i in {1..20}; do # retry
-              curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$?
-              sleep 5
-            done
-            (exit $s)
-
-      - run:
-          name: Run webknossos-tracingstore smoke test
-          command: |
-            for i in {1..20}; do # retry
-              curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$?
-              sleep 5
-            done
-            (exit $s)
-      - run:
-          name: Stop webknossos, datastore and tracingstore
-          command: docker-compose down --volumes --remove-orphans
+#      - run:
+#          name: Lint frontend code and check formatting
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run base bash -c "yarn run check-frontend"
+#      - run:
+#          name: Check for cyclic dependencies in front-end
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run base yarn check-cyclic-dependencies
+#      - run:
+#          name: Run frontend tests
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run base yarn test-verbose
+#      - run:
+#          name: Lint backend code and check formatting
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run backend-lint-format
+#      - run:
+#          name: Run backend tests
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run backend-tests
+#      - run:
+#          name: Run end-to-end tests
+#          command: |
+#            for i in {1..3}; do # retry
+#              .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
+#            done
+#            (exit $s)
+#      - run:
+#          name: Validate frontend types
+#          command: |
+#            .circleci/not-on-master.sh docker-compose run base yarn typecheck
+#      - run:
+#          name: Start webknossos
+#          background: true
+#          command: docker-compose up webknossos
+#      - run:
+#          name: Run webknossos smoke test
+#          command: |
+#            for i in {1..10}; do # retry
+#              sleep 10
+#              curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$?
+#            done
+#            (exit $s)
+#      - run:
+#          name: Stop webknossos
+#          command: docker-compose down --volumes --remove-orphans
+#
+#      - run:
+#          name: Start webknossos-datastore
+#          background: true
+#          command: docker-compose up webknossos-datastore
+#      - run:
+#          name: Run webknossos-datastore smoke test
+#          command: |
+#            for i in {1..10}; do # retry
+#              sleep 10
+#              curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$?
+#            done
+#            (exit $s)
+#      - run:
+#          name: Stop webknossos-datastore
+#          command: docker-compose down --volumes --remove-orphans
+#
+#      - run:
+#          name: Start webknossos-tracingstore
+#          background: true
+#          command: docker-compose up webknossos-tracingstore
+#      - run:
+#          name: Run webknossos-tracingstore smoke test
+#          command: |
+#            for i in {1..10}; do # retry
+#              sleep 10
+#              curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$?
+#            done
+#            (exit $s)
+#      - run:
+#          name: Stop webknossos-tracingstore
+#          command: docker-compose down --volumes --remove-orphans
 
       - run:
           name: Push docker images
@@ -176,21 +222,17 @@ jobs:
             retry docker-compose push webknossos-tracingstore
             if [ -n "$CIRCLE_BRANCH" ]; then
               docker tag \
-                scalableminds/webknossos:${DOCKER_TAG} \
-                scalableminds/webknossos:${NORMALIZED_BRANCH}
-              retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
-              docker tag \
-                scalableminds/webknossos-datastore:${DOCKER_TAG} \
-                scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
-              retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+                lincbrain/webknossos:${DOCKER_TAG} \
+                lincbrain/webknossos:${NORMALIZED_BRANCH}
+              retry docker push lincbrain/webknossos:${NORMALIZED_BRANCH}
               docker tag \
-                scalableminds/webknossos-tracingstore:${DOCKER_TAG} \
-                scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
-              retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+                lincbrain/webknossos-datastore:${DOCKER_TAG} \
+                lincbrain/webknossos-datastore:${NORMALIZED_BRANCH}
+              retry docker push lincbrain/webknossos-datastore:${NORMALIZED_BRANCH}
               docker tag \
-                scalableminds/webknossos-dev \
-                scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
-              retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+                lincbrain/webknossos-tracingstore:${DOCKER_TAG} \
+                lincbrain/webknossos-tracingstore:${NORMALIZED_BRANCH}
+              retry docker push lincbrain/webknossos-tracingstore:${NORMALIZED_BRANCH}
             fi
             docker logout
       - run:
@@ -226,7 +268,7 @@ jobs:
       - run:
           name: Install dependencies and sleep at least 3min
           command: |
-            yarn install --immutable &
+            yarn install --frozen-lockfile &
             sleep 180 &
             wait
       - run:
@@ -247,15 +289,6 @@ jobs:
       - store_artifacts:
           path: frontend/javascripts/test/snapshots/type-check
 
-      - run:
-          name: Bundle screenshots
-          when: always
-          command: |
-            tar -czvf screenshots.tar frontend/javascripts/test/screenshots
-
-      - store_artifacts:
-          path: screenshots.tar
-
   wkorg_nightly:
     docker:
       - image: scalableminds/puppeteer:master
@@ -265,24 +298,7 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            yarn install --immutable
-
-      - run:
-          name: Assert GZIP
-          command: |
-            # Test gzipped assets
-            curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.js | grep -q "content-encoding: gzip"
-            curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.css | grep -q "content-encoding: gzip"
-            # Test gzipped buckets
-            curl -s -i \
-              -H 'accept: application/octet-stream' \
-              -H 'Accept-Encoding: gzip' \
-              -H 'content-type: application/json' \
-              --data-raw '[{"position":[2752,4320,1728],"additionalCoordinates":[],"mag":[1,1,1],"cubeSize":32,"fourBit":false}]' \
-              'https://data-humerus.webknossos.org/data/datasets/scalable_minds/l4dense_motta_et_al_demo/layers/segmentation/data?token=' \
-              | grep -q "content-encoding: gzip"
-            echo Success.
-
+            yarn install --frozen-lockfile
       - run:
           name: Run screenshot-tests
           command: |

CHANGELOG.unreleased.md

@@ -11,6 +11,31 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/24.10.0...HEAD)
 
 ### Added
+- Added the option for the owner to lock explorative annotations. Locked annotations cannot be modified by any user. An annotation can be locked in the annotations table and when viewing the annotation via the navbar dropdown menu. [#7801](https://github.com/scalableminds/webknossos/pull/7801)
+- Added the option to set a default mapping for a dataset in the dataset view configuration. The default mapping is loaded when the dataset is opened and the user / url does not configure something else. [#7858](https://github.com/scalableminds/webknossos/pull/7858)
+- Uploading an annotation into a dataset that it was not created for now also works if the dataset is in a different organization. [#7816](https://github.com/scalableminds/webknossos/pull/7816)
+- When downloading + reuploading an annotation that is based on a segmentation layer with active mapping, that mapping is now still be selected after the reupload. [#7822](https://github.com/scalableminds/webknossos/pull/7822)
+- In the Voxelytics workflow list, the name of the WEBKNOSSOS user who started the job is displayed. [#7794](https://github.com/scalableminds/webknossos/pull/7795)
+- Start an alignment job (aligns the section in a dataset) via the "AI Analysis" button. [#7820](https://github.com/scalableminds/webknossos/pull/7820)
+- Added additional validation for the animation job modal. Bounding boxes must be larger then zero. [#7883](https://github.com/scalableminds/webknossos/pull/7883)
+
+### Changed
+- The "WEBKNOSSOS Changelog" modal now lazily loads its content potentially speeding up the initial loading time of WEBKNOSSOS and thus improving the UX. [#7843](https://github.com/scalableminds/webknossos/pull/7843)
+- Updated the min max settings for the histogram to allow floating point color layers to have negative min / max values. [#7873](https://github.com/scalableminds/webknossos/pull/7873)
+- Made the login, registration, forgot password and dataset dashboard pages more mobile friendly. [#7876](https://github.com/scalableminds/webknossos/pull/7876)
+- From now on only project owner get a notification email upon project overtime. The organization specific email list `overTimeMailingList` was removed. [#7842](https://github.com/scalableminds/webknossos/pull/7842)
+- Replaced skeleton comment tab component with antd's `<Tree />`component. [#7802](https://github.com/scalableminds/webknossos/pull/7802)
+
+### Fixed
+- Fixed a bug where the warning to zoom in to see the agglomerate mapping was shown to the user even when the 3D viewport was maximized and no volume data was shown. [#7865](https://github.com/scalableminds/webknossos/issues/7865) 
+- Fixed a bug where brushing on a fallback segmentation with active mapping and with segment index file would lead to failed saves. [#7833](https://github.com/scalableminds/webknossos/pull/7833)
+- Fixed a bug where sometimes old mismatching javascript code would be served after upgrades. [#7854](https://github.com/scalableminds/webknossos/pull/7854)
+- Fixed a bug where dataset uploads of zipped tiff data via the UI would be rejected. [#7856](https://github.com/scalableminds/webknossos/pull/7856)
+- Fixed a bug with incorrect valiation of layer names in the animation modal. [#7882](https://github.com/scalableminds/webknossos/pull/7882)
+
+- It is now possible to focus a bounding box in the bounding box tab by clicking its edges in a viewport or via a newly added context menu entry. [#8054](https://github.com/scalableminds/webknossos/pull/8054)
+### Added
+- Added an assertion to the backend to ensure unique keys in the metadata info of datasets and folders. [#8068](https://github.com/scalableminds/webknossos/issues/8068)
 - It is now possible to add metadata in annotations to Trees and Segments. [#7875](https://github.com/scalableminds/webknossos/pull/7875)
 - Added a summary row to the time tracking overview, where times and annotations/tasks are summed. [#8092](https://github.com/scalableminds/webknossos/pull/8092)
 - Most sliders have been improved: Wheeling above a slider now changes its value and double-clicking its knob resets it to its default value. [#8095](https://github.com/scalableminds/webknossos/pull/8095)