Skip to content

Commit

Permalink
Merge branch 'release/1.9.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
BeritJanssen committed Mar 4, 2024
2 parents a5bb3cf + 42cb3af commit f583ff6
Show file tree
Hide file tree
Showing 253 changed files with 7,538 additions and 2,643 deletions.
2 changes: 1 addition & 1 deletion .env-github-actions
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ AML_LOCATION_PROVIDER=http://ip2country:5000/{}
AML_DEBUG=True
DJANGO_SETTINGS_MODULE=aml.development_settings
REACT_APP_API_ROOT=http://localhost:8000
REACT_APP_EXPERIMENT_SLUG=fe
REACT_APP_EXPERIMENT_SLUG=gold-msi
REACT_APP_AML_HOME=https://www.amsterdammusiclab.nl
REACT_APP_HTML_PAGE_TITLE=Amsterdam Music Lab Experiment
REACT_APP_HTML_OG_DESCRIPTION=Listening experiments from the Amsterdam Music Lab. Test your musical knowledge and skills in engaging citizen-science experiments.
Expand Down
7 changes: 4 additions & 3 deletions .env.dist
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ DJANGO_SUPERUSER_USERNAME=admin # do not use in production!
DJANGO_SUPERUSER_PASSWORD=admin # do not use in production!
[email protected] # do not use in production!
AML_LOCATION_PROVIDER=http://ip2country:5000/{} # address of the ip2country container, don't change
ALLOWED_HOSTS=localhost # needs to be changed when running in production
AML_ALLOWED_HOSTS="backend.muscle.local" # needs to be changed when running in production

REACT_APP_API_ROOT=http://localhost:8000 # address of the server, don't change
REACT_APP_EXPERIMENT_SLUG=gmsi # experiment slug that the frontend redirects to
REACT_APP_API_ROOT=http://backend.muscle.local # address of the server, don't change
REACT_APP_EXPERIMENT_SLUG=gold-msi # experiment slug that the frontend redirects to
REACT_APP_AML_HOME=https://www.amsterdammusiclab.nl # website you will be redirected to if you do not agree with an informed consent form
REACT_APP_LOGO_URL= # optional: link to logo
REACT_APP_HTML_PAGE_TITLE=Amsterdam Music Lab Experiment # optional: information for web crawlers
Expand All @@ -27,3 +27,4 @@ REACT_APP_HTML_OG_TITLE=Amsterdam Music Lab Experiment # optional: information f
REACT_APP_HTML_OG_URL= # optional: information for web crawlers
REACT_APP_HTML_BODY_CLASS= # optional: assign a css class to the <body> tag in index.html, for custom styling
REACT_APP_SENTRY_DSN= # optional: link to sentry instance, e.g. https://[email protected]/xxx
REACT_APP_STRICT= # optional: use StrictMode for development
44 changes: 40 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
- name: Run Backend Tests
run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage run manage.py test"
- name: Generate Backend Coverage Report (Inline)
run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage report"
run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage report --show-missing"

# Generate coverage badge (only for main and develop branches)
- name: Generate Backend Coverage Report (XML) and Badge
Expand All @@ -24,7 +24,7 @@ jobs:
# Push coverage badge to separate branch (only for main and develop branches)
- name: Push Backend Coverage Badge to separate branch
continue-on-error: true
if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
if: github.ref == 'refs/heads/develop'
run: |
if git ls-remote --heads origin code-coverage-badges; then
git fetch origin code-coverage-badges
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Lint Backend
continue-on-error: true
continue-on-error: false
run: sudo docker-compose --env-file .env-github-actions run server bash -c "flake8"

frontend-test:
Expand All @@ -63,7 +63,43 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Run Frontend Tests
run: sudo docker-compose --env-file .env-github-actions run client yarn test:ci --watchAll=false
run: sudo docker-compose --env-file .env-github-actions run client yarn test:ci

frontend-coverage-badge:
name: Generate Frontend Coverage Badge
needs: frontend-test
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/develop'
steps:
- uses: actions/checkout@v3
- name: Generate Frontend Coverage Report (XML) and Badge
run: |
sudo docker-compose --env-file .env-github-actions run client yarn test:ci --coverageDirectory=public/coverage
sudo docker-compose --env-file .env-github-actions run client yarn coverage-badges -s public/coverage/coverage-summary.json -o public/coverage/coverage-frontend-badge-new.svg --label 'Frontend Code Coverage'
- name: Push Frontend Coverage Badge to separate branch
continue-on-error: true
run: |
if git ls-remote --heads origin code-coverage-badges; then
git fetch origin code-coverage-badges
git checkout code-coverage-badges
else
git checkout -b code-coverage-badges
git push origin code-coverage-badges
fi
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
rm frontend/coverage-frontend-badge.svg -f
sudo mv frontend/public/coverage/coverage-frontend-badge-new.svg frontend/coverage-frontend-badge.svg
git add frontend/coverage-frontend-badge.svg
git commit -m "Add frontend coverage badge for commit $GITHUB_SHA"
git push origin code-coverage-badges
# Check if there are any changes
if git diff --staged --quiet; then
echo "No changes in coverage badge. Skipping commit and push."
else
git commit -m "Add frontend coverage badge for commit $GITHUB_SHA"
git push origin code-coverage-badges
fi
frontend-lint:
name: Lint Frontend
Expand Down
132 changes: 132 additions & 0 deletions .github/workflows/podman.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
name: Podman build & deploy

on:
push:
branches:
- develop
workflow_dispatch:

# temporarily also for PRs
pull_request:
branches:
- develop

jobs:
deploy-test:
name: Deploy to test environment
environment: Test
runs-on: tst
if: github.ref == 'refs/heads/develop'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
env:

# Variables
AML_ALLOWED_HOSTS: ${{ vars.AML_ALLOWED_HOSTS }}
AML_CORS_ORIGIN_WHITELIST: ${{ vars.AML_CORS_ORIGIN_WHITELIST }}
AML_DEBUG: ${{ vars.AML_DEBUG }}
AML_LOCATION_PROVIDER: ${{ vars.AML_LOCATION_PROVIDER }}
AML_SUBPATH: ${{ vars.AML_SUBPATH }}
DJANGO_SETTINGS_MODULE: ${{ vars.DJANGO_SETTINGS_MODULE }}
SQL_DATABASE: ${{ vars.SQL_DATABASE }}
SQL_HOST: ${{ vars.SQL_HOST }}
SQL_PORT: ${{ vars.SQL_PORT }}
REACT_APP_API_ROOT: ${{ vars.REACT_APP_API_ROOT }}
REACT_APP_EXPERIMENT_SLUG: ${{ vars.REACT_APP_EXPERIMENT_SLUG }}
REACT_APP_AML_HOME: ${{ vars.REACT_APP_AML_HOME }}
REACT_APP_HTML_PAGE_TITLE: ${{ vars.REACT_APP_HTML_PAGE_TITLE }}

# Secrets
AML_SECRET_KEY: ${{ secrets.AML_SECRET_KEY }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
SQL_USER: ${{ secrets.SQL_USER }}
SQL_PASSWORD: ${{ secrets.SQL_PASSWORD }}
REACT_APP_SENTRY_DSN: ${{ secrets.REACT_APP_SENTRY_DSN }}
DJANGO_SUPERUSER_USERNAME: ${{ secrets.DJANGO_SUPERUSER_USERNAME }}
DJANGO_SUPERUSER_PASSWORD: ${{ secrets.DJANGO_SUPERUSER_PASSWORD }}
DJANGO_SUPERUSER_EMAIL: ${{ secrets.DJANGO_SUPERUSER_EMAIL }}

# Prevent podman services from exiting after startup
RUNNER_TRACKING_ID: ""

steps:
- uses: actions/checkout@v4
- name: Create .env file
run: |
touch .env
echo "REACT_APP_API_ROOT=$REACT_APP_API_ROOT" >> .env
echo "REACT_APP_EXPERIMENT_SLUG=$REACT_APP_EXPERIMENT_SLUG" >> .env
echo "REACT_APP_AML_HOME=$REACT_APP_AML_HOME" >> .env
echo "REACT_APP_HTML_PAGE_TITLE=$REACT_APP_HTML_PAGE_TITLE" >> .env
echo "REACT_APP_SENTRY_DSN=$REACT_APP_SENTRY_DSN" >> .env
cp .env frontend/.env
- name: Build Podman images
run: podman-compose -f docker-compose-deploy.yml build
- name: Deploy Podman images
run: podman-compose -f docker-compose-deploy.yml up -d --force-recreate
- name: Prune old images
run: podman image prune -a -f
- name: Check Podman images
run: podman-compose -f docker-compose-deploy.yml ps
- name: Check logs
run: podman-compose -f docker-compose-deploy.yml logs

deploy-acceptance:
name: Deploy to acceptance environment
environment: Acceptance
runs-on: ACC
if: github.ref == 'refs/heads/main' || github.ref == 'refs/tags/*' || 1 == 1 # Temporarily always run during testing
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
env:

# Variables
AML_ALLOWED_HOSTS: ${{ vars.AML_ALLOWED_HOSTS }}
AML_CORS_ORIGIN_WHITELIST: ${{ vars.AML_CORS_ORIGIN_WHITELIST }}
AML_DEBUG: ${{ vars.AML_DEBUG }}
AML_LOCATION_PROVIDER: ${{ vars.AML_LOCATION_PROVIDER }}
AML_SUBPATH: ${{ vars.AML_SUBPATH }}
DJANGO_SETTINGS_MODULE: ${{ vars.DJANGO_SETTINGS_MODULE }}
SQL_DATABASE: ${{ vars.SQL_DATABASE }}
SQL_HOST: ${{ vars.SQL_HOST }}
SQL_PORT: ${{ vars.SQL_PORT }}
REACT_APP_API_ROOT: ${{ vars.REACT_APP_API_ROOT }}
REACT_APP_EXPERIMENT_SLUG: ${{ vars.REACT_APP_EXPERIMENT_SLUG }}
REACT_APP_AML_HOME: ${{ vars.REACT_APP_AML_HOME }}
REACT_APP_HTML_PAGE_TITLE: ${{ vars.REACT_APP_HTML_PAGE_TITLE }}

# Secrets
AML_SECRET_KEY: ${{ secrets.AML_SECRET_KEY }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
SQL_USER: ${{ secrets.SQL_USER }}
SQL_PASSWORD: ${{ secrets.SQL_PASSWORD }}
REACT_APP_SENTRY_DSN: ${{ secrets.REACT_APP_SENTRY_DSN }}
DJANGO_SUPERUSER_USERNAME: ${{ secrets.DJANGO_SUPERUSER_USERNAME }}
DJANGO_SUPERUSER_PASSWORD: ${{ secrets.DJANGO_SUPERUSER_PASSWORD }}
DJANGO_SUPERUSER_EMAIL: ${{ secrets.DJANGO_SUPERUSER_EMAIL }}

# Prevent podman services from exiting after startup
RUNNER_TRACKING_ID: ""

steps:
- uses: actions/checkout@v4
- name: Create .env file
run: |
touch .env
echo "REACT_APP_API_ROOT=$REACT_APP_API_ROOT" >> .env
echo "REACT_APP_EXPERIMENT_SLUG=$REACT_APP_EXPERIMENT_SLUG" >> .env
echo "REACT_APP_AML_HOME=$REACT_APP_AML_HOME" >> .env
echo "REACT_APP_HTML_PAGE_TITLE=$REACT_APP_HTML_PAGE_TITLE" >> .env
echo "REACT_APP_SENTRY_DSN=$REACT_APP_SENTRY_DSN" >> .env
cp .env frontend/.env
- name: Build Podman images
run: podman-compose -f docker-compose-deploy.yml build
- name: Deploy Podman images
run: podman-compose -f docker-compose-deploy.yml up -d --force-recreate
- name: Prune old images
run: podman image prune -a -f
- name: Check Podman images
run: podman-compose -f docker-compose-deploy.yml ps
- name: Check logs
run: podman-compose -f docker-compose-deploy.yml logs

10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[![DOI](https://zenodo.org/badge/418963353.svg)](https://zenodo.org/badge/latestdoi/418963353) [![Test](https://github.com/Amsterdam-Music-Lab/MUSCLE/actions/workflows/ci.yml/badge.svg)](https://github.com/Amsterdam-Music-Lab/MUSCLE/actions/workflows/test.yml/badge.svg)
![Backend Code Coverage Percentage](https://raw.githubusercontent.com/Amsterdam-Music-Lab/MUSCLE/code-coverage-badges/backend/coverage-backend-badge.svg)
![Frontend Code Coverage Percentage](https://raw.githubusercontent.com/Amsterdam-Music-Lab/MUSCLE/code-coverage-badges/frontend/coverage-frontend-badge.svg)

# MUSic-related Citizen Science Listening Experiments (MUSCLE)
This application provides an easy way to implement and run online listening experiments for music research. It presents questions, and typically audio stimuli, to participants, and collects their feedback.
Expand All @@ -23,11 +24,10 @@ Install [Docker Desktop](https://docs.docker.com/desktop/).
### Linux
* Install [Docker Engine](https://docs.docker.com/engine/install/)
* Install [Docker Compose](https://docs.docker.com/compose/install/)

As of April 2022, [Docker Desktop for Linux](https://docs.docker.com/desktop/linux/) is still in Beta and have not been tested by us.
* Install [Docker Desktop](https://docs.docker.com/desktop/install/linux-install/)

## Development build
Make a copy of the file .env.dist (in the same directory as this README) and rename it to .env. This file contains variables used by Docker to start up a container network serving MUSCLE.
Make a copy of [the file](https://github.com/Amsterdam-Music-Lab/MUSCLE/blob/develop/.env.dist) `.env.dist` (in the same directory as this README) and rename it to `.env.` This file contains variables used by Docker to start up a container network serving MUSCLE.

Start Docker (the app icon is a whale carrying containers). Then, open a terminal and run
`docker-compose up` (add `sudo` on Linux).
Expand All @@ -46,3 +46,7 @@ To stop the containers, press `ctrl-c` or (in another terminal) run

## Production build
A production build should define its own `docker-compose.yaml`, making use of the `Dockerfile` of the `backend` and `frontend` environments. It should also define a custom .env file, with safe passwords for the SQL database and the Python backend. Instead of mounting the entire backend and frontend directory and using the development servers, the backend should serve with gunicorn, and the frontend should use a build script to compile static html, css and JavaScript.

## Troubleshooting

Please refer to the [wiki](https://github.com/Amsterdam-Music-Lab/MUSCLE/wiki/X.-Troubleshooting) a checklist of common issues and their solutions.
46 changes: 46 additions & 0 deletions backend/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
[flake8]
# Flake8 Configuration File

# High Priority: Security and Correctness
# These issues are critical and should be addressed first.
extend-ignore =
E722, # Do not use bare 'except'
F722, # Syntax error identified by pyflakes
F821, # Undefined name

# Medium Priority: Code Maintainability and Readability
# Improving these can greatly enhance code readability and maintainability.
E501, # Line too long
F401, # Unused import
F403, # 'from module import *' used; unable to detect undefined names
F405, # Name may be undefined, or defined from star imports
F811, # Redefinition of unused name from line N
F841, # Local variable name is assigned to but never used

# Low Priority: Style Guide Adherence
# These are mostly about whitespace and indentation, which can be adjusted later.
E201, # Whitespace after '('
E202, # Whitespace before ')'
E203, # Whitespace before ':'
E222, # Multiple spaces after operator
E225, # Missing whitespace around operator
E231, # Missing whitespace after ','
E251, # Unexpected spaces around keyword / parameter equals
E262, # Inline comment should start with '# '
E122, # Continuation line missing indentation or outdented
E124, # Closing bracket does not match visual indentation
E125, # Continuation line with same indent as next logical line
E127, # Continuation line over-indented for visual indent
E128, # Continuation line under-indented for visual indent
E131, # Continuation line unaligned for hanging indent
E261, # At least two spaces before inline comment
W191, # Indentation contains tabs
W291, # Trailing whitespace
W292, # No newline at end of file
W293, # Blank line contains whitespace
W391, # Blank line at end of file
W503, # Line break occurred before a binary operator

# General Configuration
max-line-length = 120
exclude = .venv, .git, __pycache__, build, dist
6 changes: 3 additions & 3 deletions backend/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3.8
FROM docker.io/python:3.8
ENV PYTHONUNBUFFERED 1
RUN apt-get -y update
RUN apt-get install -y ffmpeg
Expand All @@ -7,6 +7,6 @@ RUN apt-get install -y gettext
WORKDIR /server
COPY requirements/prod.txt /server/
RUN pip install -r prod.txt
# We add remainig code later, so pip install won't need to rerun if source code changes
COPY . /server/

# We add remainig code later, so pip install won't need to rerun if source code changes
COPY . /server/
5 changes: 2 additions & 3 deletions backend/DockerfileDevelop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
FROM python:3.8
FROM docker.io/python:3.8 as base
ENV PYTHONUNBUFFERED 1
RUN apt-get -y update
RUN apt-get install -y ffmpeg

WORKDIR /server
COPY requirements/dev.txt /server/
RUN pip install -r dev.txt

RUN pip install -r dev.txt
36 changes: 25 additions & 11 deletions backend/aml/base_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,12 @@
"""

import os
import logging
from corsheaders.defaults import default_headers
import sentry_sdk

logger = logging.getLogger(__name__)

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

Expand All @@ -40,6 +43,7 @@
'django.contrib.messages',
'django.contrib.staticfiles',
'inline_actions',
'django_markup',
'corsheaders',
'experiment',
'participant',
Expand Down Expand Up @@ -165,14 +169,24 @@

DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'

# Sentry
sentry_sdk.init(
dsn=os.getenv("SENTRY_DSN", ""),
# Set traces_sample_rate to 1.0 to capture 100%
# of transactions for performance monitoring.
traces_sample_rate=0.2,
# Set profiles_sample_rate to 1.0 to profile 100%
# of sampled transactions.
# We recommend adjusting this value in production.
profiles_sample_rate=0.2,
)
if os.getenv("SENTRY_DSN"):
sentry_sdk.init(
dsn=os.getenv("SENTRY_DSN", ""),
# Set traces_sample_rate to 1.0 to capture 100%
# of transactions for performance monitoring.
traces_sample_rate=0.2,
# Set profiles_sample_rate to 1.0 to profile 100%
# of sampled transactions.
# We recommend adjusting this value in production.
profiles_sample_rate=0.2,
)
else:
logger.info("SENTRY_DSN is not defined. Skipping Sentry initialization.")

MARKUP_SETTINGS = {
'markdown': {
'safe_mode': True
}
}

SUBPATH = os.getenv('AML_SUBPATH', None)
Loading

0 comments on commit f583ff6

Please sign in to comment.