diff --git a/.gitignore b/.gitignore index 9c1f13c..492097d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt +# Local dev secrets +k8s/envs/local/secrets.env + # Unit test / coverage reports htmlcov/ .tox/ @@ -161,8 +164,11 @@ cython_debug/ # vscode .vscode +.devenv +.pre-commit-config.yaml # temp files tmp/ tmp/**/* dump.rdb +local-kubeconfig diff --git a/README.md b/README.md index 41856f9..2069473 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ This application is the backend server for the PhotonRanch Datalab. It is a djan ## Local Development +### Bare Metal Start by creating a virtualenv for this project and entering it: ``` python -m venv /path/to/my/virtualenv @@ -38,6 +39,23 @@ Now start your server ./manage.py runserver ``` +### Nix development +For this mode of development, you must install: +- nix with flakes support + +Then to develop, run these commands: +- `nix develop --impure` to start your nix development environment - **called anytime you use a new terminal** +- `ctlptl apply -f local-registry.yaml -f local-cluster.yaml` to start up the registry and cluster - **should only need to be called one time within the nix environment** +- `skaffold dev -m deps` to start the dependencies - **run this in a different tab to keep running during development or use 'run' instead of 'dev'** +- Copy `./k8s/envs/local/secrets.env.changeme` to a version without `.changeme` and fill in values for connecting to the appropriate services. +- `skaffold dev -m app --port-forward` to start the servers and worker. This will auto-redeploy as you make changes to the code. + +### Connecting a frontend +You can also run a local [datalab-ui](https://github.com/LCOGT/datalab-ui) to connect to your datalab. Assuming you've cloned that repo: +- Change the `./public/config/config.json` "datalabApiBaseUrl" to be `http://127.0.0.1:8080/api/` or wherever your backend is deployed to +- `npm install` to install the libraries +- `npm run serve` to run the server at `http://127.0.0.1:8081` assuming your backend was already running (otherwise it will try to be :8080) + ## API Structure The application has a REST API with the following endpoints you can use. You must pass your user's API token in the request header to access any of the endpoints - the headers looks like `{'Authorization': 'Token 123456789abcdefg'}` if you are using python's requests library. @@ -105,9 +123,4 @@ Available Operations are introspected from the `data_operations` directory and m `DELETE /api/datasessions/datasession_id/operations/operation_id/` ## ROADMAP -* Come up with operation `wizard_description` format and add endpoint to get them for all available operations so the frontend can auto-create UI wizards for new operations. -* Figure out user accounts between PTR and datalab - datalab needs user accounts for permissions to gate access to only your own sessions. -* Implement operations to actually do something when they are added to a session - * Figure out caching and storage of intermediate results - * Figure out asynchronous task queue or temporal for executing operations - * Add in operation results/status to the serialized operations output (maybe to the model too as needed) +* TBD diff --git a/datalab/settings.py b/datalab/settings.py index 15566ff..d30fb36 100644 --- a/datalab/settings.py +++ b/datalab/settings.py @@ -223,8 +223,8 @@ def get_list_from_env(variable, default=None): # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ -STATIC_URL = '/static/' -STATIC_ROOT = '/static/' +STATIC_URL = os.getenv('STATIC_URL', '/static/') +STATIC_ROOT = os.getenv('STATIC_ROOT', '/static/') # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field diff --git a/datalab/urls.py b/datalab/urls.py index 234108b..083a77c 100644 --- a/datalab/urls.py +++ b/datalab/urls.py @@ -15,6 +15,8 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin +from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.conf import settings from django.urls import path, re_path, include from rest_framework_nested import routers import ocs_authentication.auth_profile.urls as authprofile_urls @@ -39,3 +41,6 @@ path('api/available_operations/', OperationOptionsApiView.as_view(), name='available_operations'), re_path(r'^authprofile/', include(authprofile_urls)), ] + +if settings.DEBUG: + urlpatterns += staticfiles_urlpatterns() diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..bb5ea90 --- /dev/null +++ b/flake.lock @@ -0,0 +1,756 @@ +{ + "nodes": { + "cachix": { + "inputs": { + "devenv": "devenv_2", + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1712055811, + "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", + "owner": "cachix", + "repo": "cachix", + "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + "deploy-repo-template": { + "flake": false, + "locked": { + "lastModified": 1725037657, + "narHash": "sha256-p1ZfPG60DqmcUGJX8rkCVFr0Wh7ZVLCn/BoM6IGTbOo=", + "ref": "refs/heads/main", + "rev": "82058b4c22070bdb386e13b0edadb45742dc3b2e", + "revCount": 17, + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + }, + "original": { + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": "nixpkgs_2", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1724763216, + "narHash": "sha256-oW2bwCrJpIzibCNK6zfIDaIQw765yMAuMSG2gyZfGv0=", + "owner": "cachix", + "repo": "devenv", + "rev": "1e4ef61205b9aa20fe04bf1c468b6a316281c4f1", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv-k8s": { + "inputs": { + "deploy-repo-template": "deploy-repo-template", + "devenv": "devenv", + "flake-parts": "flake-parts", + "kpt": "kpt", + "mk-shell-bin": "mk-shell-bin", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_3", + "octopilot": "octopilot", + "skaffold": "skaffold" + }, + "locked": { + "lastModified": 1725659503, + "narHash": "sha256-GklipTzYxNtpxifQaW1GFesUzIiRowrrM7IL8Ql3bqw=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "98c64c1fd378135b72963dcdf3a65fd79ebc0b81", + "type": "github" + }, + "original": { + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "cachix", + "flake-compat" + ], + "nix": "nix", + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix", + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "cachix", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1708704632, + "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", + "owner": "cachix", + "repo": "devenv", + "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "python-rewrite", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": "flake-utils_4", + "nixpkgs": [ + "devenv-k8s", + "skaffold", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1710154385, + "narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "872b63ddd28f318489c929d25f1f0a3c6039c971", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "gomod2nix", + "type": "github" + } + }, + "kpt": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "kpt": "kpt_2", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "dir": "kpt", + "lastModified": 1724262194, + "narHash": "sha256-+WcLnbPBGKOD04ogx/J9QS/OfxAVXcH1JvgahA5okRM=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "dbc88d8238fbfcb3bd08c66157bef4b8dc63b379", + "type": "github" + }, + "original": { + "dir": "kpt", + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" + } + }, + "kpt_2": { + "flake": false, + "locked": { + "lastModified": 1700611672, + "narHash": "sha256-GHEk5nezva+26cpCrgriaGIL2PvSLWgC0UtmFkNHoDc=", + "owner": "kptdev", + "repo": "kpt", + "rev": "aff697dbfc8134b059bcfbdfb792a1048aaa57b5", + "type": "github" + }, + "original": { + "owner": "kptdev", + "ref": "v1.0.0-beta.48", + "repo": "kpt", + "type": "github" + } + }, + "mk-shell-bin": { + "locked": { + "lastModified": 1677004959, + "narHash": "sha256-/uEkr1UkJrh11vD02aqufCxtbF5YnhRTIKlx5kyvf+I=", + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", + "type": "github" + }, + "original": { + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1724921078, + "narHash": "sha256-g6Zb7OrlNEM1lep9e/Neqn8Z+0VgCXAakF03GjtSfA8=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "505c19a99771ca2f7b049b3c64f190ccc158861f", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nix_2": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1692808169, + "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1722555339, + "narHash": "sha256-uFf2QeW7eAHlYXuDktm9c25OxOyCoUOQmh5SZ9amE5Q=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1710695816, + "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "614b4613980a522ba49f0d194531beddbb7220d3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1713361204, + "narHash": "sha256-TA6EDunWTkc5FvDCqU3W2T3SFn0gRZqh6D/hJnM02MM=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "285676e87ad9f0ca23d8714a6ab61e7e027020c6", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1716977621, + "narHash": "sha256-Q1UQzYcMJH4RscmpTkjlgqQDX5yi1tZL0O345Ri6vXQ=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "4267e705586473d3e5c8d50299e71503f16a6fb6", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "octopilot": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1724261402, + "narHash": "sha256-ablZLlVPhrC5cShu2/5e80cbrR5MOMGc7Fpgf37HNbQ=", + "owner": "jashandeep-sohi", + "repo": "octopilot", + "rev": "b17d9033a57525c63fc973e3d1108fefdec62210", + "type": "github" + }, + "original": { + "owner": "jashandeep-sohi", + "ref": "fix/commit-with-api", + "repo": "octopilot", + "type": "github" + } + }, + "poetry2nix": { + "inputs": { + "flake-utils": "flake-utils", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1692876271, + "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils_2", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1713775815, + "narHash": "sha256-Wu9cdYTnGQQwtT20QQMg7jzkANKQjwBD9iccfGKkfls=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "2ac4dcbf55ed43f3be0bae15e181f08a57af24a4", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv-k8s": "devenv-k8s", + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + } + }, + "skaffold": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "gomod2nix": "gomod2nix", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1711736775, + "narHash": "sha256-wRzy040KH+IfgK+g3Dj4Re4hb1S+G3xaxJ41TrsPEvE=", + "owner": "jashandeep-sohi", + "repo": "skaffold", + "rev": "e57813c8e738a0a347f8732221961584e025b6a5", + "type": "github" + }, + "original": { + "owner": "jashandeep-sohi", + "repo": "skaffold", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_4": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..826e954 --- /dev/null +++ b/flake.nix @@ -0,0 +1,67 @@ +{ + description = "Datalab"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + devenv-k8s.url = "github:LCOGT/devenv-k8s"; + + nixpkgs.follows = "devenv-k8s/nixpkgs"; + flake-parts.follows = "devenv-k8s/flake-parts"; + }; + + nixConfig = { + extra-substituters = [ + "https://devenv.cachix.org" + "https://lco-public.cachix.org" + ]; + + extra-trusted-public-keys = [ + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + "lco-public.cachix.org-1:zSmLK7CkAehZ7QzTLZKt+5Y26Lr0w885GUB4GlT1SCg=" + ]; + }; + + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + inputs.devenv-k8s.flakeModules.default + ]; + + systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ]; + + perSystem = { config, self', inputs', pkgs, system, ... }: { + # Per-system attributes can be defined here. The self' and inputs' + # module parameters provide easy access to attributes of the same + # system. + + # https://devenv.sh/basics/ + # Enter using `nix develop --impure` + config.devenv.shells.default = { + + # https://devenv.sh/packages/ + packages = [ + + ]; + + # https://devenv.sh/reference/options/#entershell + enterShell = '' + export KUBECONFIG="`pwd`/local-kubeconfig" + + echo "Setting KUBECONFIG=$KUBECONFIG" + echo + echo "This is done to sandbox Kuberenetes tools (kubectl, skaffold, etc) to the local K8s cluster for this project." + echo "If you would like to use a local K8s cluster across multiple projects, then set 'KUBECONFIG' to a common path" + echo "in both projects before running the command to create the local cluster." + ''; + }; + }; + + flake = { + # The usual flake attributes can be defined here, including system- + # agnostic ones like nixosModule and system-enumerating ones, although + # those are more easily expressed in perSystem. + + }; + }; +} diff --git a/k8s/base/deploy-server.yaml b/k8s/base/deploy-server.yaml new file mode 100644 index 0000000..48f72fd --- /dev/null +++ b/k8s/base/deploy-server.yaml @@ -0,0 +1,166 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/deployment-apps-v1.json + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server + labels: + app.kubernetes.io/component: server +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + template: + metadata: + labels: + app.kubernetes.io/component: server + spec: + securityContext: + fsGroup: 1000 + volumes: + - name: tmp + emptyDir: + sizeLimit: 128Mi + - name: static + emptyDir: + sizeLimit: 128Mi + initContainers: + - name: check-db-ready + image: postgres:14-alpine + command: + - sh + - -c + - | + until pg_isready --username=$DB_USER --dbname=$DB_NAME --host=$DB_HOST --port=$DB_PORT; + do echo waiting for database; + sleep 1; + done; + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + + resources: + requests: + cpu: 50m + memory: 16Mi + limits: + cpu: 50m + memory: 16Mi + + - name: django-migrate + image: datalab + command: + - python + - manage.py + - migrate + - --no-input + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + volumeMounts: + - name: tmp + mountPath: /tmp + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + + - name: django-collectstatic + image: datalab + command: + - python + - manage.py + - collectstatic + - --no-input + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + volumeMounts: + - name: tmp + mountPath: /tmp + - name: static + mountPath: /static + readOnly: false + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + + containers: + - name: default + image: datalab + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + command: + - gunicorn + - --bind=0.0.0.0:8080 + - --worker-class=gevent + - --workers=$(GUNICORN_WORKERS) + - --timeout=$(GUNICORN_TIMEOUT) + - --access-logfile=- + - --error-logfile=- + - datalab.wsgi + env: + - name: GUNICORN_WORKERS + value: "2" + - name: GUNICORN_TIMEOUT + value: "300" + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + ports: + - name: server + containerPort: 8080 + protocol: TCP + volumeMounts: + - name: tmp + mountPath: /tmp + readOnly: false + - name: static + mountPath: /static + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 3 + httpGet: + path: /admin/login/ + port: server + readinessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 3 + httpGet: + path: /admin/login/ + port: server diff --git a/k8s/base/deploy-static.yaml b/k8s/base/deploy-static.yaml new file mode 100644 index 0000000..50804d0 --- /dev/null +++ b/k8s/base/deploy-static.yaml @@ -0,0 +1,131 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/deployment-apps-v1.json + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static + labels: + app.kubernetes.io/component: static +spec: + selector: + matchLabels: + app.kubernetes.io/component: static + template: + metadata: + labels: + app.kubernetes.io/component: static + spec: + securityContext: + fsGroup: 1000 + + volumes: + - name: tmp + emptyDir: + sizeLimit: 32Mi + + - name: static + emptyDir: + sizeLimit: 128Mi + + initContainers: + - name: check-db-ready + image: postgres:14-alpine + command: + - sh + - -c + - | + until pg_isready --username=$DB_USER --dbname=$DB_NAME --host=$DB_HOST --port=$DB_PORT; + do echo waiting for database; + sleep 1; + done; + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + + resources: + requests: + cpu: 50m + memory: 16Mi + limits: + cpu: 50m + memory: 16Mi + + - name: django-collectstatic + image: datalab + command: + - python + - manage.py + - collectstatic + - --no-input + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + volumeMounts: + - name: tmp + mountPath: /tmp + + - name: static + mountPath: /public + + env: + - name: STATIC_ROOT + value: "/public/static/" + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + + containers: + - name: default + image: ghcr.io/static-web-server/static-web-server:2 + command: + - /static-web-server + - --root=/public + - --health + - --port=8000 + - --host=0.0.0.0 + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + + volumeMounts: + - name: tmp + mountPath: /tmp + + - name: static + mountPath: /public + + ports: + - name: static + containerPort: 8000 + protocol: TCP + + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false diff --git a/k8s/base/deploy-worker.yaml b/k8s/base/deploy-worker.yaml new file mode 100644 index 0000000..74a5252 --- /dev/null +++ b/k8s/base/deploy-worker.yaml @@ -0,0 +1,67 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/deployment-apps-v1.json + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker + labels: + app.kubernetes.io/component: worker +spec: + selector: + matchLabels: + app.kubernetes.io/component: worker + template: + metadata: + labels: + app.kubernetes.io/component: worker + spec: + securityContext: + fsGroup: 1000 + volumes: + - name: tmp + ephemeral: + volumeClaimTemplate: + metadata: + labels: + type: datalab-worker-tmp-volume + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + containers: + - name: default + image: datalab + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: true + command: + - "python" + - "manage.py" + - "rundramatiq" + - "--processes" + - "2" + - "--threads" + - "4" + envFrom: + - configMapRef: + name: env + optional: false + - secretRef: + name: env + optional: false + volumeMounts: + - name: tmp + mountPath: /tmp + readOnly: false + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 3 + failureThreshold: 3 + successThreshold: 1 + periodSeconds: 30 + exec: + command: + - /bin/true diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..65826f1 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ./deploy-server.yaml + - ./svc-server.yaml + - ./deploy-static.yaml + - ./svc-static.yaml + - ./deploy-worker.yaml + +labels: + - pairs: + app.kubernetes.io/name: datalab + includeSelectors: true + +secretGenerator: + - name: env + type: Opaque + +configMapGenerator: + - name: env diff --git a/k8s/base/svc-server.yaml b/k8s/base/svc-server.yaml new file mode 100644 index 0000000..19605d1 --- /dev/null +++ b/k8s/base/svc-server.yaml @@ -0,0 +1,16 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/service-v1.json + +apiVersion: v1 +kind: Service +metadata: + name: server + labels: + app.kubernetes.io/component: server +spec: + type: ClusterIP + selector: + app.kubernetes.io/component: server + ports: + - name: server + port: 8080 + targetPort: server diff --git a/k8s/base/svc-static.yaml b/k8s/base/svc-static.yaml new file mode 100644 index 0000000..ff217dd --- /dev/null +++ b/k8s/base/svc-static.yaml @@ -0,0 +1,16 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/service-v1.json + +apiVersion: v1 +kind: Service +metadata: + name: static + labels: + app.kubernetes.io/component: static +spec: + type: ClusterIP + selector: + app.kubernetes.io/component: static + ports: + - name: static + port: 80 + targetPort: static diff --git a/k8s/envs/local/kustomization.yaml b/k8s/envs/local/kustomization.yaml new file mode 100644 index 0000000..41ccaad --- /dev/null +++ b/k8s/envs/local/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base/ +- ./ns.yaml + +namespace: datalab + +secretGenerator: +- behavior: merge + envs: + - ./secrets.env + name: env + type: Opaque + +configMapGenerator: +- behavior: merge + envs: + - ./settings.env + name: env diff --git a/k8s/envs/local/ns.yaml b/k8s/envs/local/ns.yaml new file mode 100644 index 0000000..9e0b515 --- /dev/null +++ b/k8s/envs/local/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: datalab diff --git a/k8s/envs/local/secrets.env.changeme b/k8s/envs/local/secrets.env.changeme new file mode 100644 index 0000000..ccaa972 --- /dev/null +++ b/k8s/envs/local/secrets.env.changeme @@ -0,0 +1,8 @@ +DB_PASSWORD=changeme +SECRET_KEY=changeme +OAUTH_SERVER_KEY=changeme +OAUTH_CLIENT_ID=changeme +OAUTH_CLIENT_SECRET=changeme +ARCHIVE_API_TOKEN=changeme +AWS_ACCESS_KEY_ID=changeme +AWS_SECRET_ACCESS_KEY=changeme diff --git a/k8s/envs/local/settings.env b/k8s/envs/local/settings.env new file mode 100644 index 0000000..777896c --- /dev/null +++ b/k8s/envs/local/settings.env @@ -0,0 +1,22 @@ +DEBUG=True + +DB_ENGINE=django.db.backends.postgresql +DB_HOST=postgresql.datalab-web-deps.svc +DB_NAME=datalab +DB_USER=datalab + +DATALAB_OPERATION_BUCKET=datalab-operation-output-local-test +AWS_BUCKET=datalab-operation-output-local-test +AWS_DEFAULT_REGION=us-west-2 + +TEMP_FITS_DIR=/tmp/fits/ + +ARCHIVE_API=https://archive-api.lco.global +OAUTH_TOKEN_URL=https://observe.lco.global/o/token/ +OAUTH_PROFILE_URL=https://observe.lco.global/api/profile/ + +DRAMATIQ_BROKER_URL=redis://redis-master.datalab-web-deps.svc +DRAMATIQ_RESULT_BACKEND_URL=redis://redis-master.datalab-web-deps.svc + +CACHE_BACKEND=django.core.cache.backends.redis.RedisCache +CACHE_LOCATION=redis://redis-master.datalab-web-deps.svc diff --git a/local-cluster.yaml b/local-cluster.yaml new file mode 100644 index 0000000..9422cb8 --- /dev/null +++ b/local-cluster.yaml @@ -0,0 +1,6 @@ +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +name: kind-datalab +product: kind +registry: lco-local +kubernetesVersion: v1.23.17 diff --git a/local-registry.yaml b/local-registry.yaml new file mode 100644 index 0000000..f5de791 --- /dev/null +++ b/local-registry.yaml @@ -0,0 +1,4 @@ +apiVersion: ctlptl.dev/v1alpha1 +kind: Registry +name: lco-local +port: 12315 diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..e827cb7 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,64 @@ +apiVersion: skaffold/v4beta10 +kind: Config +metadata: + name: deps +deploy: + helm: + releases: + - name: redis + remoteChart: oci://registry-1.docker.io/bitnamicharts/redis + version: 19.1.0 + namespace: datalab-web-deps + createNamespace: true + setValues: + architecture: standalone + auth.enabled: false + master.persistence.enabled: false + + - name: postgresql + remoteChart: oci://registry-1.docker.io/bitnamicharts/postgresql + version: 12.12.10 + namespace: datalab-web-deps + createNamespace: true + setValues: + architecture: standalone + primary.persistence.size: 2Gi + auth: + database: datalab + username: datalab + password: changeme +--- + +apiVersion: skaffold/v4beta10 +kind: Config +metadata: + name: app +requires: + - configs: + - app-image +manifests: + kustomize: + paths: + - k8s/envs/local/ +deploy: + kubectl: {} + logs: + prefix: podAndContainer + +--- + +apiVersion: skaffold/v4beta10 +kind: Config +metadata: + name: app-image +build: + tagPolicy: + gitCommit: + variant: Tags + artifacts: + - image: datalab + custom: + buildCommand: skaffold-builder-buildx + dependencies: + dockerfile: + path: Dockerfile