From e3c9317824173d64f2c62d821f47ad3464a39986 Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Wed, 19 Aug 2020 15:45:14 +0200 Subject: [PATCH 01/83] Ownership for Lab Sessions. Added a new field for ownership to the Lab Session class. Listing Lab Sessions depending on owner and not on project. Removal of Lab Sessions is not possible unless an owner even when the project is shared between users. Restricted 'Grant access to project' so that only the owner of the project can share it. More info -> [JIRA-13](https://scaleoutsystems.atlassian.net/browse/STACKN-13) , [https://scaleoutsystems.atlassian.net/browse/STACKN-28) --- components/studio/labs/models.py | 1 + components/studio/labs/views.py | 7 +++---- components/studio/projects/templates/settings.html | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/components/studio/labs/models.py b/components/studio/labs/models.py index 01fe0e322..97dd2891b 100644 --- a/components/studio/labs/models.py +++ b/components/studio/labs/models.py @@ -54,6 +54,7 @@ class Session(models.Model): (ABORTED, 'Aborted'), ] project = models.ForeignKey('projects.Project', on_delete=models.CASCADE, related_name='session') + owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name='owner') session_key = models.CharField(max_length=512) session_secret = models.CharField(max_length=512) diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index 43547c89c..e398d4021 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -13,7 +13,7 @@ def index(request, user, project): template = 'labs/index.html' project = Project.objects.filter(Q(slug=project), Q(owner=request.user) | Q(authorized=request.user)).first() - sessions = Session.objects.filter(project=project) + sessions = Session.objects.filter(Q(project=project), Q(owner=request.user)) flavors = Flavor.objects.all() environments = Environment.objects.all() url = settings.DOMAIN @@ -88,9 +88,8 @@ def run(request, user, project): @login_required(login_url='/accounts/login') def delete(request, user, project, id): - template = 'labs/index.html' - project = Project.objects.filter(Q(slug=project), Q(owner=request.user) | Q(authorized=request.user)).first() - session = Session.objects.filter(id=id, project=project).first() + project = Project.objects.filter(Q(slug=project), Q(owner=user) | Q(authorized=user)).first() + session = Session.objects.filter(Q(id=id), Q(project=project), Q(owner=user)).first() if session: from .helpers import delete_session_resources diff --git a/components/studio/projects/templates/settings.html b/components/studio/projects/templates/settings.html index 48bb88efe..e5ceb4e65 100644 --- a/components/studio/projects/templates/settings.html +++ b/components/studio/projects/templates/settings.html @@ -149,6 +149,7 @@
Settings for sharing and collaboration
+ {% if project.owner == request.user %}
Grant access

For granting access to this project, select one or more users from the list below.

@@ -169,6 +170,7 @@
Grant access
+ {% endif %}
Project Settings
From b5d89d6b4ea4a20296faf9d9edbb82c37947442a Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Wed, 19 Aug 2020 17:53:30 +0200 Subject: [PATCH 02/83] Ownership for Lab Sessions. Added a new field for ownership to the Lab Session class. Listing Lab Sessions depending on owner and not on project. Removal of Lab Sessions is not possible unless an owner even when the project is shared between users. Restricted 'Grant access to project' so that only the owner of the project can share it. More info -> [JIRA-13](https://scaleoutsystems.atlassian.net/browse/STACKN-13) , [https://scaleoutsystems.atlassian.net/browse/STACKN-28) --- components/studio/labs/views.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index e398d4021..4771be4d1 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -13,7 +13,7 @@ def index(request, user, project): template = 'labs/index.html' project = Project.objects.filter(Q(slug=project), Q(owner=request.user) | Q(authorized=request.user)).first() - sessions = Session.objects.filter(Q(project=project), Q(owner=request.user)) + sessions = Session.objects.filter(Q(project=project), Q(lab_session_owner=request.user)) flavors = Flavor.objects.all() environments = Environment.objects.all() url = settings.DOMAIN @@ -70,7 +70,8 @@ def run(request, user, project): 'minio.access_key': decrypted_key, 'minio.secret_key': decrypted_secret, } - session = Session.objects.create_session(name=name, project=project, chart='lab', settings=prefs) + session = Session.objects.create_session(name=name, project=project, lab_session_owner=request.user, + chart='lab', settings=prefs) from .helpers import create_session_resources print("trying to create resources") @@ -88,8 +89,8 @@ def run(request, user, project): @login_required(login_url='/accounts/login') def delete(request, user, project, id): - project = Project.objects.filter(Q(slug=project), Q(owner=user) | Q(authorized=user)).first() - session = Session.objects.filter(Q(id=id), Q(project=project), Q(owner=user)).first() + project = Project.objects.filter(Q(slug=project), Q(owner=request.user) | Q(authorized=request.user)).first() + session = Session.objects.filter(Q(id=id), Q(project=project), Q(lab_session_owner=request.user)).first() if session: from .helpers import delete_session_resources From 7d34b209794a83c62c1131bf7bc4454aa6a955fd Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Wed, 19 Aug 2020 17:54:02 +0200 Subject: [PATCH 03/83] Ownership for Lab Sessions. Added a new field for ownership to the Lab Session class. Listing Lab Sessions depending on owner and not on project. Removal of Lab Sessions is not possible unless an owner even when the project is shared between users. Restricted 'Grant access to project' so that only the owner of the project can share it. More info -> [JIRA-13](https://scaleoutsystems.atlassian.net/browse/STACKN-13) , [https://scaleoutsystems.atlassian.net/browse/STACKN-28) --- components/studio/labs/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/studio/labs/models.py b/components/studio/labs/models.py index 97dd2891b..025dcb43e 100644 --- a/components/studio/labs/models.py +++ b/components/studio/labs/models.py @@ -15,7 +15,7 @@ def generate_passkey(self, length=20): password = ''.join(secrets.choice(alphabet) for i in range(length)) return password - def create_session(self, name, project, chart, settings, helm_repo=None): + def create_session(self, name, project, lab_session_owner, chart, settings, helm_repo=None): slug = slugify(name) key = self.generate_passkey() secret = self.generate_passkey(40) @@ -23,6 +23,7 @@ def create_session(self, name, project, chart, settings, helm_repo=None): status = 'CR' session = self.create(name=name, project=project, + lab_session_owner=lab_session_owner, status=status, slug=slug, session_key=key, @@ -54,7 +55,7 @@ class Session(models.Model): (ABORTED, 'Aborted'), ] project = models.ForeignKey('projects.Project', on_delete=models.CASCADE, related_name='session') - owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name='owner') + lab_session_owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name='lab_session_owner') session_key = models.CharField(max_length=512) session_secret = models.CharField(max_length=512) From 4506eedf7524a3189cd3c42af9c449eb2a87633e Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Fri, 21 Aug 2020 13:45:59 +0200 Subject: [PATCH 04/83] Bug fixes. [JIRA-107](https://scaleoutsystems.atlassian.net/browse/STACKN-107) Sorting lab sessions by date (when created) in descending order. [JIRA-113](https://scaleoutsystems.atlassian.net/browse/STACKN-113) Cleaning up the k8s job, used for allocating project resources, once a project has been removed. --- components/chart-controller/controller/controller.py | 6 ++++++ components/studio/labs/views.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index c987311ce..5840f6924 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -104,6 +104,12 @@ def delete(self, options): # args = 'helm --kubeconfig '+str(kubeconfig)+' delete {release}'.format(release=options['release']) #.split(' ') args = ['helm', '--kubeconfig', str(kubeconfig), 'delete', options['release']] status = subprocess.run(args, cwd=self.cwd) + + # Delete k8s job used to allocate project resources + k8s_job_cmd = "kubectl get jobs | awk '/" + options['release'] +"-/{print $1}' | xargs kubectl delete job" + k8s_job_status = subprocess.run(k8s_job_cmd, shell=True, cwd=self.cwd) + print(k8s_job_status) + return json.dumps({'helm': {'command': args, 'cwd': str(self.cwd), 'status': str(status)}}) def update(self, options, chart): diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index f48671661..d3ea39d71 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -18,7 +18,7 @@ def index(request, user, project): template = 'labs/index.html' project = Project.objects.filter(Q(slug=project), Q(owner=request.user) | Q(authorized=request.user)).first() - sessions = Session.objects.filter(Q(project=project), Q(lab_session_owner=request.user)) + sessions = Session.objects.filter(Q(project=project), Q(lab_session_owner=request.user)).order_by('-created_at') flavors = Flavor.objects.all() environments = Environment.objects.all() url = settings.DOMAIN From 0bff71f2688eed7e431fa8ae98a228475fecb13f Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Fri, 21 Aug 2020 16:35:44 +0200 Subject: [PATCH 05/83] Verify role in Keycloak. Also offload some of the work of setting up a project by submitting the Keycloak setup task to Celery. Fix in projects detail view (permission to view page) --- components/studio/modules/keycloak_lib.py | 100 +++++++++++++++--- components/studio/projects/helpers.py | 24 +++-- components/studio/projects/models.py | 4 +- .../studio/projects/templates/project.html | 6 ++ .../studio/projects/templates/settings.html | 8 -- components/studio/projects/views.py | 45 +++++--- 6 files changed, 143 insertions(+), 44 deletions(-) diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index 32e6e0998..b932ab83c 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -1,5 +1,10 @@ from django.conf import settings import requests as r +import logging +import jwt + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class KeycloakInit: admin_url: str @@ -10,6 +15,8 @@ def __init__(self, admin_url, realm, token): self.realm = realm self.token = token + + def keycloak_user_auth(username, password, client_id, admin_url, realm): token_url = '{}/realms/{}/protocol/openid-connect/token'.format(admin_url, realm) req = {'client_id': client_id, @@ -80,6 +87,45 @@ def keycloak_init(): print('Failed to init Keycloak auth') return False +def keycloak_get_detailed_user_info(request): + if not ('oidc_access_token' in request.session): + logger.warn('No access token in request session -- unable to authorize user.') + + access_token = request.session['oidc_access_token'] + + user_json = [] + discovery_url = settings.OIDC_OP_REALM_AUTH+'/'+settings.KC_REALM + res = r.get(discovery_url) + if res: + realm_info = res.json() + public_key = '-----BEGIN PUBLIC KEY-----\n'+realm_info['public_key']+'\n-----END PUBLIC KEY-----' + else: + print('Failed to discover realm settings: '+settings.KC_REALM) + return None + try: + user_json = jwt.decode(access_token, public_key, algorithms='RS256', audience='account') + except: + logger.info('Failed to authenticate user.') + return user_json + +def keycloak_verify_user_role(request, resource, role): + user_info = keycloak_get_detailed_user_info(request) + print(user_info) + if user_info: + try: + resource_info = user_info['resource_access'][resource] + except: + logger.info('User not authorized to access resource {}'.format(resource)) + return False + + resource_roles = resource_info['roles'] + print(resource_roles) + if role in resource_roles: + return True + + return False + + def keycloak_get_clients(kc, payload): get_clients_url = '{}/admin/realms/{}/clients'.format(kc.admin_url, kc.realm) res = r.get(get_clients_url, headers={'Authorization': 'bearer '+kc.token}, params=payload) @@ -111,12 +157,17 @@ def keycloak_create_client(kc, client_id, base_url, root_url=[], redirectUris=[] if not redirectUris: redirectUris = [base_url+'/*'] create_client_url = '{}/admin/realms/{}/clients'.format(kc.admin_url, kc.realm) + logger.debug("Create client endpoint: "+create_client_url) client_rep = {'clientId': client_id, 'baseUrl': base_url, 'rootUrl': root_url, 'redirectUris': redirectUris} + logger.debug("Client rep: ") + logger.debug(client_rep) res = r.post(create_client_url, json=client_rep, headers={'Authorization': 'bearer '+kc.token}) if res: + logger.debug('Created new client with clientId {}'.format(client_id)) + logger.debug('Status code returned: {}'.format(res.status_code)) return True else: print('Failed to create new client.') @@ -211,11 +262,10 @@ def keycloak_add_scope_to_client(kc, client_id, scope_id): print(res.text) return False -def keycloak_create_client_role(kc, client_nid, role_name): +def keycloak_create_client_role(kc, client_nid, role_name, session): client_role_url = '{}/admin/realms/{}/clients/{}/roles'.format(kc.admin_url, kc.realm, client_nid) role_rep = {'name': role_name} - res = r.post(client_role_url, json=role_rep, headers={'Authorization': 'bearer '+kc.token}) - print(res) + res = session.post(client_role_url, json=role_rep, headers={'Authorization': 'bearer '+kc.token, }) if res: return True else: @@ -238,7 +288,9 @@ def keycloak_get_user_id(kc, username): res = res[0] return res['id'] -def keycloak_get_client_role_id(kc, role_name, client_nid): +def keycloak_get_client_role_id(kc, role_name, client_nid, session=[]): + if not session: + session = r.session() client_role_url = '{}/admin/realms/{}/clients/{}/roles'.format(kc.admin_url, kc.realm, client_nid) res = r.get(client_role_url, headers={'Authorization': 'bearer '+kc.token}) if res: @@ -252,7 +304,9 @@ def keycloak_get_client_role_id(kc, role_name, client_nid): print(res.text) return False -def keycloak_add_user_to_client_role(kc, client_nid, username, role_name): +def keycloak_add_user_to_client_role(kc, client_nid, username, role_name, session=[]): + if not session: + session = r.session() user_id = keycloak_get_user_id(kc, username) add_user_to_role_url = '{}/admin/realms/{}/users/{}/role-mappings/clients/{}'.format(kc.admin_url, kc.realm, @@ -260,10 +314,10 @@ def keycloak_add_user_to_client_role(kc, client_nid, username, role_name): client_nid) # Get role id - role_id = keycloak_get_client_role_id(kc, role_name, client_nid) + role_id = keycloak_get_client_role_id(kc, role_name, client_nid, session) role_rep = [{'name': role_name, 'id': role_id}] - res = r.post(add_user_to_role_url, json=role_rep, headers={'Authorization': 'bearer '+kc.token}) + res = session.post(add_user_to_role_url, json=role_rep, headers={'Authorization': 'bearer '+kc.token}) if res: return True else: @@ -272,8 +326,16 @@ def keycloak_add_user_to_client_role(kc, client_nid, username, role_name): print(res.text) return False -def keycloak_setup_base_client(base_url, client_id, username): - print(type(username)) +def keycloak_add_role_to_user(clientId, username, role): + logger.info('Adding role {} to user {}. Client is {}.'.format(role, username, clientId)) + kc = keycloak_init() + # Get client id + clients = keycloak_get_clients(kc, {'clientId': clientId}) + client_nid = clients[0]['id'] + # Add role to user + keycloak_add_user_to_client_role(kc, client_nid, username, role) + +def keycloak_setup_base_client(base_url, client_id, username, roles=['default'], default_user_role=['default']): kc = keycloak_init() client_status = keycloak_create_client(kc, client_id, base_url) # Create client @@ -294,8 +356,20 @@ def keycloak_setup_base_client(base_url, client_id, username): # _________________ # Add the scope to the client keycloak_add_scope_to_client(kc, client_nid, scope_id) - # Create client role - keycloak_create_client_role(kc, client_nid, client_id+'-role') - # Add user to client role - keycloak_add_user_to_client_role(kc, client_nid, username, client_id+'-role') + # Create client roles + session = r.session() + for role in roles: + res = keycloak_create_client_role(kc, client_nid, role, session) + if res: + print(role) + + + # Give user default roles + for default_role in default_user_role: + res = keycloak_add_user_to_client_role(kc, client_nid, username, default_role, session) + if res: + print(default_role) + + session.close() + return client_id, client_secret \ No newline at end of file diff --git a/components/studio/projects/helpers.py b/components/studio/projects/helpers.py index 9ff235d8e..b1aff93d8 100644 --- a/components/studio/projects/helpers.py +++ b/components/studio/projects/helpers.py @@ -7,7 +7,7 @@ import yaml from .models import Environment from .jobs import load_definition, start_job - +from .tasks import create_keycloak_client_task import re import modules.keycloak_lib as keylib @@ -32,16 +32,26 @@ def create_settings_file(project, username, token): return yaml.dump(proj_settings) + + + def create_project_resources(project, username, repository=None): - create_environment_image(project, repository) - create_helm_resources(project, username, repository) - # Create Keycloak client for project with default project role. + # The creator of the project assumes all roles by default. + print('Creating Keycloak resources.') HOST = settings.DOMAIN - RELEASE_NAME = str(project.slug) - URL = 'https://{}/{}/{}'.format(HOST, username.username, RELEASE_NAME) - keylib.keycloak_setup_base_client(URL, RELEASE_NAME, username.username) + print('host: '+HOST) + RELEASE_NAME = str(project.slug) + print('release: '+RELEASE_NAME) + URL = 'https://{}/{}/{}'.format(HOST, username, RELEASE_NAME) + print(URL) + + create_keycloak_client_task.delay(project.slug, username.username, []) + + create_environment_image(project, repository) + create_helm_resources(project, username, repository) + def create_environment_image(project, repository=None): diff --git a/components/studio/projects/models.py b/components/studio/projects/models.py index f71fd9b49..e2216785e 100644 --- a/components/studio/projects/models.py +++ b/components/studio/projects/models.py @@ -54,9 +54,7 @@ def generate_passkey(self, length=20): def create_project(self, name, owner, description, repository): letters = string.ascii_lowercase - slug = name.replace(" ","-").replace("_","-") - from .helpers import urlify - slug = urlify(slug) + slug = slugify(name) slug_extension = ''.join(random.choice(letters) for i in range(3)) slug = '{}-{}'.format(slugify(slug), slug_extension) diff --git a/components/studio/projects/templates/project.html b/components/studio/projects/templates/project.html index b76a515a7..b24732239 100644 --- a/components/studio/projects/templates/project.html +++ b/components/studio/projects/templates/project.html @@ -1,8 +1,10 @@ + {% extends 'baseproject.html' %} {% load staticfiles %} {% block content %} +{% if is_authorized %}

{{ project.name }}

@@ -115,4 +117,8 @@

+{% endif %} {% endblock %} + + + diff --git a/components/studio/projects/templates/settings.html b/components/studio/projects/templates/settings.html index e5ceb4e65..0c1047c1a 100644 --- a/components/studio/projects/templates/settings.html +++ b/components/studio/projects/templates/settings.html @@ -172,14 +172,6 @@
Grant access
{% endif %} -
-
Project Settings
-
- You need a project settings file to use the CLI. -
- Download project.yaml -
-
Publish project on GitHub
diff --git a/components/studio/projects/views.py b/components/studio/projects/views.py index a4dd2876b..658b958f3 100644 --- a/components/studio/projects/views.py +++ b/components/studio/projects/views.py @@ -8,12 +8,16 @@ from django.conf import settings as sett import logging import markdown -from .forms import TransferProjectOwnershipForm, PublishProjectToGitHub #, GrantAccessForm +import time +from .forms import TransferProjectOwnershipForm, PublishProjectToGitHub, GrantAccessForm from django.db.models import Q from models.models import Model import requests as r import base64 from projects.helpers import get_minio_keys +import modules.keycloak_lib as kc +from multiprocessing import Process +from .tasks import create_keycloak_client_task logger = logging.getLogger(__name__) @@ -105,17 +109,25 @@ def grant_access_to_project(request, user, project_slug): project = Project.objects.filter(slug=project_slug).first() if request.method == 'POST': - print('temp test') - form = GrantAccessForm(request.POST) - if form.is_valid(): - selected_users = form.cleaned_data.get('selected_users') - project.authorized.set(selected_users) - project.save() + # form = GrantAccessForm(request.POST) + print(request.POST) + # if form.is_valid(): + selected_users = request.POST['selected_users'] #form.cleaned_data.get('selected_users') + project.authorized.set(selected_users) + project.save() + + if len(selected_users) == 1: + selected_users = list(selected_users) + + for selected_user in selected_users: + user_tmp = User.objects.get(pk=selected_user) + username_tmp = user_tmp.username + logger.info('Trying to add user {} to project.'.format(username_tmp)) + kc.keycloak_add_role_to_user(project.slug, username_tmp, 'member') return HttpResponseRedirect( reverse('projects:settings', kwargs={'user': user, 'project_slug': project.slug})) - @login_required(login_url='/accounts/login') def create(request): template = 'index_projects.html' @@ -125,18 +137,22 @@ def create(request): access = request.POST.get('access', 'org') description = request.POST.get('description', '') repository = request.POST.get('repository', '') - project = Project.objects.create_project(name=name, owner=request.user, description=description, + project = Project.objects.create_project(name=name, + owner=request.user, + description=description, repository=repository) success = True try: create_project_resources(project, request.user, repository=repository) + request.session['oidc_id_token_expiration'] = time.time()-100 + request.session.save() except ProjectCreationException as e: print("ERROR: could not create project resources") success = False - if success: - project.save() + if not success: + project.delete() next_page = request.POST.get('next', '/{}/{}'.format(request.user, project.slug)) @@ -147,15 +163,18 @@ def create(request): @login_required(login_url='/accounts/login') def details(request, user, project_slug): + + is_authorized = kc.keycloak_verify_user_role(request, project_slug, 'member') + template = 'project.html' url_domain = sett.DOMAIN project = None message = None - + username = request.user.username try: - owner = User.objects.filter(username=user).first() + owner = User.objects.filter(username=username).first() project = Project.objects.filter(Q(owner=owner) | Q(authorized=owner), Q(slug=project_slug)).first() except Exception as e: message = 'No project found' From 521b590d24e6910b7f743ae018f090aa88133da7 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Fri, 21 Aug 2020 17:20:54 +0200 Subject: [PATCH 06/83] Fix lab role access --- components/studio/api/views.py | 4 ++++ components/studio/labs/helpers.py | 2 +- components/studio/modules/keycloak_lib.py | 3 ++- components/studio/studio/KCRFbackend.py | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/components/studio/api/views.py b/components/studio/api/views.py index be86db25a..adf36dccd 100644 --- a/components/studio/api/views.py +++ b/components/studio/api/views.py @@ -11,6 +11,7 @@ from rest_framework import generics from deployments.helpers import build_definition from projects.helpers import create_project_resources +import modules.keycloak_lib as keylib from .serializers import Model, MLModelSerializer, Report, ReportSerializer, \ ReportGenerator, ReportGeneratorSerializer, Project, ProjectSerializer, \ @@ -37,7 +38,10 @@ def release(self, request): # Could we get the token here for authorization? # We should check that the authenticated user also has # the correct role in Keycloak. + project = Project.objects.get(id=request.data['project']) + is_authorized = keylib.keycloak_verify_user_role(request, project.slug, 'member') + print(is_authorized) current_user = self.request.user if current_user == project.owner: # project = model.project diff --git a/components/studio/labs/helpers.py b/components/studio/labs/helpers.py index c9fb55c2a..7a78c62fb 100644 --- a/components/studio/labs/helpers.py +++ b/components/studio/labs/helpers.py @@ -25,7 +25,7 @@ def create_session_resources(request, user, session, prefs, project): RELEASE_NAME = str(session.slug) URL = 'https://'+RELEASE_NAME+'.'+HOST - client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user) + client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) parameters = {'release': str(session.slug), 'chart': session.chart, diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index b932ab83c..821bedd2c 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -90,7 +90,8 @@ def keycloak_init(): def keycloak_get_detailed_user_info(request): if not ('oidc_access_token' in request.session): logger.warn('No access token in request session -- unable to authorize user.') - + return [] + access_token = request.session['oidc_access_token'] user_json = [] diff --git a/components/studio/studio/KCRFbackend.py b/components/studio/studio/KCRFbackend.py index 0d0c4112d..adf5a5acf 100644 --- a/components/studio/studio/KCRFbackend.py +++ b/components/studio/studio/KCRFbackend.py @@ -27,5 +27,6 @@ def authenticate(self, request): username = access_token_json['preferred_username'] user = User.objects.get(username=username) - + request.session['oidc_access_token'] = access_token + request.session.save() return (user, None) \ No newline at end of file From 60af53d0bb667dea3d3e2fd4a65e73eb58297e82 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Sun, 23 Aug 2020 12:19:01 +0200 Subject: [PATCH 07/83] Added projects tasks.py --- components/studio/projects/tasks.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 components/studio/projects/tasks.py diff --git a/components/studio/projects/tasks.py b/components/studio/projects/tasks.py new file mode 100644 index 000000000..8adb2f643 --- /dev/null +++ b/components/studio/projects/tasks.py @@ -0,0 +1,22 @@ +from celery import shared_task +# from .helpers import create_environment_image, create_helm_resources +import modules.keycloak_lib as keylib +from django.conf import settings + +@shared_task +def create_keycloak_client_task(project_slug, username, repository): + # create_environment_image(project, repository) + # create_helm_resources(project, username, repository) + # Create Keycloak client for project with default project role. + # The creator of the project assumes all roles by default. + print('Creating Keycloak resources.') + HOST = settings.DOMAIN + print('host: '+HOST) + RELEASE_NAME = str(project_slug) + print('release: '+RELEASE_NAME) + URL = 'https://{}/{}/{}'.format(HOST, username, RELEASE_NAME) + print(URL) + + keylib.keycloak_setup_base_client(URL, RELEASE_NAME, username, settings.PROJECT_ROLES, settings.PROJECT_ROLES) + + print('Done Keycloak.') \ No newline at end of file From d58d6f5ca49c66e1a8653dfe06bc3f32e6c72983 Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Mon, 24 Aug 2020 15:58:29 +0200 Subject: [PATCH 08/83] Removed obsolete code from projects/jobs.py . Reverted changes in chart-controller . Refactored code in deployments to prevent crashes from the changes . --- .../chart-controller/controller/controller.py | 5 - components/studio/deployments/helpers.py | 45 ++++- components/studio/projects/helpers.py | 11 -- components/studio/projects/jobs.py | 174 ------------------ 4 files changed, 39 insertions(+), 196 deletions(-) delete mode 100644 components/studio/projects/jobs.py diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index 5840f6924..82b53766b 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -105,11 +105,6 @@ def delete(self, options): args = ['helm', '--kubeconfig', str(kubeconfig), 'delete', options['release']] status = subprocess.run(args, cwd=self.cwd) - # Delete k8s job used to allocate project resources - k8s_job_cmd = "kubectl get jobs | awk '/" + options['release'] +"-/{print $1}' | xargs kubectl delete job" - k8s_job_status = subprocess.run(k8s_job_cmd, shell=True, cwd=self.cwd) - print(k8s_job_status) - return json.dumps({'helm': {'command': args, 'cwd': str(self.cwd), 'status': str(status)}}) def update(self, options, chart): diff --git a/components/studio/deployments/helpers.py b/components/studio/deployments/helpers.py index a5084cf98..9a920a8ce 100644 --- a/components/studio/deployments/helpers.py +++ b/components/studio/deployments/helpers.py @@ -1,10 +1,8 @@ from django.conf import settings -from django.utils.text import slugify -from .exceptions import ModelDeploymentException -from pprint import pprint from string import Template -import requests -from .models import DeploymentDefinition, DeploymentInstance +import os +from pathlib import Path + DEPLOY_DEFAULT_TEMPLATE = """apiVersion: openfaas.com/v1 kind: Function @@ -67,6 +65,42 @@ def get_instance_from_definition(instance): return ret + +def start_job(definition): + print("deploying build baseimage job!".format()) + + from kubernetes import client, config + + if settings.EXTERNAL_KUBECONF: + config.load_kube_config('cluster.conf') + else: + if 'TELEPRESENCE_ROOT' in os.environ: + from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, + SERVICE_TOKEN_FILENAME, + InClusterConfigLoader) + + token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') + ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') + cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') + ) / Path(SERVICE_CERT_FILENAME).relative_to('/') + + InClusterConfigLoader( + token_filename=token_filename, cert_filename=cert_filename + ).load_and_set() + else: + config.load_incluster_config() + + api = client.BatchV1Api() + + # create the resource + api.create_namespaced_job( + namespace=settings.NAMESPACE, + body=definition, + ) + + print("Resource created") + + def build_definition(instance): import uuid from projects.helpers import get_minio_keys @@ -86,7 +120,6 @@ def build_definition(instance): secret_key=decrypted_secret, s3endpoint='http://{}-minio:9000'.format(instance.project.slug)) import yaml - from projects.jobs import start_job job = yaml.safe_load(job) start_job(job) instance.image = image diff --git a/components/studio/projects/helpers.py b/components/studio/projects/helpers.py index 9ff235d8e..f88e0ec2d 100644 --- a/components/studio/projects/helpers.py +++ b/components/studio/projects/helpers.py @@ -3,10 +3,7 @@ from .exceptions import ProjectCreationException from django.conf import settings import requests as r -import os import yaml -from .models import Environment -from .jobs import load_definition, start_job import re @@ -33,7 +30,6 @@ def create_settings_file(project, username, token): return yaml.dump(proj_settings) def create_project_resources(project, username, repository=None): - create_environment_image(project, repository) create_helm_resources(project, username, repository) # Create Keycloak client for project with default project role. @@ -43,13 +39,6 @@ def create_project_resources(project, username, repository=None): keylib.keycloak_setup_base_client(URL, RELEASE_NAME, username.username) -def create_environment_image(project, repository=None): - - if project.environment: - definition = load_definition(project) - start_job(definition) - - def create_helm_resources(project, user, repository=None): from rest_framework.authtoken.models import Token token = Token.objects.get_or_create(user=user) diff --git a/components/studio/projects/jobs.py b/components/studio/projects/jobs.py deleted file mode 100644 index e45228e1f..000000000 --- a/components/studio/projects/jobs.py +++ /dev/null @@ -1,174 +0,0 @@ -from django.conf import settings -import os -from pathlib import Path - -job_template = '''apiVersion: batch/v1 -kind: Job -metadata: - name: {name}-{id} - namespace: {namespace} -spec: - template: - spec: - restartPolicy: OnFailure - containers: - - name: kaniko - image: gcr.io/kaniko-project/executor:latest - env: - - name: DOCKER_CONFIG - value: "/kaniko/.docker/" - args: - - "--dockerfile=Dockerfile" - - "--context=/workspace/" - - "--destination={image}" - volumeMounts: - - name: git-source - mountPath: /workspace - - name: dockerjson - mountPath: /kaniko/.docker/ - #subPath: config.json - - name: dockerfile - mountPath: /workspace/Dockerfile - subPath: Dockerfile - initContainers: - - name: git-sync - image: k8s.gcr.io/git-sync:v3.1.1 - args: - - "-repo=https://github.com/leanaiorg/examples.git" - - "-dest=work" - env: - - name: GIT_SYNC_ONE_TIME - value: "true" - volumeMounts: - - name: git-source - mountPath: /tmp/git - securityContext: - runAsUser: 65533 # git-sync user - volumes: - - name: git-source - emptyDir: {{}} - - name: dockerjson - secret: - secretName: regcred - items: - - key: .dockerconfigjson - path: config.json - - name: dockerfile - configMap: - name: {name}-dockerfile''' - - -def load_definition(project): - import yaml - definition = __replace_vars(project, definition=job_template) - __generate_and_apply_configmaps(project) - ret = yaml.safe_load(definition) - return ret - - -# TODO Refactor to also work for startup and teardown in addition to Dockerfile. -def __generate_and_apply_configmaps(project): - from kubernetes import client, config - from kubernetes.client.rest import ApiException - from pprint import pprint - - - - if settings.EXTERNAL_KUBECONF: - config.load_kube_config('cluster.conf') - else: - # adjust k8s service account paths if running inside telepresence - if 'TELEPRESENCE_ROOT' in os.environ: - from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, - SERVICE_TOKEN_FILENAME, - InClusterConfigLoader) - token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') - cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_CERT_FILENAME).relative_to('/') - - InClusterConfigLoader( - token_filename=token_filename, cert_filename=cert_filename - ).load_and_set() - else: - config.load_incluster_config() - - api = client.CoreV1Api() - - try: - api_response = api.delete_namespaced_config_map( - namespace=settings.NAMESPACE, - name="{}-dockerfile".format(project.name), - ) - pprint(api_response) - - except ApiException as e: - print("Exception when calling CoreV1Api->delete_namespaced_config_map: %s\n" % e) - - metadata = client.V1ObjectMeta( - name="{}-dockerfile".format(project.name), - namespace=settings.NAMESPACE, - ) - # Instantiate the configmap object - configmap = client.V1ConfigMap( - api_version="v1", - kind="ConfigMap", - # How do I modify here ? - data=dict(Dockerfile=str(project.environment.dockerfile)), - metadata=metadata - ) - try: - api_response = api.create_namespaced_config_map( - namespace=settings.NAMESPACE, - body=configmap, - pretty='pretty_example', - ) - pprint(api_response) - - except ApiException as e: - print("Exception when calling CoreV1Api->create_namespaced_config_map: %s\n" % e) - - -def __replace_vars(project, definition): - import uuid - # TODO fetch from settings. - url = 'registry.demo.scaleout.se' - tag = 'latest' - image = str(project.slug) - project.image = url + '/' + image + ':' + tag - new_def = str(definition).format(name=str(project.slug), image=project.image, namespace=settings.NAMESPACE, - id=str(uuid.uuid4())) - return new_def - - -def start_job(definition): - print("deploying build baseimage job!".format()) - - from kubernetes import client, config - - if settings.EXTERNAL_KUBECONF: - config.load_kube_config('cluster.conf') - else: - if 'TELEPRESENCE_ROOT' in os.environ: - from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, - SERVICE_TOKEN_FILENAME, - InClusterConfigLoader) - token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') - cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_CERT_FILENAME).relative_to('/') - - InClusterConfigLoader( - token_filename=token_filename, cert_filename=cert_filename - ).load_and_set() - else: - config.load_incluster_config() - - api = client.BatchV1Api() - - # create the resource - api.create_namespaced_job( - namespace=settings.NAMESPACE, - body=definition, - ) - print("Resource created") From 06cf0aec1d89e4d50757613ea6a3ee9a12622e9a Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Mon, 24 Aug 2020 21:53:20 +0200 Subject: [PATCH 09/83] Clean-up of ModelList in API, permissions via Keycloak client groups --- cli/scaleout/studioclient.py | 68 +++++++++++------------ components/studio/api/urls.py | 28 +++++++--- components/studio/api/views.py | 56 ++++++++----------- components/studio/modules/keycloak_lib.py | 14 +++-- components/studio/projects/views.py | 2 +- components/studio/requirements.txt | 1 + 6 files changed, 87 insertions(+), 82 deletions(-) diff --git a/cli/scaleout/studioclient.py b/cli/scaleout/studioclient.py index 331ad0639..38c3269a2 100644 --- a/cli/scaleout/studioclient.py +++ b/cli/scaleout/studioclient.py @@ -64,14 +64,14 @@ def __init__(self, config=None): self.project_slug = self.project['slug'] def get_endpoints(self): - endpoints = self.list_endpoints() - self.endpoints = endpoints - self.models_api = endpoints['models'] - self.reports_api = endpoints['reports'] - self.projects_api = endpoints['projects'] - self.generators_api = endpoints['generators'] - self.deployment_instance_api = endpoints['deploymentInstances'] - self.deployment_definition_api = endpoints['deploymentDefinitions'] + # endpoints = self.list_endpoints() + self.endpoints = dict() + self.endpoints['models'] = self.api_url+'/projects/{}/models' + self.reports_api = self.api_url+'/reports' + self.endpoints['projects'] = self.api_url+'/projects/' + self.generators_api = self.api_url+'/generators' #endpoints['generators'] + self.endpoints['deploymentInstances'] = self.api_url+'/deploymentInstances/' + self.deployment_definition_api = self.api_url+'/deploymentDefinitions' #endpoints['deploymentDefinitions'] def _get_repository_conf(self): """ Return the project minio keys. """ @@ -124,7 +124,7 @@ def list_endpoints(self): ### Projects api #### def create_project(self, name, description, repository): - url = self.projects_api+'create_project/' + url = self.endpoints['projects']+'create_project/' data = {'name': name, 'description': description, 'repository': repository} res = requests.post(url, headers=self.auth_headers, json=data) if res: @@ -139,7 +139,7 @@ def create_project(self, name, description, repository): def list_projects(self): """ List all projects a user has access to. """ - url = self.projects_api + url = self.endpoints['projects'] r = requests.get(url, headers=self.auth_headers) if (r.status_code < 200 or r.status_code > 299): print("List projects failed.") @@ -150,7 +150,7 @@ def list_projects(self): return json.loads(r.content) def get_projects(self, params=[]): - url = self.projects_api + url = self.endpoints['projects'] if params: r = requests.get(url, headers=self.auth_headers, params=params) else: @@ -258,7 +258,7 @@ def list_datasets(self): def show_model(self, model_id=None): """ Get all metadata associated with a model. """ try: - for model in self.list_models(): + for model in self.get_models(): if model['uid'] == model_id: return model except: @@ -280,11 +280,10 @@ def create_model(self, model_file, model_name, release_type='', model_descriptio model_data = {"uid": model_uid, "name": model_name, "release_type": release_type, - "description": model_description, - "project": str(self.project_id)} + "description": model_description} - url = self.models_api+'release/' - url = url.replace('http:', 'https:') + url = self.endpoints['models'].format(self.project['id'])+'/' + # url = url.replace('http:', 'https:') r = requests.post(url, json=model_data, headers=self.auth_headers) if not _check_status(r, error_msg="Failed to create model."): @@ -296,7 +295,7 @@ def create_model(self, model_file, model_name, release_type='', model_descriptio def deploy_model(self, model_name, model_version, deploy_context): - url = self.deployment_instance_api+'build_instance/' + url = self.endpoints['deploymentInstances']+'build_instance/' bd_data = {"project": self.project['id'], "name": model_name, "version": model_version, "depdef": deploy_context} r = requests.post(url, json=bd_data, headers=self.auth_headers) if not _check_status(r, error_msg="Failed to create deployment."): @@ -307,7 +306,7 @@ def deploy_model(self, model_name, model_version, deploy_context): return True def update_deployment(self, name, version, params): - url = self.deployment_instance_api+'update_instance/' + url = self.endpoints['deploymentInstances']+'update_instance/' params['name'] = name params['version'] = version r = requests.post(url, headers=self.auth_headers, json=params) @@ -327,7 +326,7 @@ def create_list(self, resource): return [] if resource == 'models': if self.found_project: - models = self.get_models({'project': self.project['id']}) + models = self.get_models(self.project['id']) return models else: return [] @@ -339,10 +338,17 @@ def create_list(self, resource): else: return json.loads(r.content) - def get_models(self, params): - r = requests.get(self.endpoints['models'], params=params, headers=self.auth_headers) + def get_models(self, project_id, params=[]): + url = self.endpoints['models'].format(project_id) + r = requests.get(url, headers=self.auth_headers, params=params) models = json.loads(r.content) return models + + def get_model(self, project_id, model_id): + url = '{}/{}'.format(self.endpoints['models'].format(project_id),model_id) + r = requests.get(url, headers=self.auth_headers) + model = json.loads(r.content) + return model def list_deployments(self): url = self.endpoints['deploymentInstances'] @@ -350,12 +356,10 @@ def list_deployments(self): if not _check_status(r, error_msg="Failed to fetch deployments"): return False deployments = json.loads(r.content) - # print(deployments) depjson = [] for deployment in deployments: - model = self.get_models({'id': deployment['model']})[0] - # print(model) - # print(deployment) + + model = self.get_model(self.project['id'], deployment['model']) dep = {'name': model['name'], 'version': model['version'], 'endpoint': deployment['endpoint'], @@ -363,12 +367,8 @@ def list_deployments(self): depjson.append(dep) return depjson - - - - def get_deployment(self, params): - url = self.deployment_instance_api + url = self.endpoints['deploymentInstances'] r = requests.get(url, params=params, headers=self.auth_headers) return json.loads(r.content) @@ -377,9 +377,9 @@ def delete_model(self, name, version=None): params = {'name': name, 'version': version} else: params = {'name': name} - models = self.get_models(params) + models = self.get_models(self.project['id'], params) for model in models: - url = os.path.join(self.models_api, '{}'.format(model['id'])) + url = '{}/{}'.format(self.endpoints['models'].format(self.project['id']), model['id']) r = requests.delete(url, headers=self.auth_headers) if not _check_status(r, error_msg="Failed to delete model {}:{}.".format(name, version)): pass @@ -391,12 +391,12 @@ def delete_deployment(self, name, version=None): params = {'name': name, 'version': version} else: params = {'name': name} - models = self.get_models(params) + models = self.get_models(self.project['id'], params) for model in models: deployment = self.get_deployment({'model':model['id'],'project':self.project['id']}) if deployment: deployment = deployment[0] - url = self.deployment_instance_api+'destroy' + url = self.endpoints['deploymentInstances']+'destroy' r = requests.delete(url, headers=self.auth_headers, params=params) if not _check_status(r, error_msg="Failed to delete deployment {}:{}.".format(model['name'], model['version'])): pass diff --git a/components/studio/api/urls.py b/components/studio/api/urls.py index 41b1eb14f..8167f75bd 100644 --- a/components/studio/api/urls.py +++ b/components/studio/api/urls.py @@ -1,20 +1,32 @@ from django.conf.urls import include from django.urls import path -from rest_framework import routers +import rest_framework.routers as drfrouters from .views import ModelList, ReportList, ReportGeneratorList, ProjectList, DeploymentInstanceList, DeploymentDefinitionList from rest_framework.authtoken.views import obtain_auth_token +from rest_framework_nested import routers app_name = 'api' -router = routers.DefaultRouter() -router.register(r'models', ModelList, basename='model') -router.register(r'reports', ReportList, basename='report') -router.register(r'generators', ReportGeneratorList, basename='report_generator') -router.register(r'projects', ProjectList, basename='project') -router.register(r'deploymentInstances', DeploymentInstanceList, basename='deploymentInstance') -router.register(r'deploymentDefinitions', DeploymentDefinitionList, basename='deploymentDefinition') +router_drf = drfrouters.DefaultRouter() +# router = routers.DefaultRouter() +router = routers.SimpleRouter() +router.register(r'reports', ReportList, base_name='report') +router.register(r'generators', ReportGeneratorList, base_name='report_generator') +router.register(r'projects', ProjectList, base_name='project') + +models_router = routers.NestedSimpleRouter(router, r'projects', lookup='project') +models_router.register(r'models', ModelList, base_name='model') +# router.register(r'models', ModelList, basename='model') +router.register(r'deploymentInstances', DeploymentInstanceList, base_name='deploymentInstance') +router.register(r'deploymentDefinitions', DeploymentDefinitionList, base_name='deploymentDefinition') +# print(router.urls) +print(models_router.urls) urlpatterns = [ + path('', include(router_drf.urls)), path('', include(router.urls)), + path('', include(models_router.urls)), path('api-token-auth', obtain_auth_token, name='api_token_auth'), ] + +print(urlpatterns) \ No newline at end of file diff --git a/components/studio/api/views.py b/components/studio/api/views.py index adf36dccd..f56a08fe7 100644 --- a/components/studio/api/views.py +++ b/components/studio/api/views.py @@ -6,9 +6,10 @@ from django.db.models import Q from rest_framework.mixins import CreateModelMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin from rest_framework.viewsets import GenericViewSet -from rest_framework.decorators import action +from rest_framework.decorators import action, api_view from rest_framework.permissions import IsAuthenticated from rest_framework import generics +from .APIpermissions import ProjectPermission from deployments.helpers import build_definition from projects.helpers import create_project_resources import modules.keycloak_lib as keylib @@ -19,54 +20,41 @@ DeploymentDefinitionSerializer class ModelList(GenericViewSet, CreateModelMixin, RetrieveModelMixin, UpdateModelMixin, ListModelMixin): - permission_classes = (IsAuthenticated,) + permission_classes = (IsAuthenticated, ProjectPermission,) serializer_class = MLModelSerializer filter_backends = [DjangoFilterBackend] - filterset_fields = ['id','name', 'version', 'project'] + filterset_fields = ['id','name', 'version'] def get_queryset(self): """ This view should return a list of all the models for the currently authenticated user. """ - current_user = self.request.user - return Model.objects.filter(project__owner__username=current_user) + return Model.objects.filter(project__pk=self.kwargs['project_pk']) + + def destroy(self, request, *args, **kwargs): + model = self.get_object() + model.delete() + return HttpResponse('ok', 200) + def create(self, request, *args, **kwargs): + project = Project.objects.get(id=self.kwargs['project_pk']) - @action(detail=False, methods=['post'], permission_classes=[IsAuthenticated]) - def release(self, request): - # Could we get the token here for authorization? - # We should check that the authenticated user also has - # the correct role in Keycloak. - - project = Project.objects.get(id=request.data['project']) - is_authorized = keylib.keycloak_verify_user_role(request, project.slug, 'member') - print(is_authorized) - current_user = self.request.user - if current_user == project.owner: - # project = model.project + try: model_name = request.data['name'] release_type = request.data['release_type'] description = request.data['description'] model_uid = request.data['uid'] - # project_id = request.data['project'] - new_model = Model(name=model_name, - release_type=release_type, - description=description, - uid=model_uid, - project=project) - new_model.save() - return HttpResponse('ok', 200) - + except: + return HttpResponse('Failed to create model.', 400) - def destroy(self, request, *args, **kwargs): - model = self.get_object() - current_user = self.request.user - if current_user == model.project.owner: - model.delete() - return HttpResponse('ok', 200) - else: - return HttpResponse('Not Allowed', 400) + new_model = Model(name=model_name, + release_type=release_type, + description=description, + uid=model_uid, + project=project) + new_model.save() + return HttpResponse('ok', 200) class DeploymentDefinitionList(GenericViewSet, CreateModelMixin, RetrieveModelMixin, UpdateModelMixin, ListModelMixin): permission_classes = (IsAuthenticated,) diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index 821bedd2c..a8b2e87f0 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -91,7 +91,7 @@ def keycloak_get_detailed_user_info(request): if not ('oidc_access_token' in request.session): logger.warn('No access token in request session -- unable to authorize user.') return [] - + access_token = request.session['oidc_access_token'] user_json = [] @@ -109,7 +109,11 @@ def keycloak_get_detailed_user_info(request): logger.info('Failed to authenticate user.') return user_json -def keycloak_verify_user_role(request, resource, role): +def keycloak_verify_user_role(request, resource, roles): + ''' + Checks if user has on of the roles in 'role' for resource given by 'resource' + Variable 'role' has to be iterable. + ''' user_info = keycloak_get_detailed_user_info(request) print(user_info) if user_info: @@ -120,9 +124,9 @@ def keycloak_verify_user_role(request, resource, role): return False resource_roles = resource_info['roles'] - print(resource_roles) - if role in resource_roles: - return True + for role in roles: + if role in resource_roles: + return True return False diff --git a/components/studio/projects/views.py b/components/studio/projects/views.py index 658b958f3..182708a6c 100644 --- a/components/studio/projects/views.py +++ b/components/studio/projects/views.py @@ -164,7 +164,7 @@ def create(request): @login_required(login_url='/accounts/login') def details(request, user, project_slug): - is_authorized = kc.keycloak_verify_user_role(request, project_slug, 'member') + is_authorized = kc.keycloak_verify_user_role(request, project_slug, ['member']) template = 'project.html' diff --git a/components/studio/requirements.txt b/components/studio/requirements.txt index acd671fbe..81746145d 100644 --- a/components/studio/requirements.txt +++ b/components/studio/requirements.txt @@ -34,3 +34,4 @@ django-yamlfield==1.0.3 Markdown==3.2.1 mozilla-django-oidc pyjwt +drf-nested-routers From cfaaf4af6f44ceab31cb04cfaf182c119b3d7f73 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Mon, 24 Aug 2020 21:53:54 +0200 Subject: [PATCH 10/83] Added API permission class --- components/studio/api/APIpermissions.py | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 components/studio/api/APIpermissions.py diff --git a/components/studio/api/APIpermissions.py b/components/studio/api/APIpermissions.py new file mode 100644 index 000000000..03ce9feba --- /dev/null +++ b/components/studio/api/APIpermissions.py @@ -0,0 +1,33 @@ +from rest_framework.permissions import BasePermission +from django.http import QueryDict +from .serializers import Model, MLModelSerializer, Report, ReportSerializer, \ + ReportGenerator, ReportGeneratorSerializer, Project, ProjectSerializer, \ + DeploymentInstance, DeploymentInstanceSerializer, DeploymentDefinition, \ + DeploymentDefinitionSerializer +import modules.keycloak_lib as keylib + + +class ProjectPermission(BasePermission): + + def has_permission(self, request, view): + """ + Should simply return, or raise a 403 response. + """ + + project = Project.objects.get(pk=view.kwargs['project_pk']) + print(request.method) + if request.method == 'GET': + is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['guest', 'member', 'admin']) + print('Is authorized: {}'.format(is_authorized)) + return is_authorized + if request.method in ['POST', 'PUT']: + is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['member', 'admin']) + print('Is authorized: {}'.format(is_authorized)) + return is_authorized + if request.method in ['DELETE']: + is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['admin']) + print('Is authorized: {}'.format(is_authorized)) + return is_authorized + + return False + From 98386c4a815de7607576ed57e9d8a64202b11848 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Mon, 24 Aug 2020 22:08:00 +0200 Subject: [PATCH 11/83] Clean-up APIpermission --- components/studio/api/APIpermissions.py | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/components/studio/api/APIpermissions.py b/components/studio/api/APIpermissions.py index 03ce9feba..355621b81 100644 --- a/components/studio/api/APIpermissions.py +++ b/components/studio/api/APIpermissions.py @@ -15,19 +15,19 @@ def has_permission(self, request, view): """ project = Project.objects.get(pk=view.kwargs['project_pk']) - print(request.method) - if request.method == 'GET': - is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['guest', 'member', 'admin']) - print('Is authorized: {}'.format(is_authorized)) - return is_authorized - if request.method in ['POST', 'PUT']: - is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['member', 'admin']) - print('Is authorized: {}'.format(is_authorized)) - return is_authorized - if request.method in ['DELETE']: - is_authorized = keylib.keycloak_verify_user_role(request, project.slug, ['admin']) - print('Is authorized: {}'.format(is_authorized)) - return is_authorized - - return False + + project_rules = { + 'GET': ['guest', 'member', 'admin'], + 'POST': ['member', 'admin'], + 'PUT': ['member', 'admin'], + 'DELETE': ['admin'] + } + + is_authorized = False + if request.method in project_rules: + is_authorized = keylib.keycloak_verify_user_role(request, project.slug, project_rules[request.method]) + + print('Is authorized: {}'.format(is_authorized)) + return is_authorized + From 7ede925e050d5d24e2e28e8a7277803380ba11e9 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 10:53:16 +0200 Subject: [PATCH 12/83] ... --- components/studio/api/APIpermissions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/studio/api/APIpermissions.py b/components/studio/api/APIpermissions.py index 355621b81..8964978cd 100644 --- a/components/studio/api/APIpermissions.py +++ b/components/studio/api/APIpermissions.py @@ -29,5 +29,3 @@ def has_permission(self, request, view): print('Is authorized: {}'.format(is_authorized)) return is_authorized - - From c92b0eaf08fd66e0a15e201e58b1fa3c1b37aa94 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 10:54:29 +0200 Subject: [PATCH 13/83] Started updating experiments model for cronjob --- components/studio/experiments/models.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/components/studio/experiments/models.py b/components/studio/experiments/models.py index 6502239bd..e8c411788 100644 --- a/components/studio/experiments/models.py +++ b/components/studio/experiments/models.py @@ -1,11 +1,29 @@ from django.db import models +from django.db.models.signals import pre_delete, pre_save +from deployments.models import HelmResource +from django.dispatch import receiver import uuid class Experiment(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + username = models.CharField(max_length=512) command = models.CharField(max_length=1024) environment = models.ForeignKey('projects.Environment', on_delete=models.CASCADE, related_name='job_environment') project = models.ForeignKey('projects.Project', on_delete=models.CASCADE, related_name='job_project') + helmchart = models.OneToOneField('deployments.HelmResource', on_delete=models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) uploaded_at = models.DateTimeField(auto_now=True) + +@receiver(pre_save, sender=Experiment, dispatch_uid='experiment_pre_save_signal') +def pre_save_experiments(sender, instance, using, **kwargs): + print('creating cronjob chart') + release_name = '{}-{}-{}'.format(instance.project.slug, 'cronjob', instance.id) + parameters = { + + } + helmchart = HelmResource(name=release_name, + namespace='Default', + chart='deploy', + params=parameters, + username=instance.username) \ No newline at end of file From 4bc28774c0c446a0652f8799d2473f27acafcf25 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 11:00:59 +0200 Subject: [PATCH 14/83] Fixes after review --- components/studio/projects/helpers.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/components/studio/projects/helpers.py b/components/studio/projects/helpers.py index b1aff93d8..6865b6173 100644 --- a/components/studio/projects/helpers.py +++ b/components/studio/projects/helpers.py @@ -38,16 +38,9 @@ def create_settings_file(project, username, token): def create_project_resources(project, username, repository=None): # Create Keycloak client for project with default project role. # The creator of the project assumes all roles by default. - print('Creating Keycloak resources.') - HOST = settings.DOMAIN - print('host: '+HOST) - RELEASE_NAME = str(project.slug) - print('release: '+RELEASE_NAME) - URL = 'https://{}/{}/{}'.format(HOST, username, RELEASE_NAME) - print(URL) create_keycloak_client_task.delay(project.slug, username.username, []) - + create_environment_image(project, repository) create_helm_resources(project, username, repository) From 4942101aeea9b9712634aba3e791e46952d804ff Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 16:24:00 +0200 Subject: [PATCH 15/83] Submit CronJobs under Experiments --- .../chart-controller/controller/controller.py | 2 +- components/studio/deployments/models.py | 3 + components/studio/experiments/forms.py | 2 +- components/studio/experiments/jobs.py | 88 +++++++++---------- components/studio/experiments/models.py | 22 ++++- .../templates/experiments/index.html | 15 +++- components/studio/experiments/urls.py | 1 + components/studio/experiments/views.py | 22 ++++- 8 files changed, 97 insertions(+), 58 deletions(-) diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index 82b53766b..15ec1ddaa 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -87,7 +87,7 @@ def deploy(self, options, action='install'): for key in options: args.append('--set') # args.append('{}={}'.format(key, options[key])) - args.append(key+"="+options[key]) + args.append(key+"="+options[key].replace(',', '\,')) print(args) status = subprocess.run(args, cwd=self.cwd) diff --git a/components/studio/deployments/models.py b/components/studio/deployments/models.py index 5e5e247d4..27aaa7f47 100644 --- a/components/studio/deployments/models.py +++ b/components/studio/deployments/models.py @@ -25,6 +25,7 @@ def pre_save_helmresource(sender, instance, using, **kwargs): if update: action = 'upgrade' url = settings.CHART_CONTROLLER_URL + '/'+action + print(instance.params) retval = requests.get(url, instance.params) if retval: print('Resource: '+instance.name) @@ -32,6 +33,8 @@ def pre_save_helmresource(sender, instance, using, **kwargs): instance.status = 'OK' else: print('Failed to deploy resource: '+instance.name) + print('Reason: {}'.format(retval.text)) + print('Status code: {}'.format(retval.status_code)) instance.status = 'Failed' @receiver(pre_delete, sender=HelmResource, dispatch_uid='helmresource_pre_delete_signal') diff --git a/components/studio/experiments/forms.py b/components/studio/experiments/forms.py index 4cc06c4de..19b3e8881 100644 --- a/components/studio/experiments/forms.py +++ b/components/studio/experiments/forms.py @@ -5,4 +5,4 @@ class ExperimentForm(forms.ModelForm): class Meta: model = Experiment - fields = ('command', 'environment', 'project') + fields = ('command', 'environment', 'schedule') diff --git a/components/studio/experiments/jobs.py b/components/studio/experiments/jobs.py index 8213876a2..c160a5b52 100644 --- a/components/studio/experiments/jobs.py +++ b/components/studio/experiments/jobs.py @@ -41,52 +41,52 @@ def get_instance_from_definition(instance): return ret -def run_job(instance): - print("deploying job with {}!".format(instance)) - - from kubernetes import client, config - - if settings.EXTERNAL_KUBECONF: - config.load_kube_config('cluster.conf') - else: - if 'TELEPRESENCE_ROOT' in os.environ: - from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, - SERVICE_TOKEN_FILENAME, - InClusterConfigLoader) - token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') - cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_CERT_FILENAME).relative_to('/') - - InClusterConfigLoader( - token_filename=token_filename, cert_filename=cert_filename - ).load_and_set() - else: - config.load_incluster_config() - - api = client.BatchV1Api() - - yaml_definition = get_instance_from_definition(instance) - - # create the resource - api.create_namespaced_job( - namespace=settings.NAMESPACE, - body=yaml_definition, - ) - print("Resource created") +# def run_job(instance): +# print("deploying job with {}!".format(instance)) + + # from kubernetes import client, config + + # if settings.EXTERNAL_KUBECONF: + # config.load_kube_config('cluster.conf') + # else: + # if 'TELEPRESENCE_ROOT' in os.environ: + # from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, + # SERVICE_TOKEN_FILENAME, + # InClusterConfigLoader) + # token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') + # ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') + # cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') + # ) / Path(SERVICE_CERT_FILENAME).relative_to('/') + + # InClusterConfigLoader( + # token_filename=token_filename, cert_filename=cert_filename + # ).load_and_set() + # else: + # config.load_incluster_config() + + # api = client.BatchV1Api() + + # yaml_definition = get_instance_from_definition(instance) + + # # create the resource + # api.create_namespaced_job( + # namespace=settings.NAMESPACE, + # body=yaml_definition, + # ) + # print("Resource created") - # get the resource and print out data - print("getting logs:") - resource = api.read_namespaced_job( - name=str(instance.id), - namespace=settings.NAMESPACE, - ) - print("got logs?") - # resource = api.list_namespaced_job( - # namespace="stack-fn", + # # get the resource and print out data + # print("getting logs:") + # resource = api.read_namespaced_job( + # name=str(instance.id), + # namespace=settings.NAMESPACE, # ) - print("Resources details:") - pprint(resource) + # print("got logs?") + # # resource = api.list_namespaced_job( + # # namespace="stack-fn", + # # ) + # print("Resources details:") + # pprint(resource) def delete_job(instance): diff --git a/components/studio/experiments/models.py b/components/studio/experiments/models.py index e8c411788..218bbf45c 100644 --- a/components/studio/experiments/models.py +++ b/components/studio/experiments/models.py @@ -11,6 +11,7 @@ class Experiment(models.Model): command = models.CharField(max_length=1024) environment = models.ForeignKey('projects.Environment', on_delete=models.CASCADE, related_name='job_environment') project = models.ForeignKey('projects.Project', on_delete=models.CASCADE, related_name='job_project') + schedule = models.CharField(max_length=128) helmchart = models.OneToOneField('deployments.HelmResource', on_delete=models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) uploaded_at = models.DateTimeField(auto_now=True) @@ -18,12 +19,25 @@ class Experiment(models.Model): @receiver(pre_save, sender=Experiment, dispatch_uid='experiment_pre_save_signal') def pre_save_experiments(sender, instance, using, **kwargs): print('creating cronjob chart') - release_name = '{}-{}-{}'.format(instance.project.slug, 'cronjob', instance.id) + job_id = uuid.uuid1().hex[0:5] + release_name = '{}-{}-{}'.format(instance.project.slug, 'cronjob', job_id) parameters = { - + "release": release_name, + "chart": "cronjob", + "project.slug": instance.project.slug, + "image": instance.environment.image, + "command": str(instance.command.split(' ')), + "cronjob.schedule": instance.schedule, + "cronjob.port": "8786", + "resources.limits.cpu": "500m", + "resources.limits.memory": "1Gi", + "resources.requests.cpu": "100m", + "resources.requests.memory": "256Gi" } helmchart = HelmResource(name=release_name, namespace='Default', - chart='deploy', + chart='cronjob', params=parameters, - username=instance.username) \ No newline at end of file + username=instance.username) + helmchart.save() + instance.helmchart = helmchart diff --git a/components/studio/experiments/templates/experiments/index.html b/components/studio/experiments/templates/experiments/index.html index 8dca369c2..4c336fce0 100644 --- a/components/studio/experiments/templates/experiments/index.html +++ b/components/studio/experiments/templates/experiments/index.html @@ -21,7 +21,7 @@

Experiments

placeholder="Enter command to run"> Enter a command to run
- +
+ @@ -41,9 +42,12 @@

Experiments

- # + Command + Schedule Environment + + Created Updated @@ -51,13 +55,16 @@

Experiments

{% for experiment in experiments %} - + {{ experiment.command }} + {{ experiment.schedule }} {{ experiment.environment }} + Settings + Delete {{ experiment.created_at }} {{ experiment.updated_at }} diff --git a/components/studio/experiments/urls.py b/components/studio/experiments/urls.py index bfe9daaf6..4d34d027e 100644 --- a/components/studio/experiments/urls.py +++ b/components/studio/experiments/urls.py @@ -7,5 +7,6 @@ path('', views.index, name='index'), path('run', views.run, name='run'), path('/details', views.details, name='details'), + path('/delete', views.delete, name='delete'), ] diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index 018f29a25..e446b5bf5 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -28,12 +28,19 @@ def run(request, user, project): deployment = None if request.method == "POST": + print(request.POST) form = ExperimentForm(request.POST) if form.is_valid(): print("valid form! Saving") - instance = form.save() - from .jobs import run_job - run_job(instance) + instance = Experiment() + instance.username = user + instance.schedule = form.cleaned_data['schedule'] + instance.command = form.cleaned_data['command'] + # environment = Environment.objects.get(pk=request.POST['environment']) + instance.environment = form.cleaned_data['environment'] + instance.project = project + instance.save() + return HttpResponseRedirect( reverse('experiments:index', kwargs={'user': request.user, 'project': str(project.slug)})) else: @@ -43,7 +50,6 @@ def run(request, user, project): return render(request, temp, locals()) - @login_required(login_url='/accounts/login') def details(request, user, project, id): temp = 'experiments/details.html' @@ -60,3 +66,11 @@ def details(request, user, project, id): reverse('experiments:index', kwargs={'user': request.user, 'project': str(project.slug)})) return render(request, temp, locals()) + +@login_required(login_url='/accounts/login') +def delete(request, user, project, id): + temp = 'experiments/index.html' + instance = Experiment.objects.get(id=id) + instance.helmchart.delete() + return HttpResponseRedirect( + reverse('experiments:index', kwargs={'user': user, 'project': project})) \ No newline at end of file From 5cdb1f61a004e3bfa0c4a96cfa7ac041c5d89a13 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 16:59:20 +0200 Subject: [PATCH 16/83] Allow both cronjobs and jobs --- components/studio/experiments/forms.py | 1 + components/studio/experiments/models.py | 4 ++++ components/studio/experiments/views.py | 6 ++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/studio/experiments/forms.py b/components/studio/experiments/forms.py index 19b3e8881..a9ac4f67b 100644 --- a/components/studio/experiments/forms.py +++ b/components/studio/experiments/forms.py @@ -3,6 +3,7 @@ class ExperimentForm(forms.ModelForm): + schedule = forms.CharField(max_length=128, required=False) class Meta: model = Experiment fields = ('command', 'environment', 'schedule') diff --git a/components/studio/experiments/models.py b/components/studio/experiments/models.py index 218bbf45c..3e20e5ee8 100644 --- a/components/studio/experiments/models.py +++ b/components/studio/experiments/models.py @@ -21,12 +21,16 @@ def pre_save_experiments(sender, instance, using, **kwargs): print('creating cronjob chart') job_id = uuid.uuid1().hex[0:5] release_name = '{}-{}-{}'.format(instance.project.slug, 'cronjob', job_id) + is_cron = 1 + if instance.schedule == "None": + is_cron = 0 parameters = { "release": release_name, "chart": "cronjob", "project.slug": instance.project.slug, "image": instance.environment.image, "command": str(instance.command.split(' ')), + "iscron": str(is_cron), "cronjob.schedule": instance.schedule, "cronjob.port": "8786", "resources.limits.cpu": "500m", diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index e446b5bf5..3c8657dfe 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -28,13 +28,15 @@ def run(request, user, project): deployment = None if request.method == "POST": - print(request.POST) form = ExperimentForm(request.POST) if form.is_valid(): print("valid form! Saving") instance = Experiment() instance.username = user - instance.schedule = form.cleaned_data['schedule'] + if not form.cleaned_data['schedule']: + instance.schedule = "None" + else: + instance.schedule = form.cleaned_data['schedule'] instance.command = form.cleaned_data['command'] # environment = Environment.objects.get(pk=request.POST['environment']) instance.environment = form.cleaned_data['environment'] From 1741f8e0084cfdcb153804471fbcbdfe55edb74c Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 25 Aug 2020 23:21:54 +0200 Subject: [PATCH 17/83] Various bug fixes --- cli/scaleout/studioclient.py | 1 + components/studio/api/urls.py | 9 +- components/studio/experiments/jobs.py | 147 ------------------ .../templates/experiments/details.html | 18 +-- .../templates/experiments/index.html | 8 +- components/studio/experiments/views.py | 25 ++- components/studio/modules/keycloak_lib.py | 8 +- 7 files changed, 42 insertions(+), 174 deletions(-) delete mode 100644 components/studio/experiments/jobs.py diff --git a/cli/scaleout/studioclient.py b/cli/scaleout/studioclient.py index 38c3269a2..531333f0e 100644 --- a/cli/scaleout/studioclient.py +++ b/cli/scaleout/studioclient.py @@ -332,6 +332,7 @@ def create_list(self, resource): return [] url = self.endpoints[resource] + r = requests.get(url, headers=self.auth_headers) if not _check_status(r, error_msg="Failed to list {}.".format(resource)): return False diff --git a/components/studio/api/urls.py b/components/studio/api/urls.py index 8167f75bd..664613bf6 100644 --- a/components/studio/api/urls.py +++ b/components/studio/api/urls.py @@ -8,7 +8,7 @@ app_name = 'api' router_drf = drfrouters.DefaultRouter() -# router = routers.DefaultRouter() + router = routers.SimpleRouter() router.register(r'reports', ReportList, base_name='report') @@ -17,16 +17,13 @@ models_router = routers.NestedSimpleRouter(router, r'projects', lookup='project') models_router.register(r'models', ModelList, base_name='model') -# router.register(r'models', ModelList, basename='model') + router.register(r'deploymentInstances', DeploymentInstanceList, base_name='deploymentInstance') router.register(r'deploymentDefinitions', DeploymentDefinitionList, base_name='deploymentDefinition') -# print(router.urls) -print(models_router.urls) + urlpatterns = [ path('', include(router_drf.urls)), path('', include(router.urls)), path('', include(models_router.urls)), path('api-token-auth', obtain_auth_token, name='api_token_auth'), ] - -print(urlpatterns) \ No newline at end of file diff --git a/components/studio/experiments/jobs.py b/components/studio/experiments/jobs.py deleted file mode 100644 index c160a5b52..000000000 --- a/components/studio/experiments/jobs.py +++ /dev/null @@ -1,147 +0,0 @@ -from django.conf import settings -import os -from pathlib import Path -from pprint import pprint - -example = """apiVersion: batch/v1 -kind: Job -metadata: - name: {id} - namespace: {namespace} -spec: - template: - spec: - containers: - - name: experiment - image: python - command: {command} - volumeMounts: - - name: jobstorage - mountPath: /home/app/ - restartPolicy: Never - volumes: - - name: jobstorage - persistentVolumeClaim: - claimName: {name}-project-files - backoffLimit: 4""" - - -# TODO make a generalized shared function between jobs, workflows and deployments. -def get_instance_from_definition(instance): - import yaml - - ret = example.format( - name=instance.project.slug, - id=instance.id, - command=str(instance.command.split(' ')), - namespace=settings.NAMESPACE - ) - ret = yaml.safe_load(ret) - - return ret - - -# def run_job(instance): -# print("deploying job with {}!".format(instance)) - - # from kubernetes import client, config - - # if settings.EXTERNAL_KUBECONF: - # config.load_kube_config('cluster.conf') - # else: - # if 'TELEPRESENCE_ROOT' in os.environ: - # from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, - # SERVICE_TOKEN_FILENAME, - # InClusterConfigLoader) - # token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - # ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') - # cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - # ) / Path(SERVICE_CERT_FILENAME).relative_to('/') - - # InClusterConfigLoader( - # token_filename=token_filename, cert_filename=cert_filename - # ).load_and_set() - # else: - # config.load_incluster_config() - - # api = client.BatchV1Api() - - # yaml_definition = get_instance_from_definition(instance) - - # # create the resource - # api.create_namespaced_job( - # namespace=settings.NAMESPACE, - # body=yaml_definition, - # ) - # print("Resource created") - - # # get the resource and print out data - # print("getting logs:") - # resource = api.read_namespaced_job( - # name=str(instance.id), - # namespace=settings.NAMESPACE, - # ) - # print("got logs?") - # # resource = api.list_namespaced_job( - # # namespace="stack-fn", - # # ) - # print("Resources details:") - # pprint(resource) - - -def delete_job(instance): - from kubernetes import client, config - - if settings.EXTERNAL_KUBECONF: - config.load_kube_config('cluster.conf') - else: - config.load_incluster_config() - - api = client.BatchV1Api() - - api.delete_namespaced_job( - name=str(instance.id), - namespace=settings.NAMESPACE, - body=client.V1DeleteOptions(), - ) - print("Resource deleted") - - -def get_logs(experiment): - from kubernetes import client, config - - if settings.EXTERNAL_KUBECONF: - config.load_kube_config('cluster.conf') - else: - if 'TELEPRESENCE_ROOT' in os.environ: - from kubernetes.config.incluster_config import (SERVICE_CERT_FILENAME, - SERVICE_TOKEN_FILENAME, - InClusterConfigLoader) - token_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_TOKEN_FILENAME).relative_to('/') - cert_filename = Path(os.getenv('TELEPRESENCE_ROOT', '/') - ) / Path(SERVICE_CERT_FILENAME).relative_to('/') - - InClusterConfigLoader( - token_filename=token_filename, cert_filename=cert_filename - ).load_and_set() - else: - config.load_incluster_config() - - api = client.BatchV1Api() - - ret = api.read_namespaced_job( - name=str(experiment.id), - namespace=settings.NAMESPACE, - ) - print("getting job name:") - job_name = ret.metadata.labels['job-name'] - - api = client.CoreV1Api() - ret = api.list_namespaced_pod(namespace=settings.NAMESPACE, label_selector='job-name={}'.format(job_name)) - - ret = api.read_namespaced_pod_log( - name=ret.items[0].metadata.name, - namespace=settings.NAMESPACE) - - return ret diff --git a/components/studio/experiments/templates/experiments/details.html b/components/studio/experiments/templates/experiments/details.html index f12fa86d9..0c69e3291 100644 --- a/components/studio/experiments/templates/experiments/details.html +++ b/components/studio/experiments/templates/experiments/details.html @@ -6,27 +6,21 @@

Experiment details

- -{{ experiment.environment }} -
- +
+ +
-
-
-
-
-
- + {% for line in logs %} + {{ line }}
+ {% endfor %}
-
-
{% endblock %} \ No newline at end of file diff --git a/components/studio/experiments/templates/experiments/index.html b/components/studio/experiments/templates/experiments/index.html index 4c336fce0..8592bc951 100644 --- a/components/studio/experiments/templates/experiments/index.html +++ b/components/studio/experiments/templates/experiments/index.html @@ -42,7 +42,7 @@

Experiments

- + Command Schedule Environment @@ -55,11 +55,11 @@

Experiments

{% for experiment in experiments %} - + {{ experiment.command }} {{ experiment.schedule }} {{ experiment.environment }} diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index 3c8657dfe..fcb559ef8 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -1,8 +1,10 @@ from django.contrib.auth.decorators import login_required from django.http import HttpResponseRedirect from django.shortcuts import render +from django.conf import settings from kubernetes.client.rest import ApiException - +import requests +import time from projects.models import Project from .models import Experiment from projects.models import Environment @@ -59,9 +61,21 @@ def details(request, user, project, id): project = Project.objects.filter(slug=project).first() experiment = Experiment.objects.filter(id=id).first() - from .jobs import get_logs try: - logs = get_logs(experiment) + url = settings.LOKI_SVC+'/loki/api/v1/query_range' + query = { + 'query': '{type="cronjob", project="demo-vqo", app="'+experiment.helmchart.name+'"}', + 'limit': 50, + 'start': 0, + } + res = requests.get(url, params=query) + res_json = res.json()['data']['result'] + logs = [] + for item in res_json: + logline = '' + for iline in item['values']: + logs.append(iline[1]) + logs.append('--------------------') except ApiException as e: print(e) return HttpResponseRedirect( @@ -69,6 +83,11 @@ def details(request, user, project, id): return render(request, temp, locals()) +# @login_required(login_url='/accounts/login') +# def settings(request, user, project, id): +# a=1 + + @login_required(login_url='/accounts/login') def delete(request, user, project, id): temp = 'experiments/index.html' diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index a8b2e87f0..0a56bb1de 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -145,7 +145,11 @@ def keycloak_get_clients(kc, payload): def keycloak_delete_client(kc, client_id): # Get id (not clientId) clients = keycloak_get_clients(kc, {'clientId': client_id}) - client_nid = clients[0]['id'] + try: + client_nid = clients[0]['id'] + except: + print('Cannot find client with clientId: {}'.format(client_id)) + return False # Delete client delete_client_url = '{}/admin/realms/{}/clients/{}'.format(kc.admin_url, kc.realm, client_nid) res = r.delete(delete_client_url, headers={'Authorization': 'bearer '+kc.token}) @@ -230,7 +234,7 @@ def keycloak_delete_client_scope(kc, scope_id): if res: return True else: - print('Failed to delete client scope '+scope_id) + print('Failed to delete client scope '.format(scope_id)) print('Status code: '+str(res.status_code)) print(res.text) return False From ba65f2c23b9789b840e43c88cea5cf4f2a6fdd84 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Wed, 26 Aug 2020 22:41:13 +0200 Subject: [PATCH 18/83] Bug fixes --- components/chart-controller/controller/controller.py | 2 +- components/studio/deployments/models.py | 3 ++- components/studio/experiments/models.py | 2 ++ components/studio/labs/views.py | 9 +++------ 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index 15ec1ddaa..babbb94b8 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -16,7 +16,7 @@ def refresh_charts(branch='master'): status = subprocess.run('rm -rf charts-{}'.format(branch).split(' '), cwd=cwd) status = subprocess.run('wget -O {}.zip {}'.format(branch, charts_url).split(' '), cwd=cwd) - status = subprocess.run('unzip {}.zip'.format(branch).split(' '),cwd=cwd) + status = subprocess.run('unzip {}.zip'.format(branch.replace('/', '-')).split(' '),cwd=cwd) class Controller: diff --git a/components/studio/deployments/models.py b/components/studio/deployments/models.py index 27aaa7f47..4654d12e3 100644 --- a/components/studio/deployments/models.py +++ b/components/studio/deployments/models.py @@ -170,10 +170,11 @@ def pre_save_deployment(sender, instance, using, **kwargs): instance.appname =instance.model.project.slug+'-'+slugify(instance.model.name)+'-'+slugify(instance.model.version) # Create Keycloak client corresponding to this deployment - client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, instance.created_by.username) + client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, instance.created_by.username, ['owner'], ['owner']) parameters = {'release': RELEASE_NAME, 'chart': 'deploy', + 'namespace': settings.NAMESPACE, 'appname': instance.appname, 'replicas': '1', 'global.domain': global_domain, diff --git a/components/studio/experiments/models.py b/components/studio/experiments/models.py index 3e20e5ee8..17ccf9f98 100644 --- a/components/studio/experiments/models.py +++ b/components/studio/experiments/models.py @@ -1,5 +1,6 @@ from django.db import models from django.db.models.signals import pre_delete, pre_save +from django.conf import settings from deployments.models import HelmResource from django.dispatch import receiver import uuid @@ -27,6 +28,7 @@ def pre_save_experiments(sender, instance, using, **kwargs): parameters = { "release": release_name, "chart": "cronjob", + "namespace": settings.NAMESPACE, "project.slug": instance.project.slug, "image": instance.environment.image, "command": str(instance.command.split(' ')), diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index d3ea39d71..cc628a649 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -66,18 +66,15 @@ def run(request, user, project): settings_file = JSONRenderer().render(settings_file.data) settings_file = settings_file.decode('utf-8') - print(settings_file) - # settings_file = yaml.load(settings_file, Loader=yaml.FullLoader) + settings_file = json.loads(settings_file) settings_file = yaml.dump(settings_file) - print(settings_file) - # settings_file = json.dumps(json.loads(settings_file)) - # settings_file = yaml.dump(settings_file) user_config_file = create_user_settings(user) user_config_file = yaml.dump(json.loads(user_config_file)) - prefs = {'labs.resources.requests.cpu': str(flavor.cpu), + prefs = {'namespace': settings.NAMESPACE, + 'labs.resources.requests.cpu': str(flavor.cpu), 'labs.resources.limits.cpu': str(flavor.cpu), 'labs.resources.requests.memory': str(flavor.mem), 'labs.resources.limits.memory': str(flavor.mem), From 8277a80e6be29ad1ef5c20727cadd67820c602c0 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Thu, 27 Aug 2020 11:04:07 +0200 Subject: [PATCH 19/83] Added permissioning to experiments --- .../chart-controller/controller/controller.py | 6 +++-- .../studio/experiments/experimentsauth.py | 20 ++++++++++++++ .../templates/experiments/index.html | 8 ++++++ components/studio/experiments/views.py | 25 +++++++++++++---- components/studio/modules/keycloak_lib.py | 27 ++++++++++++++----- 5 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 components/studio/experiments/experimentsauth.py diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index babbb94b8..9f637654d 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -15,7 +15,7 @@ def refresh_charts(branch='master'): charts_url = 'https://github.com/scaleoutsystems/charts/archive/{}.zip'.format(branch) status = subprocess.run('rm -rf charts-{}'.format(branch).split(' '), cwd=cwd) - status = subprocess.run('wget -O {}.zip {}'.format(branch, charts_url).split(' '), cwd=cwd) + status = subprocess.run('wget -O {}.zip {}'.format(branch.replace('/', '-'), charts_url).split(' '), cwd=cwd) status = subprocess.run('unzip {}.zip'.format(branch.replace('/', '-')).split(' '),cwd=cwd) @@ -75,7 +75,9 @@ def deploy(self, options, action='install'): chart = 'charts/scaleout/'+options['chart'] else: refresh_charts(self.branch) - chart = 'charts-{}/scaleout/{}'.format(self.branch, options['chart']) + fname = self.branch.replace('/', '-') + print(fname) + chart = 'charts-{}/scaleout/{}'.format(fname, options['chart']) args = ['helm', action, '--kubeconfig', kubeconfig, options['release'], chart] # tmp_file_name = uuid.uuid1().hex+'.yaml' diff --git a/components/studio/experiments/experimentsauth.py b/components/studio/experiments/experimentsauth.py new file mode 100644 index 000000000..24cdeb384 --- /dev/null +++ b/components/studio/experiments/experimentsauth.py @@ -0,0 +1,20 @@ +import modules.keycloak_lib as keylib + +def get_permissions(request, project): + + rules = { + 'view': ['guest', 'member', 'admin'], + 'create': ['member', 'admin'], + 'update': ['member', 'admin'], + 'delete': ['admin'] + } + + from_key = keylib.keycloak_get_user_roles(request, project, aud=project) + print(from_key) + user_roles = set(keylib.keycloak_get_user_roles(request, project,aud=project)) + + user_permissions = dict() + for rule in rules: + user_permissions[rule] = bool(set(rules[rule]) & user_roles) + + return user_permissions \ No newline at end of file diff --git a/components/studio/experiments/templates/experiments/index.html b/components/studio/experiments/templates/experiments/index.html index 8592bc951..043db1d77 100644 --- a/components/studio/experiments/templates/experiments/index.html +++ b/components/studio/experiments/templates/experiments/index.html @@ -63,8 +63,16 @@

Experiments

{{ experiment.command }} {{ experiment.schedule }} {{ experiment.environment }} + {% if user_permissions.update %} Settings + {% else %} + + {% endif %} + {% if user_permissions.delete %} Delete + {% else %} + + {% endif %} {{ experiment.created_at }} {{ experiment.updated_at }} diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index fcb559ef8..3b88c2ad6 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -1,5 +1,5 @@ from django.contrib.auth.decorators import login_required -from django.http import HttpResponseRedirect +from django.http import HttpResponseRedirect, HttpResponse from django.shortcuts import render from django.conf import settings from kubernetes.client.rest import ApiException @@ -10,10 +10,16 @@ from projects.models import Environment from .forms import ExperimentForm from django.urls import reverse +import modules.keycloak_lib as keylib +from .experimentsauth import get_permissions @login_required(login_url='/accounts/login') def index(request, user, project): + user_permissions = get_permissions(request, project) + if not user_permissions['view']: + return HttpResponse('Not authorized', status=401) + temp = 'experiments/index.html' project = Project.objects.filter(slug=project).first() @@ -22,9 +28,15 @@ def index(request, user, project): return render(request, temp, locals()) + + @login_required(login_url='/accounts/login') def run(request, user, project): + user_permissions = get_permissions(request, project) + if not user_permissions['create']: + return HttpResponse('Not authorized', status=401) + temp = 'experiments/run.html' project = Project.objects.filter(slug=project).first() @@ -56,6 +68,10 @@ def run(request, user, project): @login_required(login_url='/accounts/login') def details(request, user, project, id): + user_permissions = get_permissions(request, project) + if not user_permissions['view']: + return HttpResponse('Not authorized', status=401) + temp = 'experiments/details.html' project = Project.objects.filter(slug=project).first() @@ -83,13 +99,12 @@ def details(request, user, project, id): return render(request, temp, locals()) -# @login_required(login_url='/accounts/login') -# def settings(request, user, project, id): -# a=1 - @login_required(login_url='/accounts/login') def delete(request, user, project, id): + user_permissions = get_permissions(request, project) + if not user_permissions['delete']: + return HttpResponse('Not authorized', status=401) temp = 'experiments/index.html' instance = Experiment.objects.get(id=id) instance.helmchart.delete() diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index 0a56bb1de..84e6b4db6 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -87,13 +87,12 @@ def keycloak_init(): print('Failed to init Keycloak auth') return False -def keycloak_get_detailed_user_info(request): +def keycloak_get_detailed_user_info(request, aud='account'): if not ('oidc_access_token' in request.session): logger.warn('No access token in request session -- unable to authorize user.') return [] access_token = request.session['oidc_access_token'] - user_json = [] discovery_url = settings.OIDC_OP_REALM_AUTH+'/'+settings.KC_REALM res = r.get(discovery_url) @@ -104,18 +103,34 @@ def keycloak_get_detailed_user_info(request): print('Failed to discover realm settings: '+settings.KC_REALM) return None try: - user_json = jwt.decode(access_token, public_key, algorithms='RS256', audience='account') + user_json = jwt.decode(access_token, public_key, algorithms='RS256', audience=aud) except: logger.info('Failed to authenticate user.') return user_json -def keycloak_verify_user_role(request, resource, roles): +def keycloak_get_user_roles(request, resource, aud='account'): + ''' + Checks if user has on of the roles in 'role' for resource given by 'resource' + Variable 'role' has to be iterable. + ''' + user_info = keycloak_get_detailed_user_info(request, aud) + if user_info: + try: + resource_info = user_info['resource_access'][resource] + except: + logger.info('User not authorized to access resource {}'.format(resource)) + return False + + return resource_info['roles'] + + return [] + +def keycloak_verify_user_role(request, resource, roles, aud='account'): ''' Checks if user has on of the roles in 'role' for resource given by 'resource' Variable 'role' has to be iterable. ''' - user_info = keycloak_get_detailed_user_info(request) - print(user_info) + user_info = keycloak_get_detailed_user_info(request, aud) if user_info: try: resource_info = user_info['resource_access'][resource] From 548f380215104ce082be0b67f36aa67f73473ac7 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Thu, 27 Aug 2020 11:07:21 +0200 Subject: [PATCH 20/83] deleted debug prints etc --- components/chart-controller/controller/controller.py | 1 - components/studio/experiments/experimentsauth.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/components/chart-controller/controller/controller.py b/components/chart-controller/controller/controller.py index 9f637654d..231c4bf5a 100644 --- a/components/chart-controller/controller/controller.py +++ b/components/chart-controller/controller/controller.py @@ -76,7 +76,6 @@ def deploy(self, options, action='install'): else: refresh_charts(self.branch) fname = self.branch.replace('/', '-') - print(fname) chart = 'charts-{}/scaleout/{}'.format(fname, options['chart']) args = ['helm', action, '--kubeconfig', kubeconfig, options['release'], chart] diff --git a/components/studio/experiments/experimentsauth.py b/components/studio/experiments/experimentsauth.py index 24cdeb384..9fc99aec6 100644 --- a/components/studio/experiments/experimentsauth.py +++ b/components/studio/experiments/experimentsauth.py @@ -9,8 +9,6 @@ def get_permissions(request, project): 'delete': ['admin'] } - from_key = keylib.keycloak_get_user_roles(request, project, aud=project) - print(from_key) user_roles = set(keylib.keycloak_get_user_roles(request, project,aud=project)) user_permissions = dict() From 303abe7c30c7b309e069e8fe0213e6620ec0bcc9 Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Thu, 27 Aug 2020 19:06:00 +0200 Subject: [PATCH 21/83] Project Activity Added ProjectLog model class. Added logs within STACKn modules. Refactored UI - fixed select item, introduced time period for the logs, render the demanded logs only. [JIRA](https://scaleoutsystems.atlassian.net/browse/STACKN-127) --- components/studio/labs/views.py | 12 +++- components/studio/models/views.py | 21 +++++- components/studio/projects/admin.py | 5 +- components/studio/projects/models.py | 18 +++++ .../studio/projects/templates/project.html | 61 +++++++++-------- .../projects/templates/project_activity.html | 15 +++++ components/studio/projects/urls.py | 1 + components/studio/projects/views.py | 66 +++++++++++++++---- components/studio/reports/helpers.py | 7 +- components/studio/reports/views.py | 16 ++++- components/studio/templates/baseproject.html | 2 + 11 files changed, 178 insertions(+), 46 deletions(-) create mode 100644 components/studio/projects/templates/project_activity.html diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index d3ea39d71..bd9a34a70 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -1,5 +1,5 @@ from django.shortcuts import render, HttpResponseRedirect, reverse -from projects.models import Project +from projects.models import Project, ProjectLog from .models import Session from projects.models import Environment, Flavor from django.contrib.auth.decorators import login_required @@ -103,6 +103,11 @@ def run(request, user, project): print("saving session!") project.save() session.save() + + l = ProjectLog(project=project, module='LA', headline='Lab Session', + description='A new Lab Session {name} has been created'.format(name=name)) + l.save() + return HttpResponseRedirect( reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) @@ -118,6 +123,11 @@ def delete(request, user, project, id): if session: from .helpers import delete_session_resources delete_session_resources(session) + + l = ProjectLog(project=project, module='LA', headline='Lab Session', + description='Lab Session {name} has been removed'.format(name=session.name)) + l.save() + session.delete() return HttpResponseRedirect( diff --git a/components/studio/models/views.py b/components/studio/models/views.py index 9493def98..e1a88ec59 100644 --- a/components/studio/models/views.py +++ b/components/studio/models/views.py @@ -2,7 +2,7 @@ from django.shortcuts import render from django.http import HttpResponseRedirect from django.urls import reverse -from projects.models import Project +from projects.models import Project, ProjectLog from reports.models import Report, ReportGenerator from .models import Model from .forms import ModelForm @@ -49,6 +49,10 @@ def create(request, user, project): if form.is_valid(): obj = form.save() + l = ProjectLog(project=project, module='MO', headline='Model', + description='A new Model {name} has been added'.format(name=obj.name)) + l.save() + url = '/{}/{}/models/{}'.format(user, project.slug, obj.pk) else: url = '/{}/{}/models/'.format(user, project.slug) @@ -63,6 +67,7 @@ def create(request, user, project): @login_required(login_url='/accounts/login') def change_access(request, user, project, id): model = Model.objects.filter(pk=id).first() + previous = model.get_access_display() if request.method == 'POST': visibility = request.POST.get('access', '') @@ -70,6 +75,11 @@ def change_access(request, user, project, id): model.access = visibility model.save() + l = ProjectLog(project=project, module='MO', headline='Model - {name}'.format(name=model.name), + description='Changed Access Level from {previous} to {current}'.format(previous=previous, + current=model.get_access_display())) + l.save() + return HttpResponseRedirect( reverse('models:details', kwargs={'user': user, 'project': project, 'id': id})) @@ -118,6 +128,10 @@ def details(request, user, project, id): new_report = Report(model=model, report="", job_id=instance['id'], generator=generator_object, status='P') new_report.save() + l = ProjectLog(project=project, module='MO', headline='Model - {name}'.format(name=model.name), + description='Newly generated Metrics #{id}'.format(id=new_report.pk)) + l.save() + from reports.jobs import run_job run_job(instance) @@ -186,7 +200,12 @@ def delete(request, user, project, id): model = Model.objects.get(id=id) if request.method == "POST": + l = ProjectLog(project=project, module='MO', headline='Model', + description='Model {name} has been removed'.format(name=model.name)) + l.save() + model.delete() + return HttpResponseRedirect(reverse('models:list', kwargs={'user':user, 'project':project.slug})) return render(request, template, locals()) diff --git a/components/studio/projects/admin.py b/components/studio/projects/admin.py index afd671700..5f055ad0d 100644 --- a/components/studio/projects/admin.py +++ b/components/studio/projects/admin.py @@ -1,8 +1,9 @@ from django.contrib import admin -from .models import Project, Environment, Flavor +from .models import Project, Environment, Flavor, ProjectLog admin.site.register(Project) admin.site.register(Environment) -admin.site.register(Flavor) \ No newline at end of file +admin.site.register(Flavor) +admin.site.register(ProjectLog) diff --git a/components/studio/projects/models.py b/components/studio/projects/models.py index e2216785e..f3972537c 100644 --- a/components/studio/projects/models.py +++ b/components/studio/projects/models.py @@ -93,3 +93,21 @@ def __str__(self): environment = models.ForeignKey('projects.Environment', on_delete=models.DO_NOTHING, default=DEFAULT_ENVIRONMENT_ID) clone_url = models.CharField(max_length=512, null=True, blank=True) + +class ProjectLog(models.Model): + project = models.ForeignKey(Project, on_delete=models.CASCADE) + + MODULE_CHOICES = [ + ('DE', 'deployments'), + ('LA', 'labs'), + ('MO', 'models'), + ('PR', 'projects'), + ('RE', 'reports'), + ('UN', 'undefined'), + ] + module = models.CharField(max_length=2, choices=MODULE_CHOICES, default='UN') + + headline = models.CharField(max_length=256) + description = models.CharField(max_length=512) + created_at = models.DateTimeField(auto_now_add=True) + diff --git a/components/studio/projects/templates/project.html b/components/studio/projects/templates/project.html index b24732239..6a3294af7 100644 --- a/components/studio/projects/templates/project.html +++ b/components/studio/projects/templates/project.html @@ -87,38 +87,45 @@

Import code from another repository

{% endif %}
-

- Project activity - -

-
-
-
- -
+
+ {% csrf_token %} + +

+ + Project activity + + +

+
-
+
{% endif %} {% endblock %} +{% block extrascripts %} + +{% endblock %} diff --git a/components/studio/projects/templates/project_activity.html b/components/studio/projects/templates/project_activity.html new file mode 100644 index 000000000..acae4b30b --- /dev/null +++ b/components/studio/projects/templates/project_activity.html @@ -0,0 +1,15 @@ +
+
+
+ +
+
+
diff --git a/components/studio/projects/urls.py b/components/studio/projects/urls.py index 85ae21fbe..35d085a52 100644 --- a/components/studio/projects/urls.py +++ b/components/studio/projects/urls.py @@ -14,4 +14,5 @@ path('//details/change', views.change_description, name='change_description'), path('//project/publish', views.publish_project, name='publish_project'), path('//project/access/grant', views.grant_access_to_project, name='grant_access'), + path('//logs', views.load_project_activity, name='project_activity'), ] diff --git a/components/studio/projects/views.py b/components/studio/projects/views.py index 182708a6c..bfba48516 100644 --- a/components/studio/projects/views.py +++ b/components/studio/projects/views.py @@ -1,5 +1,5 @@ from django.shortcuts import render, reverse -from .models import Project, Environment +from .models import Project, Environment, ProjectLog from django.contrib.auth.decorators import login_required from django.http import HttpResponseRedirect, HttpResponse from .exceptions import ProjectCreationException @@ -9,15 +9,14 @@ import logging import markdown import time -from .forms import TransferProjectOwnershipForm, PublishProjectToGitHub, GrantAccessForm +from .forms import TransferProjectOwnershipForm, PublishProjectToGitHub from django.db.models import Q from models.models import Model import requests as r import base64 from projects.helpers import get_minio_keys import modules.keycloak_lib as kc -from multiprocessing import Process -from .tasks import create_keycloak_client_task +from datetime import datetime, timedelta logger = logging.getLogger(__name__) @@ -53,6 +52,11 @@ def settings(request, user, project_slug): new_owner = User.objects.filter(pk=new_owner_id).first() project.owner = new_owner project.save() + + l = ProjectLog(project=project, module='PR', headline='Project owner', + description='Transferred Project ownership to {owner}'.format(owner=project.owner.username)) + l.save() + return HttpResponseRedirect('/projects/') else: form = TransferProjectOwnershipForm() @@ -83,6 +87,11 @@ def change_environment(request, user, project_slug): if environment: project.environment = environment project.save() + + l = ProjectLog(project=project, module='PR', headline='New environment', + description='Project environment has been changed to {name}'. project.environment.name) + l.save() + # TODO fix the create_environment_image creation #from .helpers import create_environment_image #create_environment_image(project) @@ -98,6 +107,10 @@ def change_description(request, user, project_slug): if description is not '': project.description = description project.save() + + l = ProjectLog(project=project, module='PR', headline='Project description', + description='Changed description for project') + l.save() # TODO fix the create_environment_image creation return HttpResponseRedirect( @@ -116,6 +129,11 @@ def grant_access_to_project(request, user, project_slug): project.authorized.set(selected_users) project.save() + l = ProjectLog(project=project, module='PR', headline='New members', + description='{number} new members have been added to the Project'.format( + number=len(selected_users))) + l.save() + if len(selected_users) == 1: selected_users = list(selected_users) @@ -153,6 +171,14 @@ def create(request): if not success: project.delete() + else: + l1 = ProjectLog(project=project, module='PR', headline='Project created', + description='Created project {}'.format(project.name)) + l1.save() + + l2 = ProjectLog(project=project, module='PR', headline='Getting started', + description='Getting started with project {}'.format(project.name)) + l2.save() next_page = request.POST.get('next', '/{}/{}'.format(request.user, project.slug)) @@ -194,6 +220,8 @@ def details(request, user, project_slug): except Exception as e: logger.error("Failed to get response from {} with error: {}".format(url, e)) + project_logs = ProjectLog.objects.filter(project=project).order_by('-created_at') + return render(request, template, locals()) @@ -251,6 +279,11 @@ def publish_project(request, user, project_slug): if clone_url: project.clone_url = clone_url project.save() + + l = ProjectLog(project=project, module='PR', headline='GitHub repository', + description='Published project files to a GitHub repository {url}'.format( + url=project.clone_url)) + l.save() except Exception as e: logger.error("Failed to get response from {} with error: {}".format(url, e)) @@ -258,11 +291,20 @@ def publish_project(request, user, project_slug): reverse('projects:settings', kwargs={'user': user, 'project_slug': project_slug})) -# def auth(request): -# print(dir(request)) -# print(request.user) -# if request.user.is_authenticated: -# return HttpResponse('Ok', status=200) -# # return HttpResponse(status=200) -# else: -# return HttpResponse(status=403) +@login_required(login_url='/accounts/login') +def load_project_activity(request, user, project_slug): + template = 'project_activity.html' + + time_period = request.GET.get('period') + if time_period == 'week': + last_week = datetime.today() - timedelta(days=7) + project_logs = ProjectLog.objects.filter(created_at__gte=last_week).order_by('-created_at') + elif time_period == 'month': + last_month = datetime.today() - timedelta(days=30) + project_logs = ProjectLog.objects.filter(created_at__gte=last_month).order_by('-created_at') + else: + project_logs = ProjectLog.objects.all().order_by('-created_at') + + return render(request, template, {'project_logs': project_logs}) + + diff --git a/components/studio/reports/helpers.py b/components/studio/reports/helpers.py index f46e8703d..392fcae27 100644 --- a/components/studio/reports/helpers.py +++ b/components/studio/reports/helpers.py @@ -6,7 +6,7 @@ import json import subprocess from studio.minio import MinioRepository, ResponseError -from projects.models import Project +from projects.models import Project, ProjectLog import logging from projects.helpers import get_minio_keys @@ -45,6 +45,11 @@ def upload_report_json(report_id, client=None): client.put_object('reports', filename.replace('reports/', ''), file_data, file_stat.st_size, content_type='application/json') + l = ProjectLog(project=project, module='RE', headline='Metrics', + description='JSON file {filename} has been uploaded to MinIO'.format( + filename=filename.replace('reports/', ''))) + l.save() + except ResponseError as err: print(err) diff --git a/components/studio/reports/views.py b/components/studio/reports/views.py index 6dc3b3156..37b917876 100644 --- a/components/studio/reports/views.py +++ b/components/studio/reports/views.py @@ -2,7 +2,7 @@ from django.contrib.auth.decorators import login_required from django.http import HttpResponseRedirect from django.shortcuts import render -from projects.models import Project +from projects.models import Project, ProjectLog from .forms import ReportGeneratorForm from .models import Report, ReportGenerator from django.db.models import Q @@ -45,6 +45,10 @@ def add(request, user, project): if form.is_valid(): obj = form.save() + l = ProjectLog(project=project, module='RE', headline='Metrics', + description='A new Generator {id} has been added'.format(id=obj.pk)) + l.save() + get_visualiser_file(project.pk, obj.visualiser) url = '/{}/{}/reports/{}'.format(request.user, project.slug, obj.pk) @@ -104,7 +108,11 @@ def delete_generator(request, user, project, id): if request.method == "POST": if os.path.exists(path): os.unlink(path) - + + l = ProjectLog(project=project, module='RE', headline='Metrics', + description='Generator {id} has been removed'.format(id=report.pk)) + l.save() + report.delete() return HttpResponseRedirect('/{}/{}/reports/'.format(request.user, project.slug)) @@ -123,6 +131,10 @@ def delete_report(request, user, project, id): if os.path.exists(path): os.unlink(path) + l = ProjectLog(project=project, module='RE', headline='Metrics', + description='Metrics {id} has been removed'.format(id=report.pk)) + l.save() + report.delete() return HttpResponseRedirect('/{}/{}/models/'.format(request.user, project.slug)) diff --git a/components/studio/templates/baseproject.html b/components/studio/templates/baseproject.html index ccd738c6a..b95c7e0ed 100644 --- a/components/studio/templates/baseproject.html +++ b/components/studio/templates/baseproject.html @@ -102,3 +102,5 @@ + +{% block extrascripts %} {% endblock %} From 2fcb841c3e86cdecfd4688356c3aca24937a7c80 Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Fri, 28 Aug 2020 15:48:25 +0200 Subject: [PATCH 22/83] Set resource limits for deployments [JIRA](https://scaleoutsystems.atlassian.net/browse/STACKN-121) --- components/studio/deployments/forms.py | 6 +- .../templates/deploy/settings.html | 65 +++++++++++++++---- components/studio/deployments/views.py | 4 ++ components/studio/static/css/project.css | 20 +++++- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/components/studio/deployments/forms.py b/components/studio/deployments/forms.py index 8695871d6..eb87a94a1 100644 --- a/components/studio/deployments/forms.py +++ b/components/studio/deployments/forms.py @@ -17,4 +17,8 @@ class PredictForm(forms.Form): file = forms.FileField() class SettingsForm(forms.Form): - replicas = forms.IntegerField(min_value=1, max_value=25, label="Replicas") \ No newline at end of file + replicas = forms.IntegerField(min_value=1, max_value=25, label="Replicas") + limits_cpu = forms.IntegerField() + limits_memory = forms.IntegerField() + requests_cpu = forms.IntegerField() + requests_memory = forms.IntegerField() diff --git a/components/studio/deployments/templates/deploy/settings.html b/components/studio/deployments/templates/deploy/settings.html index 324918fa7..04c520493 100644 --- a/components/studio/deployments/templates/deploy/settings.html +++ b/components/studio/deployments/templates/deploy/settings.html @@ -9,16 +9,59 @@

Settings — {{ deployment.model.name }}:{{ deployment.model.tag }}

-
- {% csrf_token %} -
- {{ form }} -
-
- -
-
+
+ {% csrf_token %} + +
+ +
+ +
+
+ +
+ Limits +
+ +
+ +
+
+
+ +
+ +
+
+
- +
+ Requests +
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+
+ +
+
+
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/components/studio/deployments/views.py b/components/studio/deployments/views.py index 922e38ae0..412991d11 100644 --- a/components/studio/deployments/views.py +++ b/components/studio/deployments/views.py @@ -88,6 +88,10 @@ def serve_settings(request, id, project): form = SettingsForm(request.POST) if form.is_valid(): params['replicas'] = request.POST.get('replicas', 1) + params['resources.limits.cpu'] = request.POST.get('limits_cpu', 1000) + params['resources.limits.memory'] = request.POST.get('limits_memory', 2048) + params['resources.requests.cpu'] = request.POST.get('requests_cpu', 300) + params['resources.requests.memory'] = request.POST.get('requests_memory', 1024) print(params) deployment.helmchart.params = params deployment.helmchart.save() diff --git a/components/studio/static/css/project.css b/components/studio/static/css/project.css index 47eeb3c2c..9c4d11fdb 100644 --- a/components/studio/static/css/project.css +++ b/components/studio/static/css/project.css @@ -93,4 +93,22 @@ ul.timeline > li:before { .project-breadcrumb-ol { padding-left: 5px; background-color: #FBFCFC; -} \ No newline at end of file +} + +/* Deploy - Settings */ + +.fieldset-custom { + border: 1px solid #dee2e6; + border-radius: 5px; + padding: 20px; + margin-top: 32px; +} + +.fieldset-legend-custom { + padding: 0.2em 0.5em; + border: 1px solid #dee2e6; + font-size: 125%; + width: 25%; + margin-left: 15px; + border-radius: 5px; +} From ee7d3afe6399c0ac3f86f575fc7e99dca537522b Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Mon, 31 Aug 2020 10:10:09 +0200 Subject: [PATCH 23/83] Fix Django migrations. Don't remove existing migrations. --- components/studio/scripts/run_web.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/components/studio/scripts/run_web.sh b/components/studio/scripts/run_web.sh index 1df6ec50e..dae53e247 100755 --- a/components/studio/scripts/run_web.sh +++ b/components/studio/scripts/run_web.sh @@ -14,18 +14,10 @@ cd .. # If we have set a local, custom settings.py, then use that. [ -f studio/local_settings.py ] && echo "Using local settings file" && export DJANGO_SETTINGS_MODULE=studio.local_settings -echo "deleting all existing migrations..." -find . -path "*/migrations/*.py" -not -name "__init__.py" -delete -find . -path "*/migrations/*.pyc" -delete -echo "done!" echo "Installing all migrations" -python3 manage.py makemigrations auth -python3 manage.py migrate python3 manage.py makemigrations -python3 manage.py makemigrations ingress datasets deployments experiments files labs models projects reports workflows -python3 manage.py migrate -python3 manage.py makemigrations ingress datasets deployments experiments files labs models projects reports workflows python3 manage.py migrate + echo "loading seed data..." python3 manage.py loaddata projects/fixtures/fixtures.json python3 manage.py loaddata projects/fixtures/data.json From 40b45865c2e2ea3899105ac571227258329705fc Mon Sep 17 00:00:00 2001 From: Desislava Stoyanova Date: Mon, 31 Aug 2020 11:43:04 +0200 Subject: [PATCH 24/83] Fix Django migrations. Make migrations files part of the studio pod since they are needed when updating the database schema. This requires running `python manage.py makemigrations` locally if you have done changes to the model classes. This commit includes initial migrations based on `develop` branch. --- .gitignore | 1 - .../deployments/migrations/0001_initial.py | 63 ++++++++++++++++ .../studio/deployments/migrations/__init__.py | 0 .../experiments/migrations/0001_initial.py | 32 ++++++++ .../studio/experiments/migrations/__init__.py | 0 .../studio/ingress/migrations/0001_initial.py | 24 ++++++ .../studio/ingress/migrations/__init__.py | 0 .../studio/labs/migrations/0001_initial.py | 37 +++++++++ components/studio/labs/migrations/__init__.py | 0 .../studio/models/migrations/0001_initial.py | 40 ++++++++++ .../studio/models/migrations/__init__.py | 0 .../alliance_admin/migrations/__init__.py | 0 .../projects/migrations/0001_initial.py | 75 +++++++++++++++++++ .../studio/projects/migrations/__init__.py | 0 .../studio/reports/migrations/0001_initial.py | 41 ++++++++++ .../studio/reports/migrations/__init__.py | 0 components/studio/scripts/run_web.sh | 1 - .../workflows/migrations/0001_initial.py | 39 ++++++++++ .../studio/workflows/migrations/__init__.py | 0 19 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 components/studio/deployments/migrations/0001_initial.py create mode 100644 components/studio/deployments/migrations/__init__.py create mode 100644 components/studio/experiments/migrations/0001_initial.py create mode 100644 components/studio/experiments/migrations/__init__.py create mode 100644 components/studio/ingress/migrations/0001_initial.py create mode 100644 components/studio/ingress/migrations/__init__.py create mode 100644 components/studio/labs/migrations/0001_initial.py create mode 100644 components/studio/labs/migrations/__init__.py create mode 100644 components/studio/models/migrations/0001_initial.py create mode 100644 components/studio/models/migrations/__init__.py create mode 100644 components/studio/modules/scaleout-studio-alliance-admin/alliance_admin/migrations/__init__.py create mode 100644 components/studio/projects/migrations/0001_initial.py create mode 100644 components/studio/projects/migrations/__init__.py create mode 100644 components/studio/reports/migrations/0001_initial.py create mode 100644 components/studio/reports/migrations/__init__.py create mode 100644 components/studio/workflows/migrations/0001_initial.py create mode 100644 components/studio/workflows/migrations/__init__.py diff --git a/.gitignore b/.gitignore index 85eef7793..342246687 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ # Byte-compiled / optimized / DLL files -migrations/ .idea/ .DS_Store __pycache__/ diff --git a/components/studio/deployments/migrations/0001_initial.py b/components/studio/deployments/migrations/0001_initial.py new file mode 100644 index 000000000..c75b6a2e5 --- /dev/null +++ b/components/studio/deployments/migrations/0001_initial.py @@ -0,0 +1,63 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + ('models', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='DeploymentDefinition', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('access', models.CharField(choices=[('PR', 'Private'), ('PU', 'Public')], default='PR', max_length=2)), + ('name', models.CharField(max_length=512, unique=True)), + ('bucket', models.CharField(blank=True, max_length=512, null=True)), + ('filename', models.CharField(blank=True, max_length=512, null=True)), + ('image', models.CharField(blank=True, max_length=512, null=True)), + ('path_predict', models.CharField(max_length=512)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('uploaded_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='project_owner', to='projects.Project')), + ], + ), + migrations.CreateModel( + name='HelmResource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512, unique=True)), + ('namespace', models.CharField(max_length=512)), + ('chart', models.CharField(max_length=512)), + ('params', models.CharField(max_length=2048)), + ('username', models.CharField(max_length=512)), + ('status', models.CharField(max_length=20)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='DeploymentInstance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('appname', models.CharField(max_length=512)), + ('access', models.CharField(choices=[('PR', 'Private'), ('LI', 'Limited'), ('PU', 'Public')], default='PR', max_length=2)), + ('endpoint', models.CharField(max_length=512)), + ('path', models.CharField(max_length=512)), + ('release', models.CharField(max_length=512)), + ('created_by', models.CharField(max_length=512)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('uploaded_at', models.DateTimeField(auto_now=True)), + ('deployment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='deployments.DeploymentDefinition')), + ('helmchart', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='deployments.HelmResource')), + ('model', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='deployed_model', to='models.Model')), + ], + ), + ] diff --git a/components/studio/deployments/migrations/__init__.py b/components/studio/deployments/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/experiments/migrations/0001_initial.py b/components/studio/experiments/migrations/0001_initial.py new file mode 100644 index 000000000..3e8c589ee --- /dev/null +++ b/components/studio/experiments/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + ('deployments', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Experiment', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('username', models.CharField(max_length=512)), + ('command', models.CharField(max_length=1024)), + ('schedule', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('uploaded_at', models.DateTimeField(auto_now=True)), + ('environment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='job_environment', to='projects.Environment')), + ('helmchart', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='deployments.HelmResource')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='job_project', to='projects.Project')), + ], + ), + ] diff --git a/components/studio/experiments/migrations/__init__.py b/components/studio/experiments/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/ingress/migrations/0001_initial.py b/components/studio/ingress/migrations/0001_initial.py new file mode 100644 index 000000000..2e4cf1ccd --- /dev/null +++ b/components/studio/ingress/migrations/0001_initial.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Page', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('page', models.TextField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/components/studio/ingress/migrations/__init__.py b/components/studio/ingress/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/labs/migrations/0001_initial.py b/components/studio/labs/migrations/0001_initial.py new file mode 100644 index 000000000..36adba5c9 --- /dev/null +++ b/components/studio/labs/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Session', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=512)), + ('slug', models.CharField(max_length=512)), + ('session_key', models.CharField(max_length=512)), + ('session_secret', models.CharField(max_length=512)), + ('settings', models.TextField()), + ('chart', models.CharField(max_length=512)), + ('helm_repo', models.CharField(blank=True, max_length=1024, null=True)), + ('status', models.CharField(choices=[('CR', 'Created'), ('ST', 'Started'), ('ST', 'Stopped'), ('FN', 'Finished'), ('AB', 'Aborted')], default='CR', max_length=2)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('lab_session_owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='lab_session_owner', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='session', to='projects.Project')), + ], + ), + ] diff --git a/components/studio/labs/migrations/__init__.py b/components/studio/labs/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/models/migrations/0001_initial.py b/components/studio/models/migrations/0001_initial.py new file mode 100644 index 000000000..897da42ec --- /dev/null +++ b/components/studio/models/migrations/0001_initial.py @@ -0,0 +1,40 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.manager + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Model', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('version', models.CharField(max_length=255)), + ('release_type', models.CharField(max_length=255)), + ('description', models.CharField(blank=True, max_length=255, null=True)), + ('access', models.CharField(choices=[('PR', 'Private'), ('LI', 'Limited'), ('PU', 'Public')], default='PR', max_length=2)), + ('resource', models.URLField(blank=True, max_length=2048, null=True)), + ('url', models.URLField(blank=True, max_length=512, null=True)), + ('uploaded_at', models.DateTimeField(auto_now_add=True)), + ('status', models.CharField(choices=[('CR', 'Created'), ('DP', 'Deployed'), ('AR', 'Archived')], default='CR', max_length=2)), + ('project', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='model_owner', to='projects.Project')), + ], + options={ + 'unique_together': {('name', 'version', 'project')}, + }, + managers=[ + ('objects_version', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/components/studio/models/migrations/__init__.py b/components/studio/models/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/modules/scaleout-studio-alliance-admin/alliance_admin/migrations/__init__.py b/components/studio/modules/scaleout-studio-alliance-admin/alliance_admin/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/projects/migrations/0001_initial.py b/components/studio/projects/migrations/0001_initial.py new file mode 100644 index 000000000..7e4b79550 --- /dev/null +++ b/components/studio/projects/migrations/0001_initial.py @@ -0,0 +1,75 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Environment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512)), + ('slug', models.CharField(blank=True, max_length=512, null=True)), + ('image', models.CharField(max_length=512)), + ('dockerfile', models.TextField(default='FROM jupyter/base-notebook')), + ('startup', models.TextField(blank=True, null=True)), + ('teardown', models.TextField(blank=True, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Flavor', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512)), + ('slug', models.CharField(max_length=512)), + ('cpu', models.TextField(blank=True, null=True)), + ('mem', models.TextField(blank=True, null=True)), + ('gpu', models.TextField(blank=True, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512, unique=True)), + ('description', models.TextField(blank=True, null=True)), + ('slug', models.CharField(max_length=512, unique=True)), + ('image', models.CharField(blank=True, max_length=2048, null=True)), + ('project_key', models.CharField(max_length=512)), + ('project_secret', models.CharField(max_length=512)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('repository', models.CharField(blank=True, max_length=512, null=True)), + ('repository_imported', models.BooleanField(default=False)), + ('clone_url', models.CharField(blank=True, max_length=512, null=True)), + ('authorized', models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL)), + ('environment', models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='projects.Environment')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, related_name='owner', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='ProjectLog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('module', models.CharField(choices=[('DE', 'deployments'), ('LA', 'labs'), ('MO', 'models'), ('PR', 'projects'), ('RE', 'reports'), ('UN', 'undefined')], default='UN', max_length=2)), + ('headline', models.CharField(max_length=256)), + ('description', models.CharField(max_length=512)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.Project')), + ], + ), + ] diff --git a/components/studio/projects/migrations/__init__.py b/components/studio/projects/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/reports/migrations/0001_initial.py b/components/studio/reports/migrations/0001_initial.py new file mode 100644 index 000000000..a763935f0 --- /dev/null +++ b/components/studio/reports/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + ('models', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ReportGenerator', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField(blank=True)), + ('generator', models.CharField(max_length=256)), + ('visualiser', models.CharField(max_length=256)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.Project')), + ], + ), + migrations.CreateModel( + name='Report', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField(blank=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('report', models.TextField(blank=True)), + ('job_id', models.CharField(max_length=256)), + ('status', models.CharField(choices=[('I', 'Initiated'), ('P', 'Processing'), ('C', 'Completed'), ('F', 'Failed')], default='I', max_length=1)), + ('generator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='reports.ReportGenerator')), + ('model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='models.Model')), + ], + ), + ] diff --git a/components/studio/reports/migrations/__init__.py b/components/studio/reports/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/scripts/run_web.sh b/components/studio/scripts/run_web.sh index dae53e247..446a81136 100755 --- a/components/studio/scripts/run_web.sh +++ b/components/studio/scripts/run_web.sh @@ -15,7 +15,6 @@ cd .. [ -f studio/local_settings.py ] && echo "Using local settings file" && export DJANGO_SETTINGS_MODULE=studio.local_settings echo "Installing all migrations" -python3 manage.py makemigrations python3 manage.py migrate echo "loading seed data..." diff --git a/components/studio/workflows/migrations/0001_initial.py b/components/studio/workflows/migrations/0001_initial.py new file mode 100644 index 000000000..5530fd7dd --- /dev/null +++ b/components/studio/workflows/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.13 on 2020-08-31 08:56 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='WorkflowDefinition', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=512, unique=True)), + ('definition', models.TextField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('uploaded_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='WorkflowInstance', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('CR', 'Created'), ('ST', 'Started'), ('ST', 'Stopped'), ('FN', 'Finished'), ('AB', 'Aborted')], default='CR', max_length=2)), + ('name', models.CharField(max_length=512)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('uploaded_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project', to='projects.Project')), + ('workflow', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='workflow_definition', to='workflows.WorkflowDefinition')), + ], + ), + ] diff --git a/components/studio/workflows/migrations/__init__.py b/components/studio/workflows/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From 343b013e4571961d03609a1f711464faca62d505 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Mon, 31 Aug 2020 21:48:17 +0200 Subject: [PATCH 25/83] Fix: Access token expiration problem --- components/studio/experiments/views.py | 4 +++ components/studio/modules/keycloak_lib.py | 33 ++++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index 3b88c2ad6..a6682b4a4 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -12,11 +12,15 @@ from django.urls import reverse import modules.keycloak_lib as keylib from .experimentsauth import get_permissions +import logging +logger = logging.getLogger(__name__) @login_required(login_url='/accounts/login') def index(request, user, project): + print('User: {}'.format(user)) user_permissions = get_permissions(request, project) + logger.info(user_permissions) if not user_permissions['view']: return HttpResponse('Not authorized', status=401) diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index 84e6b4db6..9f8a7d3da 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -2,6 +2,7 @@ import requests as r import logging import jwt +import time logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -87,7 +88,7 @@ def keycloak_init(): print('Failed to init Keycloak auth') return False -def keycloak_get_detailed_user_info(request, aud='account'): +def keycloak_get_detailed_user_info(request, aud='account', renew_token_if_expired=True): if not ('oidc_access_token' in request.session): logger.warn('No access token in request session -- unable to authorize user.') return [] @@ -103,9 +104,35 @@ def keycloak_get_detailed_user_info(request, aud='account'): print('Failed to discover realm settings: '+settings.KC_REALM) return None try: + print('Decoding user token: {}'.format(request.user)) user_json = jwt.decode(access_token, public_key, algorithms='RS256', audience=aud) - except: - logger.info('Failed to authenticate user.') + print('Successfully decoded token.') + print('Token expires: {}'.format(request.session['oidc_id_token_expiration'])) + print('Time now: {}'.format(time.time())) + time_left = (request.session['oidc_id_token_expiration']-time.time())/60 + print(time_left) + print(request.session.keys()) + logger.debug(user_json) + except jwt.ExpiredSignatureError: + print('Token has expired.') + print('Attempting renewal.') + if renew_token_if_expired: + kc = keycloak_init() + token, refresh_token, token_url, public_key = keycloak_token_exchange_studio(kc, request.user.username) + request.session['oidc_access_token'] = token + request.session.save() + return keycloak_get_detailed_user_info(request, aud='account', renew_token_if_expired=False) + + except Exception as err: + print('Failed to authenticate user.') + print('Reason: ') + print(err) + print(request.session.keys()) + print(request.session['oidc_id_token_expiration']) + print(request.session['oidc_id_token']) + print(time.time()) + + return user_json def keycloak_get_user_roles(request, resource, aud='account'): From 4a03804a0e84b24002237e8cb35bcfccd54b16af Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Mon, 31 Aug 2020 22:03:52 +0200 Subject: [PATCH 26/83] Fix in experiments log view query --- components/studio/experiments/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/studio/experiments/views.py b/components/studio/experiments/views.py index a6682b4a4..2faf2a8f8 100644 --- a/components/studio/experiments/views.py +++ b/components/studio/experiments/views.py @@ -27,6 +27,7 @@ def index(request, user, project): temp = 'experiments/index.html' project = Project.objects.filter(slug=project).first() + print('Project: '+project.slug) experiments = Experiment.objects.filter(project=project) environments = Environment.objects.all() @@ -43,7 +44,6 @@ def run(request, user, project): temp = 'experiments/run.html' project = Project.objects.filter(slug=project).first() - deployment = None if request.method == "POST": form = ExperimentForm(request.POST) @@ -84,7 +84,7 @@ def details(request, user, project, id): try: url = settings.LOKI_SVC+'/loki/api/v1/query_range' query = { - 'query': '{type="cronjob", project="demo-vqo", app="'+experiment.helmchart.name+'"}', + 'query': '{type="cronjob", project="'+project.slug+'", app="'+experiment.helmchart.name+'"}', 'limit': 50, 'start': 0, } From 77730002c790520315a691b7f82f92195ceca489 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 1 Sep 2020 10:36:39 +0200 Subject: [PATCH 27/83] Fix print messages not working with CLI --- components/studio/modules/keycloak_lib.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/studio/modules/keycloak_lib.py b/components/studio/modules/keycloak_lib.py index 9f8a7d3da..87a95ee6a 100644 --- a/components/studio/modules/keycloak_lib.py +++ b/components/studio/modules/keycloak_lib.py @@ -104,14 +104,14 @@ def keycloak_get_detailed_user_info(request, aud='account', renew_token_if_expir print('Failed to discover realm settings: '+settings.KC_REALM) return None try: - print('Decoding user token: {}'.format(request.user)) + # print('Decoding user token: {}'.format(request.user)) user_json = jwt.decode(access_token, public_key, algorithms='RS256', audience=aud) - print('Successfully decoded token.') - print('Token expires: {}'.format(request.session['oidc_id_token_expiration'])) - print('Time now: {}'.format(time.time())) - time_left = (request.session['oidc_id_token_expiration']-time.time())/60 - print(time_left) - print(request.session.keys()) + # print('Successfully decoded token.') + # print('Token expires: {}'.format(request.session['oidc_id_token_expiration'])) + # print('Time now: {}'.format(time.time())) + # time_left = (request.session['oidc_id_token_expiration']-time.time())/60 + # print(time_left) + # print(request.session.keys()) logger.debug(user_json) except jwt.ExpiredSignatureError: print('Token has expired.') From 278a522780503ae884142783258a4d764dda5a87 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 1 Sep 2020 12:59:38 +0200 Subject: [PATCH 28/83] refactor labs creation --- cli/scaleout/studioclient.py | 9 +- components/studio/labs/helpers.py | 44 ++--- components/studio/labs/models.py | 122 ++++++++++++- components/studio/labs/views.py | 161 +++++++++--------- components/studio/monitor/helpers.py | 20 ++- .../monitor/templates/monitor_overview.html | 92 +++++++--- components/studio/monitor/views.py | 34 +++- 7 files changed, 348 insertions(+), 134 deletions(-) diff --git a/cli/scaleout/studioclient.py b/cli/scaleout/studioclient.py index 531333f0e..139a50ca4 100644 --- a/cli/scaleout/studioclient.py +++ b/cli/scaleout/studioclient.py @@ -342,7 +342,14 @@ def create_list(self, resource): def get_models(self, project_id, params=[]): url = self.endpoints['models'].format(project_id) r = requests.get(url, headers=self.auth_headers, params=params) - models = json.loads(r.content) + try: + models = json.loads(r.content) + except Exception as err: + print('Failed to list models.') + print('Status code: {}'.format(r.status_code)) + print('Message: {}'.format(r.text)) + print('Error: {}'.format(err)) + return [] return models def get_model(self, project_id, model_id): diff --git a/components/studio/labs/helpers.py b/components/studio/labs/helpers.py index 7a78c62fb..b747e5f39 100644 --- a/components/studio/labs/helpers.py +++ b/components/studio/labs/helpers.py @@ -17,36 +17,36 @@ def create_user_settings(user): user_settings['studio_url'] = 'https://'+settings.DOMAIN return json.dumps(user_settings) -def create_session_resources(request, user, session, prefs, project): +# def create_session_resources(request, user, session, prefs, project): - print("1; going for the dispatch!") +# print("1; going for the dispatch!") - HOST = settings.DOMAIN - RELEASE_NAME = str(session.slug) +# HOST = settings.DOMAIN +# RELEASE_NAME = str(session.slug) - URL = 'https://'+RELEASE_NAME+'.'+HOST - client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) +# URL = 'https://'+RELEASE_NAME+'.'+HOST +# client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) - parameters = {'release': str(session.slug), - 'chart': session.chart, - 'global.domain': settings.DOMAIN, - 'project.name': project.slug, - 'gatekeeper.realm': settings.KC_REALM, - 'gatekeeper.client_secret': client_secret, - 'gatekeeper.client_id': client_id, - 'gatekeeper.auth_endpoint': settings.OIDC_OP_REALM_AUTH - } - parameters.update(prefs) +# parameters = {'release': str(session.slug), +# 'chart': session.chart, +# 'global.domain': settings.DOMAIN, +# 'project.name': project.slug, +# 'gatekeeper.realm': settings.KC_REALM, +# 'gatekeeper.client_secret': client_secret, +# 'gatekeeper.client_id': client_id, +# 'gatekeeper.auth_endpoint': settings.OIDC_OP_REALM_AUTH +# } +# parameters.update(prefs) - url = settings.CHART_CONTROLLER_URL + '/deploy' +# url = settings.CHART_CONTROLLER_URL + '/deploy' - retval = r.get(url, parameters) - print("CREATE_SESSION:helm chart creator returned {}".format(retval)) +# retval = r.get(url, parameters) +# print("CREATE_SESSION:helm chart creator returned {}".format(retval)) - if retval.status_code >= 200 or retval.status_code < 205: - return True +# if retval.status_code >= 200 or retval.status_code < 205: +# return True - return False +# return False def delete_session_resources(session): diff --git a/components/studio/labs/models.py b/components/studio/labs/models.py index 025dcb43e..7b3678610 100644 --- a/components/studio/labs/models.py +++ b/components/studio/labs/models.py @@ -1,10 +1,23 @@ from django.db import models from django.contrib.auth.models import User +from django.db.models.signals import pre_delete, pre_save +from django.dispatch import receiver +from django.conf import settings +from django.db.models import Q +from projects.helpers import get_minio_keys +from django.core import serializers +from .helpers import create_user_settings +from api.serializers import ProjectSerializer +from rest_framework.renderers import JSONRenderer import uuid import yaml +import json from django.contrib.postgres.fields import ArrayField from django.utils.text import slugify - +from deployments.models import HelmResource +from projects.models import Environment, Flavor +from projects.models import Project, ProjectLog +from modules import keycloak_lib as keylib class SessionManager(models.Manager): @@ -57,19 +70,110 @@ class Session(models.Model): project = models.ForeignKey('projects.Project', on_delete=models.CASCADE, related_name='session') lab_session_owner = models.ForeignKey(User, on_delete=models.CASCADE, related_name='lab_session_owner') - session_key = models.CharField(max_length=512) - session_secret = models.CharField(max_length=512) + helmchart = models.OneToOneField('deployments.HelmResource', on_delete=models.CASCADE) + keycloak_client_id = models.CharField(max_length=512) + appname = models.CharField(max_length=512) + flavor_slug = models.CharField(max_length=512) + environment_slug = models.CharField(max_length=512) + + # session_key = models.CharField(max_length=512) + # session_secret = models.CharField(max_length=512) settings = models.TextField() - chart = models.CharField(max_length=512) + # chart = models.CharField(max_length=512) helm_repo = models.CharField(max_length=1024, null=True, blank=True) status = models.CharField(max_length=2, choices=STATUS, default=CREATED) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) -# class Chart(models.Model): -# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) -# name = models.CharField(max_length=512, unique=False) +@receiver(pre_delete, sender=Session, dispatch_uid='session_pre_delete_signal') +def pre_delete_labs(sender, instance, using, **kwargs): + kc = keylib.keycloak_init() + keylib.keycloak_delete_client(kc, instance.keycloak_client_id) + + scope_id = keylib.keycloak_get_client_scope_id(kc, instance.keycloak_client_id+'-scope') + keylib.keycloak_delete_client_scope(kc, scope_id) + + l = ProjectLog(project=instance.project, module='LA', headline='Lab Session', + description='Lab Session {name} has been removed'.format(name=instance.name)) + l.save() + +@receiver(pre_save, sender=Session, dispatch_uid='session_pre_save_signal') +def pre_save_labs(sender, instance, using, **kwargs): + + instance.slug = slugify(instance.name) + + RELEASE_NAME = str(instance.slug) + HOST = settings.DOMAIN + URL = 'https://'+RELEASE_NAME+'.'+HOST + user = instance.lab_session_owner + client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) + + instance.keycloak_client_id = client_id + + parameters = {'release': RELEASE_NAME, + 'chart': instance.chart, + 'global.domain': settings.DOMAIN, + 'project.name': instance.project.slug, + 'gatekeeper.realm': settings.KC_REALM, + 'gatekeeper.client_secret': client_secret, + 'gatekeeper.client_id': client_id, + 'gatekeeper.auth_endpoint': settings.OIDC_OP_REALM_AUTH + } + + ingress_secret_name = 'prod-ingress' + try: + ingress_secret_name = settings.LABS['ingress']['secretName'] + except: + pass + + project = instance.project + minio_keys = get_minio_keys(project) + decrypted_key = minio_keys['project_key'] + decrypted_secret = minio_keys['project_secret'] + + settings_file = ProjectSerializer(project) + + settings_file = JSONRenderer().render(settings_file.data) + settings_file = settings_file.decode('utf-8') + + settings_file = json.loads(settings_file) + settings_file = yaml.dump(settings_file) + + user_config_file = create_user_settings(user) + user_config_file = yaml.dump(json.loads(user_config_file)) + + flavor = Flavor.objects.filter(slug=instance.flavor_slug).first() + environment = Environment.objects.filter(slug=instance.environment_slug).first() + + prefs = {'namespace': settings.NAMESPACE, + 'labs.resources.requests.cpu': str(flavor.cpu), + 'labs.resources.limits.cpu': str(flavor.cpu), + 'labs.resources.requests.memory': str(flavor.mem), + 'labs.resources.limits.memory': str(flavor.mem), + 'labs.resources.requests.gpu': str(flavor.gpu), + 'labs.resources.limits.gpu': str(flavor.gpu), + 'labs.gpu.enabled': str("true" if flavor.gpu else "false"), + 'labs.image': environment.image, + 'ingress.secretName': ingress_secret_name, + 'minio.access_key': decrypted_key, + 'minio.secret_key': decrypted_secret, + 'settings_file': settings_file, + 'user_settings_file': user_config_file, + 'project.slug': project.slug + } + + parameters.update(prefs) + instance.settings = parameters + + helmchart = HelmResource(name=RELEASE_NAME, + namespace='Default', + chart='lab', + params=parameters, + username=user) + helmchart.save() + instance.helmchart = helmchart -# created_at = models.DateTimeField(auto_now_add=True) -# updated_at = models.DateTimeField(auto_now=True) + l = ProjectLog(project=project, module='LA', headline='Lab Session', + description='A new Lab Session {name} has been created'.format(name=RELEASE_NAME)) + l.save() \ No newline at end of file diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index b6f68ec24..3aee31643 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -35,82 +35,89 @@ def run(request, user, project): name = str(project.slug) + str(uid)[0:7] flavor_slug = request.POST.get('flavor', None) environment_slug = request.POST.get('environment', None) - - if flavor_slug: - flavor = Flavor.objects.filter(slug=flavor_slug).first() - else: - flavor = Flavor.objects.all().first() - - if environment_slug: - environment = Environment.objects.filter(slug=environment_slug).first() - else: - environment = Environment.objects.filter.all().first() - - print("dispatching with {} {}".format(flavor, name)) - import base64 - if name != '' and flavor is not None: - # Default values here, because otherwise an old deployment can stop working - # if the deployment configuration files are not compatible with the latest - # studio image. - ingress_secret_name = 'prod-ingress' - try: - ingress_secret_name = settings.LABS['ingress']['secretName'] - except: - pass - - minio_keys = get_minio_keys(project) - decrypted_key = minio_keys['project_key'] - decrypted_secret = minio_keys['project_secret'] - - settings_file = ProjectSerializer(project) - - settings_file = JSONRenderer().render(settings_file.data) - settings_file = settings_file.decode('utf-8') - - settings_file = json.loads(settings_file) - settings_file = yaml.dump(settings_file) - - user_config_file = create_user_settings(user) - user_config_file = yaml.dump(json.loads(user_config_file)) - - prefs = {'namespace': settings.NAMESPACE, - 'labs.resources.requests.cpu': str(flavor.cpu), - 'labs.resources.limits.cpu': str(flavor.cpu), - 'labs.resources.requests.memory': str(flavor.mem), - 'labs.resources.limits.memory': str(flavor.mem), - 'labs.resources.requests.gpu': str(flavor.gpu), - 'labs.resources.limits.gpu': str(flavor.gpu), - 'labs.gpu.enabled': str("true" if flavor.gpu else "false"), - 'labs.image': environment.image, - 'ingress.secretName': ingress_secret_name, - # 'labs.setup': environment.setup, - 'minio.access_key': decrypted_key, - 'minio.secret_key': decrypted_secret, - 'settings_file': settings_file, - 'user_settings_file': user_config_file, - 'project.slug': project.slug - } - session = Session.objects.create_session(name=name, project=project, lab_session_owner=request.user, - chart='lab', settings=prefs) - from .helpers import create_session_resources - - print("trying to create resources") - retval = create_session_resources(request, user, session, prefs, project) - if retval: - print("saving session!") - project.save() - session.save() - - l = ProjectLog(project=project, module='LA', headline='Lab Session', - description='A new Lab Session {name} has been created'.format(name=name)) - l.save() - - return HttpResponseRedirect( - reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) + + lab_instance = Session(name=name, + id=uid, + flavor_slug=flavor_slug, + environment_slug=environment_slug, + project=project, + lab_session_owner=user) + lab_instance.save() + # if flavor_slug: + # flavor = Flavor.objects.filter(slug=flavor_slug).first() + # else: + # flavor = Flavor.objects.all().first() + + # if environment_slug: + # environment = Environment.objects.filter(slug=environment_slug).first() + # else: + # environment = Environment.objects.filter.all().first() + + # print("dispatching with {} {}".format(flavor, name)) + # import base64 + # if name != '' and flavor is not None: + # # Default values here, because otherwise an old deployment can stop working + # # if the deployment configuration files are not compatible with the latest + # # studio image. + # ingress_secret_name = 'prod-ingress' + # try: + # ingress_secret_name = settings.LABS['ingress']['secretName'] + # except: + # pass + + # minio_keys = get_minio_keys(project) + # decrypted_key = minio_keys['project_key'] + # decrypted_secret = minio_keys['project_secret'] + + # settings_file = ProjectSerializer(project) + + # settings_file = JSONRenderer().render(settings_file.data) + # settings_file = settings_file.decode('utf-8') + + # settings_file = json.loads(settings_file) + # settings_file = yaml.dump(settings_file) + + # user_config_file = create_user_settings(user) + # user_config_file = yaml.dump(json.loads(user_config_file)) + + # prefs = {'namespace': settings.NAMESPACE, + # 'labs.resources.requests.cpu': str(flavor.cpu), + # 'labs.resources.limits.cpu': str(flavor.cpu), + # 'labs.resources.requests.memory': str(flavor.mem), + # 'labs.resources.limits.memory': str(flavor.mem), + # 'labs.resources.requests.gpu': str(flavor.gpu), + # 'labs.resources.limits.gpu': str(flavor.gpu), + # 'labs.gpu.enabled': str("true" if flavor.gpu else "false"), + # 'labs.image': environment.image, + # 'ingress.secretName': ingress_secret_name, + # # 'labs.setup': environment.setup, + # 'minio.access_key': decrypted_key, + # 'minio.secret_key': decrypted_secret, + # 'settings_file': settings_file, + # 'user_settings_file': user_config_file, + # 'project.slug': project.slug + # } + # session = Session.objects.create_session(name=name, project=project, lab_session_owner=request.user, + # chart='lab', settings=prefs) + # from .helpers import create_session_resources + + # print("trying to create resources") + # retval = create_session_resources(request, user, session, prefs, project) + # if retval: + # print("saving session!") + # project.save() + # session.save() + + # l = ProjectLog(project=project, module='LA', headline='Lab Session', + # description='A new Lab Session {name} has been created'.format(name=name)) + # l.save() return HttpResponseRedirect( reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) + # return HttpResponseRedirect( + # reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) + @login_required(login_url='/accounts/login') def delete(request, user, project, id): @@ -118,12 +125,12 @@ def delete(request, user, project, id): session = Session.objects.filter(Q(id=id), Q(project=project), Q(lab_session_owner=request.user)).first() if session: - from .helpers import delete_session_resources - delete_session_resources(session) + # from .helpers import delete_session_resources + # delete_session_resources(session) - l = ProjectLog(project=project, module='LA', headline='Lab Session', - description='Lab Session {name} has been removed'.format(name=session.name)) - l.save() + # l = ProjectLog(project=project, module='LA', headline='Lab Session', + # description='Lab Session {name} has been removed'.format(name=session.name)) + # l.save() session.delete() diff --git a/components/studio/monitor/helpers.py b/components/studio/monitor/helpers.py index 5730d436e..5ac5f2bee 100644 --- a/components/studio/monitor/helpers.py +++ b/components/studio/monitor/helpers.py @@ -96,4 +96,22 @@ def get_labs_cpu_requests(project_slug): if result: num_cpus = result[0]['value'][1] return num_cpus - return 0 \ No newline at end of file + return 0 + +def get_resource(project_slug, resource_type, q_type, mem_or_cpu): + query = 'sum(kube_pod_container_resource_'+q_type+'_'+mem_or_cpu+' * on(pod) group_left kube_pod_labels{label_project="'+project_slug+'", label_type="'+resource_type+'"})' + print(query) + response = r.get(settings.PROMETHEUS_SVC+'/api/v1/query', params={'query':query}) + result = response.json()['data']['result'] + if result: + res = result[0]['value'][1] + # if mem_or_cpu == 'memory_bytes': + # res = "{:.2f}".format(float(res)/1e9*0.931323) + return res + return 0 + +def get_all(): + query = 'kube_pod_container_resource_limits_memory_bytes * on(pod) group_left kube_pod_labels{label_type="lab", label_project="stochss-dev-tiz"}' + response = r.get(settings.PROMETHEUS_SVC+'/api/v1/query', params={'query':query}) + result = response.json()['data']['result'] + print(result) \ No newline at end of file diff --git a/components/studio/monitor/templates/monitor_overview.html b/components/studio/monitor/templates/monitor_overview.html index 378a60102..6aa91523d 100644 --- a/components/studio/monitor/templates/monitor_overview.html +++ b/components/studio/monitor/templates/monitor_overview.html @@ -10,29 +10,81 @@

Monitor

+

Project Quota

+ + + + + + + + + + + + + + + + + + + +
Limits
TotalCPU100 cores
Memory100 Gi
+ + +

Project Usage

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RequestsLimits
LabsCPU{{ resource_status.lab.requests.cpu_cores }}{{ resource_status.lab.limits.cpu_cores }}
Memory{{ resource_status.lab.requests.memory_bytes }} Gi{{ resource_status.lab.limits.memory_bytes }} Gi
DeploymentsCPU{{ resource_status.deployment.requests.cpu_cores }}{{ resource_status.deployment.limits.cpu_cores }}
Memory Gi
TotalCPU{{ total_cpu }}
Memory{{ total_mem }} Gi
-

Total requests

- CPU:
- Memory: - -

Lab sessions

- -

Resources

-
-

- Labs requests CPU: {{ labs_cpu_requests }} cores
- Labs requests memory: {{ labs_memory_requests }} Gi
-

-
+ -
- -

Deployments

-
- -
+
diff --git a/components/studio/monitor/views.py b/components/studio/monitor/views.py index 629ba4aae..dd277fe32 100644 --- a/components/studio/monitor/views.py +++ b/components/studio/monitor/views.py @@ -11,16 +11,42 @@ from .helpers import get_total_labs_cpu_usage_60s, get_total_labs_memory_usage_60s from .helpers import get_labs_cpu_requests, get_labs_memory_requests from .helpers import get_total_cpu_usage_60s_ts +from .helpers import get_resource, get_all +from labs.models import Session + @login_required(login_url='/accounts/login') def overview(request, user, project): template = 'monitor_overview.html' project = Project.objects.filter(slug=project).first() - labs_cpu_requests = get_labs_cpu_requests(project.slug) - labs_memory_requests = get_labs_memory_requests(project.slug) - labs_cpu_usage = get_total_labs_cpu_usage_60s(project.slug) - labs_memory_usage = get_total_labs_memory_usage_60s(project.slug) + resource_types = ['lab', 'deployment'] + q_types = ['requests', 'limits'] + r_types = ['memory_bytes', 'cpu_cores'] + + resource_status = dict() + for resource_type in resource_types: + resource_status[resource_type] = dict() + for q_type in q_types: + resource_status[resource_type][q_type] = dict() + for r_type in r_types: + tmp = get_resource(project.slug, resource_type, q_type, r_type) + if r_type == 'memory_bytes': + tmp ="{:.2f}".format(float(tmp)/1e9*0.931323) + else: + tmp = "{:.2f}".format(float(tmp)) + resource_status[resource_type][q_type][r_type] = tmp + + total_cpu = float(resource_status['lab']['limits']['cpu_cores'])+float(resource_status['deployment']['limits']['cpu_cores']) + total_mem = float(resource_status['lab']['limits']['memory_bytes'])+float(resource_status['deployment']['limits']['memory_bytes']) + + labs = Session.objects.filter(project=project) + for lab in labs: + print(lab.lab_session_owner) + + # print(total_cpu) + # print(total_mem) + # print(get_all()) return render(request, template, locals()) From 7f1399b71402c3d503a51f6b910c5c1aee153e10 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 1 Sep 2020 13:44:59 +0200 Subject: [PATCH 29/83] Labs uses helmresource model for creation/deletion --- components/studio/deployments/models.py | 2 +- components/studio/labs/models.py | 9 +-- components/studio/labs/views.py | 82 +------------------------ components/studio/models/models.py | 5 +- components/studio/projects/views.py | 4 +- components/studio/studio/settings.py | 19 ++++-- 6 files changed, 23 insertions(+), 98 deletions(-) diff --git a/components/studio/deployments/models.py b/components/studio/deployments/models.py index 4654d12e3..fa1c3cb41 100644 --- a/components/studio/deployments/models.py +++ b/components/studio/deployments/models.py @@ -12,7 +12,7 @@ class HelmResource(models.Model): name = models.CharField(max_length=512, unique=True) namespace = models.CharField(max_length=512) chart = models.CharField(max_length=512) - params = models.CharField(max_length=2048) + params = models.CharField(max_length=10000) username = models.CharField(max_length=512) status = models.CharField(max_length=20) created = models.DateTimeField(auto_now_add=True) diff --git a/components/studio/labs/models.py b/components/studio/labs/models.py index 7b3678610..c54c79e32 100644 --- a/components/studio/labs/models.py +++ b/components/studio/labs/models.py @@ -76,10 +76,6 @@ class Session(models.Model): flavor_slug = models.CharField(max_length=512) environment_slug = models.CharField(max_length=512) - # session_key = models.CharField(max_length=512) - # session_secret = models.CharField(max_length=512) - settings = models.TextField() - # chart = models.CharField(max_length=512) helm_repo = models.CharField(max_length=1024, null=True, blank=True) status = models.CharField(max_length=2, choices=STATUS, default=CREATED) @@ -106,13 +102,13 @@ def pre_save_labs(sender, instance, using, **kwargs): RELEASE_NAME = str(instance.slug) HOST = settings.DOMAIN URL = 'https://'+RELEASE_NAME+'.'+HOST - user = instance.lab_session_owner + user = instance.lab_session_owner.username client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) instance.keycloak_client_id = client_id parameters = {'release': RELEASE_NAME, - 'chart': instance.chart, + 'chart': 'lab', 'global.domain': settings.DOMAIN, 'project.name': instance.project.slug, 'gatekeeper.realm': settings.KC_REALM, @@ -164,7 +160,6 @@ def pre_save_labs(sender, instance, using, **kwargs): } parameters.update(prefs) - instance.settings = parameters helmchart = HelmResource(name=RELEASE_NAME, namespace='Default', diff --git a/components/studio/labs/views.py b/components/studio/labs/views.py index 3aee31643..652bf1903 100644 --- a/components/studio/labs/views.py +++ b/components/studio/labs/views.py @@ -41,83 +41,12 @@ def run(request, user, project): flavor_slug=flavor_slug, environment_slug=environment_slug, project=project, - lab_session_owner=user) + lab_session_owner=request.user) lab_instance.save() - # if flavor_slug: - # flavor = Flavor.objects.filter(slug=flavor_slug).first() - # else: - # flavor = Flavor.objects.all().first() - - # if environment_slug: - # environment = Environment.objects.filter(slug=environment_slug).first() - # else: - # environment = Environment.objects.filter.all().first() - - # print("dispatching with {} {}".format(flavor, name)) - # import base64 - # if name != '' and flavor is not None: - # # Default values here, because otherwise an old deployment can stop working - # # if the deployment configuration files are not compatible with the latest - # # studio image. - # ingress_secret_name = 'prod-ingress' - # try: - # ingress_secret_name = settings.LABS['ingress']['secretName'] - # except: - # pass - - # minio_keys = get_minio_keys(project) - # decrypted_key = minio_keys['project_key'] - # decrypted_secret = minio_keys['project_secret'] - - # settings_file = ProjectSerializer(project) - - # settings_file = JSONRenderer().render(settings_file.data) - # settings_file = settings_file.decode('utf-8') - - # settings_file = json.loads(settings_file) - # settings_file = yaml.dump(settings_file) - - # user_config_file = create_user_settings(user) - # user_config_file = yaml.dump(json.loads(user_config_file)) - - # prefs = {'namespace': settings.NAMESPACE, - # 'labs.resources.requests.cpu': str(flavor.cpu), - # 'labs.resources.limits.cpu': str(flavor.cpu), - # 'labs.resources.requests.memory': str(flavor.mem), - # 'labs.resources.limits.memory': str(flavor.mem), - # 'labs.resources.requests.gpu': str(flavor.gpu), - # 'labs.resources.limits.gpu': str(flavor.gpu), - # 'labs.gpu.enabled': str("true" if flavor.gpu else "false"), - # 'labs.image': environment.image, - # 'ingress.secretName': ingress_secret_name, - # # 'labs.setup': environment.setup, - # 'minio.access_key': decrypted_key, - # 'minio.secret_key': decrypted_secret, - # 'settings_file': settings_file, - # 'user_settings_file': user_config_file, - # 'project.slug': project.slug - # } - # session = Session.objects.create_session(name=name, project=project, lab_session_owner=request.user, - # chart='lab', settings=prefs) - # from .helpers import create_session_resources - - # print("trying to create resources") - # retval = create_session_resources(request, user, session, prefs, project) - # if retval: - # print("saving session!") - # project.save() - # session.save() - - # l = ProjectLog(project=project, module='LA', headline='Lab Session', - # description='A new Lab Session {name} has been created'.format(name=name)) - # l.save() return HttpResponseRedirect( reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) - # return HttpResponseRedirect( - # reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) - @login_required(login_url='/accounts/login') def delete(request, user, project, id): @@ -125,14 +54,7 @@ def delete(request, user, project, id): session = Session.objects.filter(Q(id=id), Q(project=project), Q(lab_session_owner=request.user)).first() if session: - # from .helpers import delete_session_resources - # delete_session_resources(session) - - # l = ProjectLog(project=project, module='LA', headline='Lab Session', - # description='Lab Session {name} has been removed'.format(name=session.name)) - # l.save() - - session.delete() + session.helmchart.delete() return HttpResponseRedirect( reverse('labs:index', kwargs={'user': request.user, 'project': str(project.slug)})) diff --git a/components/studio/models/models.py b/components/studio/models/models.py index 850a8cf2a..f3707867f 100644 --- a/components/studio/models/models.py +++ b/components/studio/models/models.py @@ -10,9 +10,10 @@ from projects.helpers import get_minio_keys from minio import Minio -VERSION_CLASS = import_string(settings.VERSION_BACKEND) + def compare_version(v1, v2): + VERSION_CLASS = import_string(settings.VERSION_BACKEND) v1obj = VERSION_CLASS(v1.version) v2obj = VERSION_CLASS(v2.version) if v1obj < v2obj: @@ -91,7 +92,7 @@ def __str__(self): @receiver(pre_save, sender=Model, dispatch_uid='model_pre_save_signal') def pre_save_model(sender, instance, using, **kwargs): # Load version backend - + VERSION_CLASS = import_string(settings.VERSION_BACKEND) # Set version release_type = instance.release_type # If version is not already set, create new release diff --git a/components/studio/projects/views.py b/components/studio/projects/views.py index bfba48516..c10f653b8 100644 --- a/components/studio/projects/views.py +++ b/components/studio/projects/views.py @@ -82,7 +82,7 @@ def change_environment(request, user, project_slug): if request.method == 'POST': environment_slug = request.POST.get('environment', '') - if environment_slug is not '': + if environment_slug != '': environment = Environment.objects.filter(slug=environment_slug).first() if environment: project.environment = environment @@ -104,7 +104,7 @@ def change_description(request, user, project_slug): if request.method == 'POST': description = request.POST.get('description', '') - if description is not '': + if description != '': project.description = description project.save() diff --git a/components/studio/studio/settings.py b/components/studio/studio/settings.py index bfdab9fef..c30a3223d 100644 --- a/components/studio/studio/settings.py +++ b/components/studio/studio/settings.py @@ -118,17 +118,24 @@ # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql', +# 'NAME': 'postgres', +# 'USER': 'postgres', +# 'PASSWORD': 'postgres', +# 'HOST': 'stack-studio-db', +# 'PORT': 5432, +# } +# } +# Dummy backend here to allow for creating migrations locally. DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'postgres', - 'USER': 'postgres', - 'PASSWORD': 'postgres', - 'HOST': 'stack-studio-db', - 'PORT': 5432, + 'ENGINE': 'django.db.backends.dummy', } } + # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators From f452c52287950e172f84c83bee3ed8df122194c6 Mon Sep 17 00:00:00 2001 From: Stefan Hellander Date: Tue, 1 Sep 2020 17:03:54 +0200 Subject: [PATCH 30/83] Basic monitoring overview --- .gitignore | 2 + .../studio/datasets/migrations/__init__.py | 0 .../migrations/0002_auto_20200901_1133.py | 18 ++++ .../migrations/0003_auto_20200901_1138.py | 18 ++++ components/studio/labs/helpers.py | 49 ---------- .../migrations/0002_auto_20200901_1117.py | 57 +++++++++++ .../0003_remove_session_settings.py | 17 ++++ components/studio/labs/models.py | 2 + components/studio/modules/project_auth.py | 11 +++ components/studio/monitor/helpers.py | 13 +-- .../studio/monitor/migrations/__init__.py | 0 .../monitor/templates/monitor_overview.html | 74 ++++++++++++-- components/studio/monitor/urls.py | 4 +- components/studio/monitor/views.py | 97 +++++++++++++++++-- .../studio/projects/templates/settings.html | 6 +- components/studio/projects/views.py | 3 + components/studio/templates/baseproject.html | 7 ++ 17 files changed, 307 insertions(+), 71 deletions(-) create mode 100644 components/studio/datasets/migrations/__init__.py create mode 100644 components/studio/deployments/migrations/0002_auto_20200901_1133.py create mode 100644 components/studio/deployments/migrations/0003_auto_20200901_1138.py create mode 100644 components/studio/labs/migrations/0002_auto_20200901_1117.py create mode 100644 components/studio/labs/migrations/0003_remove_session_settings.py create mode 100644 components/studio/modules/project_auth.py create mode 100644 components/studio/monitor/migrations/__init__.py diff --git a/.gitignore b/.gitignore index 342246687..e150f5e1c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +Pipfile +Pipfile.lock # PyInstaller # Usually these files are written by a python script from a template diff --git a/components/studio/datasets/migrations/__init__.py b/components/studio/datasets/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/deployments/migrations/0002_auto_20200901_1133.py b/components/studio/deployments/migrations/0002_auto_20200901_1133.py new file mode 100644 index 000000000..b3e769744 --- /dev/null +++ b/components/studio/deployments/migrations/0002_auto_20200901_1133.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-09-01 11:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('deployments', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='helmresource', + name='params', + field=models.CharField(max_length=4096), + ), + ] diff --git a/components/studio/deployments/migrations/0003_auto_20200901_1138.py b/components/studio/deployments/migrations/0003_auto_20200901_1138.py new file mode 100644 index 000000000..5169e79ba --- /dev/null +++ b/components/studio/deployments/migrations/0003_auto_20200901_1138.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-09-01 11:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('deployments', '0002_auto_20200901_1133'), + ] + + operations = [ + migrations.AlterField( + model_name='helmresource', + name='params', + field=models.CharField(max_length=10000), + ), + ] diff --git a/components/studio/labs/helpers.py b/components/studio/labs/helpers.py index b747e5f39..d063589e7 100644 --- a/components/studio/labs/helpers.py +++ b/components/studio/labs/helpers.py @@ -16,52 +16,3 @@ def create_user_settings(user): user_settings['keycloak_url'] = settings.KC_ADMIN_URL.replace('/auth', '') user_settings['studio_url'] = 'https://'+settings.DOMAIN return json.dumps(user_settings) - -# def create_session_resources(request, user, session, prefs, project): - -# print("1; going for the dispatch!") - -# HOST = settings.DOMAIN -# RELEASE_NAME = str(session.slug) - -# URL = 'https://'+RELEASE_NAME+'.'+HOST -# client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) - -# parameters = {'release': str(session.slug), -# 'chart': session.chart, -# 'global.domain': settings.DOMAIN, -# 'project.name': project.slug, -# 'gatekeeper.realm': settings.KC_REALM, -# 'gatekeeper.client_secret': client_secret, -# 'gatekeeper.client_id': client_id, -# 'gatekeeper.auth_endpoint': settings.OIDC_OP_REALM_AUTH -# } -# parameters.update(prefs) - -# url = settings.CHART_CONTROLLER_URL + '/deploy' - -# retval = r.get(url, parameters) -# print("CREATE_SESSION:helm chart creator returned {}".format(retval)) - -# if retval.status_code >= 200 or retval.status_code < 205: -# return True - -# return False - - -def delete_session_resources(session): - print("trying to delete {}".format(session.slug)) - parameters = {'release': str(session.slug)} - retval = r.get(settings.CHART_CONTROLLER_URL + '/delete', parameters) - - kc = keylib.keycloak_init() - keylib.keycloak_delete_client(kc, str(session.slug)) - - scope_id = keylib.keycloak_get_client_scope_id(kc, str(session.slug)+'-scope') - keylib.keycloak_delete_client_scope(kc, scope_id) - - if retval: - print('delete success!') - return True - print('delete failed!?') - return False diff --git a/components/studio/labs/migrations/0002_auto_20200901_1117.py b/components/studio/labs/migrations/0002_auto_20200901_1117.py new file mode 100644 index 000000000..e93cf651c --- /dev/null +++ b/components/studio/labs/migrations/0002_auto_20200901_1117.py @@ -0,0 +1,57 @@ +# Generated by Django 2.2.13 on 2020-09-01 11:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('deployments', '0001_initial'), + ('labs', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='session', + name='chart', + ), + migrations.RemoveField( + model_name='session', + name='session_key', + ), + migrations.RemoveField( + model_name='session', + name='session_secret', + ), + migrations.AddField( + model_name='session', + name='appname', + field=models.CharField(default='app-default', max_length=512), + preserve_default=False, + ), + migrations.AddField( + model_name='session', + name='environment_slug', + field=models.CharField(default='slug', max_length=512), + preserve_default=False, + ), + migrations.AddField( + model_name='session', + name='flavor_slug', + field=models.CharField(default='slug', max_length=512), + preserve_default=False, + ), + migrations.AddField( + model_name='session', + name='helmchart', + field=models.OneToOneField(default=0, on_delete=django.db.models.deletion.CASCADE, to='deployments.HelmResource'), + preserve_default=False, + ), + migrations.AddField( + model_name='session', + name='keycloak_client_id', + field=models.CharField(default='keycloak_client', max_length=512), + preserve_default=False, + ), + ] diff --git a/components/studio/labs/migrations/0003_remove_session_settings.py b/components/studio/labs/migrations/0003_remove_session_settings.py new file mode 100644 index 000000000..d732a826a --- /dev/null +++ b/components/studio/labs/migrations/0003_remove_session_settings.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.13 on 2020-09-01 11:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('labs', '0002_auto_20200901_1117'), + ] + + operations = [ + migrations.RemoveField( + model_name='session', + name='settings', + ), + ] diff --git a/components/studio/labs/models.py b/components/studio/labs/models.py index c54c79e32..bef7cc818 100644 --- a/components/studio/labs/models.py +++ b/components/studio/labs/models.py @@ -106,11 +106,13 @@ def pre_save_labs(sender, instance, using, **kwargs): client_id, client_secret = keylib.keycloak_setup_base_client(URL, RELEASE_NAME, user, ['owner'], ['owner']) instance.keycloak_client_id = client_id + instance.appname = '{}-{}-lab'.format(instance.slug, instance.project.slug) parameters = {'release': RELEASE_NAME, 'chart': 'lab', 'global.domain': settings.DOMAIN, 'project.name': instance.project.slug, + 'appname': instance.appname, 'gatekeeper.realm': settings.KC_REALM, 'gatekeeper.client_secret': client_secret, 'gatekeeper.client_id': client_id, diff --git a/components/studio/modules/project_auth.py b/components/studio/modules/project_auth.py new file mode 100644 index 000000000..b44fb153f --- /dev/null +++ b/components/studio/modules/project_auth.py @@ -0,0 +1,11 @@ +import modules.keycloak_lib as keylib + +def get_permissions(request, project, rules): + + user_roles = set(keylib.keycloak_get_user_roles(request, project, aud=project)) + + user_permissions = dict() + for rule in rules: + user_permissions[rule] = bool(set(rules[rule]) & user_roles) + + return user_permissions \ No newline at end of file diff --git a/components/studio/monitor/helpers.py b/components/studio/monitor/helpers.py index 5ac5f2bee..6525edde7 100644 --- a/components/studio/monitor/helpers.py +++ b/components/studio/monitor/helpers.py @@ -98,17 +98,18 @@ def get_labs_cpu_requests(project_slug): return num_cpus return 0 -def get_resource(project_slug, resource_type, q_type, mem_or_cpu): - query = 'sum(kube_pod_container_resource_'+q_type+'_'+mem_or_cpu+' * on(pod) group_left kube_pod_labels{label_project="'+project_slug+'", label_type="'+resource_type+'"})' - print(query) +def get_resource(project_slug, resource_type, q_type, mem_or_cpu, app_name=[]): + query = 'sum(kube_pod_container_resource_'+q_type+'_'+mem_or_cpu+' * on(pod) group_left kube_pod_labels{label_project="'+project_slug+'", label_type="'+resource_type+'"' + if app_name: + query += ', label_app="'+app_name+'"})' + else: + query += '})' response = r.get(settings.PROMETHEUS_SVC+'/api/v1/query', params={'query':query}) result = response.json()['data']['result'] if result: res = result[0]['value'][1] - # if mem_or_cpu == 'memory_bytes': - # res = "{:.2f}".format(float(res)/1e9*0.931323) return res - return 0 + return '0.0' def get_all(): query = 'kube_pod_container_resource_limits_memory_bytes * on(pod) group_left kube_pod_labels{label_type="lab", label_project="stochss-dev-tiz"}' diff --git a/components/studio/monitor/migrations/__init__.py b/components/studio/monitor/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/components/studio/monitor/templates/monitor_overview.html b/components/studio/monitor/templates/monitor_overview.html index 6aa91523d..51915cf7d 100644 --- a/components/studio/monitor/templates/monitor_overview.html +++ b/components/studio/monitor/templates/monitor_overview.html @@ -9,9 +9,10 @@

Monitor

+ {% if is_authorized %}

Project Quota

- +
@@ -34,7 +35,7 @@

Project Quota

Project Usage

-
+
@@ -63,22 +64,80 @@

Project Usage

- - + + - + - +
Memory Gi{{ resource_status.deployment.requests.memory_bytes }} Gi{{ resource_status.deployment.limits.memory_bytes }} Gi
Total CPU{{ total_cpu_req }} {{ total_cpu }}
Memory{{ total_mem_req }} {{ total_mem }} Gi
+ +

Lab Sessions

+ + + + + + + + + + + + + + {% for lab in lab_list %} + + + + + + + + + + {% endfor %} + +
OwnerFlavorCPU limitCPU requestMem limitMem request
{{ lab.0 }}{{ lab.1 }}{{ lab.2 }}{{ lab.3 }}{{ lab.4 }}{{ lab.5 }}Delete
+ +

Deployments

+ + + + + + + + + + + + + + + {% for dep in dep_list %} + + + + + + + + + + + {% endfor %} + +
OwnerModelVersionCPU limitCPU requestMem limitMem request
{{ dep.0 }}{{ dep.7 }}{{ dep.8 }}{{ dep.2 }}{{ dep.3 }}{{ dep.4 }}{{ dep.5 }}Delete