From 6b74dc63facc6276c703dc67679daabc427e0171 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Wed, 7 Sep 2022 21:20:18 +0530 Subject: [PATCH 1/6] Add /kill route for teuthology-kill cmd Request github API to verify user belongs in an "admin team". Currently, only members of an "admin team" of ceph can kill jobs. Exactly how it works in production currently. TODO: Users can kill their own runs Signed-off-by: Vallari Agrawal --- src/config.py | 3 +++ src/main.py | 3 ++- src/routes/kill.py | 25 +++++++++++++++++++++++++ src/schemas/kill.py | 14 ++++++++++++++ src/services/helpers.py | 22 +++++++++++++++++++--- src/services/kill.py | 23 +++++++++++++++++++++++ 6 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 src/routes/kill.py create mode 100644 src/schemas/kill.py create mode 100644 src/services/kill.py diff --git a/src/config.py b/src/config.py index 82b0c5e..2915b0d 100644 --- a/src/config.py +++ b/src/config.py @@ -6,6 +6,9 @@ class APISettings(BaseSettings): PADDLES_URL: str = "http://paddles:8080" + # TODO: team names need to be changed below when created + admin_team:str = "ceph" # ceph's github team with *sudo* access to sepia + teuth_team: str = "teuth" # ceph's github team with access to sepia class Config: env_file = ".env" diff --git a/src/main.py b/src/main.py index 42b47ef..8995d5b 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,7 @@ import logging from fastapi import FastAPI -from routes import suite +from routes import suite, kill log = logging.getLogger(__name__) app = FastAPI() @@ -12,3 +12,4 @@ def read_root(): return {"root": "success"} app.include_router(suite.router) +app.include_router(kill.router) diff --git a/src/routes/kill.py b/src/routes/kill.py new file mode 100644 index 0000000..60da8b6 --- /dev/null +++ b/src/routes/kill.py @@ -0,0 +1,25 @@ +from fastapi import APIRouter, HTTPException +from services.kill import run +from schemas.kill import KillArgs + +import logging + +log = logging.getLogger(__name__) + +router = APIRouter( + prefix="/kill", + tags=["kill"], +) + +@router.post("/", status_code=200) +def create_run(args: KillArgs, access_token: str, logs: bool = False): + """ + access_token should be of format ` ` + Example: "bearer " + """ + try: + args = args.dict(by_alias=True) + results = run(args, logs, access_token) + return results + except Exception as exc: + raise HTTPException(status_code=404, detail=repr(exc)) diff --git a/src/schemas/kill.py b/src/schemas/kill.py new file mode 100644 index 0000000..69941ad --- /dev/null +++ b/src/schemas/kill.py @@ -0,0 +1,14 @@ +from typing import Union, Optional +from pydantic import Field + +from .base import BaseArgs + + +class KillArgs(BaseArgs): + owner: Union[str, None] = Field(default=None, alias="--owner") + run: Union[str, None] = Field(default=None, alias="--run") + preserve_queue: Union[bool, None] = Field(default=None, alias="--preserve-queue") + job: Union[int, None] = Field(default=None, alias="--job") + jobspec: Union[str, None] = Field(default=None, alias="--jobspec") + machine_type: Union[str, None] = Field(default='default', alias="--machine-type") + archive: Union[str, None] = Field(default=None, alias="--archive") diff --git a/src/services/helpers.py b/src/services/helpers.py index a2ae497..e579be6 100644 --- a/src/services/helpers.py +++ b/src/services/helpers.py @@ -1,6 +1,10 @@ -import uuid, os -from multiprocessing import Process import teuthology +import uuid, os, requests, logging # Note: import requests after teuthology +from multiprocessing import Process +from config import settings + + +logger = logging.getLogger(__name__) def logs_run(func, args): """ @@ -28,4 +32,16 @@ def _execute_with_logs(func, args, log_file): and then execute the command function. """ teuthology.setup_log_file(log_file) - func(args) \ No newline at end of file + func(args) + +def github_user_details(access_token: str, username: str): + team_name = settings.admin_team + url = f"https://api.github.com/orgs/ceph/teams/{team_name}/memberships/{username}" + headers = {"Authorization": access_token} + + resp = requests.get(url=url, headers=headers) + logger.info(resp.json()) + + if resp.status_code == 200: + return True + return resp.json() diff --git a/src/services/kill.py b/src/services/kill.py new file mode 100644 index 0000000..bc115dd --- /dev/null +++ b/src/services/kill.py @@ -0,0 +1,23 @@ +import teuthology.kill +from services.helpers import logs_run, github_user_details +import logging, requests # Note: import requests after teuthology + + +log = logging.getLogger(__name__) + + +def run(args, send_logs: bool, access_token: str): + """ + Kill running teuthology jobs. + """ + try: + if not access_token: + raise Exception("No access token. User not authenticated yet to perform this operation.") + authenticate = github_user_details(access_token, args["--user"]) + if authenticate == True: + logs = logs_run(teuthology.kill.main, args) + return { "logs": logs } + raise Exception(f"Github authentication for user '{args['--user']}' failed.") + except Exception as exc: + log.error("teuthology.suite.main failed with the error: " + repr(exc)) + raise \ No newline at end of file From c4ccdeafa030eb42f729f0ba92ca4d9ffa73598f Mon Sep 17 00:00:00 2001 From: Kamoltat Sirivadhna Date: Mon, 20 Feb 2023 15:45:43 -0500 Subject: [PATCH 2/6] Auth: init github login & logout Created login and logout route storing the token in session using `starlette.middleware.sessions`. Only github users who are a part of Ceph organization can login successfully. Moreover, we have included exceptions for all routes. Signed-off-by: Kamoltat Sirivadhna --- Dockerfile | 2 +- requirements.txt | 2 ++ src/main.py | 20 ++++++++++++--- src/routes/login.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ src/routes/logout.py | 17 +++++++++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/routes/login.py create mode 100644 src/routes/logout.py diff --git a/Dockerfile b/Dockerfile index db4abde..a8cfcd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && \ apt-get clean all COPY .teuthology.yaml /root - +COPY .env* /teuthology_api/ WORKDIR /teuthology_api COPY requirements.txt /teuthology_api/ RUN pip3 install -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 6b12856..5a33b97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ fastapi uvicorn[standard] gunicorn +httpx +itsdangerous # Temporarily, using teuthology without monkey patching the thread git+https://github.com/VallariAg/teuthology@teuth-api#egg=teuthology[test] # Original: git+https://github.com/ceph/teuthology#egg=teuthology[test] diff --git a/src/main.py b/src/main.py index 8995d5b..bee1ee8 100644 --- a/src/main.py +++ b/src/main.py @@ -1,15 +1,27 @@ +from fastapi import FastAPI, Request +from starlette.middleware.sessions import SessionMiddleware +from routes import suite, kill, login, logout +from dotenv import load_dotenv import logging -from fastapi import FastAPI +import os -from routes import suite, kill +load_dotenv() + +SESSION_SECRET_KEY = os.getenv('SESSION_SECRET_KEY') log = logging.getLogger(__name__) app = FastAPI() @app.get("/") -def read_root(): - return {"root": "success"} +def read_root(request: Request): + return { + "root": "success", + "session": request.session.get('user', None) + } +app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET_KEY) app.include_router(suite.router) app.include_router(kill.router) +app.include_router(login.router) +app.include_router(logout.router) diff --git a/src/routes/login.py b/src/routes/login.py new file mode 100644 index 0000000..fc9ccf4 --- /dev/null +++ b/src/routes/login.py @@ -0,0 +1,60 @@ +from fastapi import APIRouter, HTTPException, Request, Depends +from starlette.responses import RedirectResponse, HTMLResponse +from dotenv import load_dotenv +import os +import httpx +import logging + +load_dotenv() + +GH_CLIENT_ID = os.getenv('GH_CLIENT_ID') +GH_CLIENT_SECRET = os.getenv('GH_CLIENT_SECRET') +GH_AUTHORIZATION_BASE_URL = os.getenv('GH_AUTHORIZATION_BASE_URL') +GH_TOKEN_URL = os.getenv('GH_TOKEN_URL') +GH_FETCH_MEMBERSHIP_URL = os.getenv('GH_FETCH_MEMBERSHIP_URL') + +log = logging.getLogger(__name__) +router = APIRouter( + prefix="/login", + tags=["login"], + responses={404: {"description": "Not found"}}, +) + +@router.get("/", status_code=200) +async def github_login(request: Request): + scope = 'read:org' + return RedirectResponse(f'{GH_AUTHORIZATION_BASE_URL}?client_id={GH_CLIENT_ID}&scope={scope}', status_code=302) + +@router.get("/callback", status_code=200) +async def handle_callback(code: str, request: Request): + params = { + 'client_id': GH_CLIENT_ID, + 'client_secret': GH_CLIENT_SECRET, + 'code': code + } + headers = {'Accept' : 'application/json'} + async with httpx.AsyncClient() as client: + response_token = await client.post(url=GH_TOKEN_URL, params=params, headers=headers) + log.info(response_token.json()) + response_token_dic = dict(response_token.json()) + token = response_token_dic.get('access_token') + if response_token_dic.get('error') or not token: + raise HTTPException(status_code=401, detail="The code passed is incorrect or expired.") + log.info(token) + headers = {'Authorization': 'token ' + token} + response_org = await client.get(url=GH_FETCH_MEMBERSHIP_URL, headers=headers) + log.info(response_org.json()) + if response_org.status_code == 404: + raise HTTPException(status_code=404, detail="User is not part of the Ceph Organization, please contact ") + elif response_org.status_code == 403: + raise HTTPException(status_code=403, detail="The application doesn't have permission to view github org") + response_org_dic = dict(response_org.json()) + data = { + "id": response_org_dic.get('user').get('id'), + "name": response_org_dic.get('user').get('login'), + "state": response_org_dic.get('state'), + "role": response_org_dic.get('role'), + "access_token": token, + } + request.session['user'] = data + return RedirectResponse(url='/') \ No newline at end of file diff --git a/src/routes/logout.py b/src/routes/logout.py new file mode 100644 index 0000000..056b894 --- /dev/null +++ b/src/routes/logout.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter, Request +from starlette.responses import RedirectResponse +import logging + +log = logging.getLogger(__name__) +router = APIRouter( + prefix="/logout", + tags=["logout"], + responses={404: {"description": "Not found"}}, +) + +@router.get('/', status_code=200) +def logout(request: Request): + request.session.pop('user', None) + return { + "logout": "success" + } \ No newline at end of file From 34c29379bc8a94aab7b1ed05452f11e107f2eca1 Mon Sep 17 00:00:00 2001 From: Kamoltat Sirivadhna Date: Thu, 23 Feb 2023 16:01:14 -0500 Subject: [PATCH 3/6] auth: Added auth to teuth-suite and teuth-kill User must be logged in to use teuth-suite or teuth-kill functions. User cannot kill a job that has been scheduled by other users or not through teuthology-api. Furthermore, added more exception handling to certain helper functions. Signed-off-by: Kamoltat Sirivadhna --- src/routes/kill.py | 26 +++++++++--------- src/routes/login.py | 4 +-- src/routes/logout.py | 16 +++++++---- src/routes/suite.py | 17 ++++++------ src/services/helpers.py | 61 +++++++++++++++++++++++++++++++++-------- src/services/kill.py | 35 +++++++++++++++++------ src/services/suite.py | 31 ++++++++------------- 7 files changed, 123 insertions(+), 67 deletions(-) diff --git a/src/routes/kill.py b/src/routes/kill.py index 60da8b6..75185a9 100644 --- a/src/routes/kill.py +++ b/src/routes/kill.py @@ -1,7 +1,7 @@ -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Depends, Request from services.kill import run +from services.helpers import get_token from schemas.kill import KillArgs - import logging log = logging.getLogger(__name__) @@ -12,14 +12,14 @@ ) @router.post("/", status_code=200) -def create_run(args: KillArgs, access_token: str, logs: bool = False): - """ - access_token should be of format ` ` - Example: "bearer " - """ - try: - args = args.dict(by_alias=True) - results = run(args, logs, access_token) - return results - except Exception as exc: - raise HTTPException(status_code=404, detail=repr(exc)) +def create_run( + request: Request, + args: KillArgs, + logs: bool = False, + access_token: str = Depends(get_token), +): + # Note: I needed to put `request` before `args` + # or else it will SyntaxError: non-dafault + # argument follows default argument error. + args = args.dict(by_alias=True) + return run(args, logs, access_token, request) diff --git a/src/routes/login.py b/src/routes/login.py index fc9ccf4..d64070a 100644 --- a/src/routes/login.py +++ b/src/routes/login.py @@ -50,8 +50,8 @@ async def handle_callback(code: str, request: Request): raise HTTPException(status_code=403, detail="The application doesn't have permission to view github org") response_org_dic = dict(response_org.json()) data = { - "id": response_org_dic.get('user').get('id'), - "name": response_org_dic.get('user').get('login'), + "id": response_org_dic.get('user', {}).get('id'), + "username": response_org_dic.get('user', {}).get('login'), "state": response_org_dic.get('state'), "role": response_org_dic.get('role'), "access_token": token, diff --git a/src/routes/logout.py b/src/routes/logout.py index 056b894..6690e8e 100644 --- a/src/routes/logout.py +++ b/src/routes/logout.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Request +from fastapi import APIRouter, HTTPException, Request from starlette.responses import RedirectResponse import logging @@ -11,7 +11,13 @@ @router.get('/', status_code=200) def logout(request: Request): - request.session.pop('user', None) - return { - "logout": "success" - } \ No newline at end of file + user = request.session.get('user') + if user: + request.session.pop('user', None) + return {"logout": "success"} + else: + log.warning("No session found, probably already logged out.") + raise HTTPException( + status_code=204, + detail="No session found, probably already logged out." + ) diff --git a/src/routes/suite.py b/src/routes/suite.py index ec0ff78..290a3af 100644 --- a/src/routes/suite.py +++ b/src/routes/suite.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Depends from services.suite import run +from services.helpers import get_token from schemas.suite import SuiteArgs import logging @@ -12,10 +13,10 @@ ) @router.post("/", status_code=200) -def create_run(args: SuiteArgs, dry_run: bool = False, logs: bool = False): - try: - args = args.dict(by_alias=True) - results = run(args, dry_run, logs) - return results - except Exception as exc: - raise HTTPException(status_code=404, detail=repr(exc)) +def create_run(args: SuiteArgs, + access_token: str = Depends(get_token), + dry_run: bool = False, + logs: bool = False +): + args = args.dict(by_alias=True) + return run(args, dry_run, logs, access_token) diff --git a/src/services/helpers.py b/src/services/helpers.py index e579be6..6e18abf 100644 --- a/src/services/helpers.py +++ b/src/services/helpers.py @@ -1,10 +1,13 @@ -import teuthology -import uuid, os, requests, logging # Note: import requests after teuthology from multiprocessing import Process from config import settings +from fastapi import HTTPException, Request +import teuthology +import uuid, os, requests, logging # Note: import requests after teuthology +from requests.exceptions import HTTPError +PADDLES_URL = settings.PADDLES_URL -logger = logging.getLogger(__name__) +log = logging.getLogger(__name__) def logs_run(func, args): """ @@ -34,14 +37,48 @@ def _execute_with_logs(func, args, log_file): teuthology.setup_log_file(log_file) func(args) -def github_user_details(access_token: str, username: str): - team_name = settings.admin_team - url = f"https://api.github.com/orgs/ceph/teams/{team_name}/memberships/{username}" - headers = {"Authorization": access_token} +def get_run_details(run_name: str): + """ + Queries paddles to look if run is created. + """ + url = f'{PADDLES_URL}/runs/{run_name}/' + try: + run_info = requests.get(url) + run_info.raise_for_status() + return run_info.json() + except HTTPError as http_err: + log.error(http_err) + raise HTTPException( + status_code=http_err.response.status_code, + detail=repr(http_err) + ) + except Exception as err: + log.error(err) + raise HTTPException( + status_code=500, + detail=repr(err) + ) - resp = requests.get(url=url, headers=headers) - logger.info(resp.json()) +def get_username(request: Request): + username = request.session.get('user', {}).get('username') + if username: + return username + else: + log.error("username empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) - if resp.status_code == 200: - return True - return resp.json() +def get_token(request: Request): + token = request.session.get('user', {}).get('access_token') + if token: + return {"access_token": token, "token_type": "bearer"} + else: + log.error("access_token empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) \ No newline at end of file diff --git a/src/services/kill.py b/src/services/kill.py index bc115dd..7810b07 100644 --- a/src/services/kill.py +++ b/src/services/kill.py @@ -1,23 +1,42 @@ +from fastapi import HTTPException, Request +from services.helpers import logs_run, get_username, get_run_details import teuthology.kill -from services.helpers import logs_run, github_user_details import logging, requests # Note: import requests after teuthology log = logging.getLogger(__name__) -def run(args, send_logs: bool, access_token: str): +def run(args, send_logs: bool, access_token: str, request: Request): """ Kill running teuthology jobs. """ + if not access_token: + log.error("access_token empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) + username = get_username(request) + run_name = args.get("--run") + if run_name: + run_details = get_run_details(run_name) + run_username = run_details.get("user") + else: + log.error("teuthology-kill is missing --run") + raise HTTPException(status_code=400, detail="--run is a required argument") + #TODO if user has admin priviledge, then they can kill any run/job. + if run_username != username: + log.error("%s doesn't have permission to kill a job scheduled by: %s" % (username, run_username)) + raise HTTPException(status_code=401, detail="You don't have permission to kill this run/job") try: - if not access_token: - raise Exception("No access token. User not authenticated yet to perform this operation.") - authenticate = github_user_details(access_token, args["--user"]) - if authenticate == True: + if send_logs: logs = logs_run(teuthology.kill.main, args) return { "logs": logs } - raise Exception(f"Github authentication for user '{args['--user']}' failed.") + else: + teuthology.kill.main(args) + return {"kill": "success"} except Exception as exc: log.error("teuthology.suite.main failed with the error: " + repr(exc)) - raise \ No newline at end of file + raise HTTPException(status_code=500, detail=repr(exc)) \ No newline at end of file diff --git a/src/services/suite.py b/src/services/suite.py index ed07218..88fd084 100644 --- a/src/services/suite.py +++ b/src/services/suite.py @@ -1,20 +1,24 @@ -import teuthology.suite -from services.helpers import logs_run -import logging, requests # Note: import requests after teuthology +from fastapi import HTTPException +from services.helpers import logs_run, get_run_details from datetime import datetime - from config import settings - -PADDLES_URL = settings.PADDLES_URL +import teuthology.suite +import logging log = logging.getLogger(__name__) -def run(args, dry_run: bool, send_logs: bool): +def run(args, dry_run: bool, send_logs: bool, access_token: str): """ Schedule a suite. :returns: Run details (dict) and logs (list). """ + if not access_token: + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) try: args["--timestamp"] = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') if dry_run: @@ -42,18 +46,7 @@ def run(args, dry_run: bool, send_logs: bool): return { "run": run_details, "logs": logs } except Exception as exc: log.error("teuthology.suite.main failed with the error: " + repr(exc)) - raise - -def get_run_details(run_name): - """ - Queries paddles to look if run is created. - """ - try: - url = f'{PADDLES_URL}/runs/{run_name}/' - run_info = requests.get(url).json() - return run_info - except: - raise RuntimeError(f"Unable to find run `{run_name}` in paddles database.") + raise HTTPException(status_code=500, detail=repr(exc)) def make_run_name(run): """ From 6da152fa5937061a1163805d9dd7654f4b685e5a Mon Sep 17 00:00:00 2001 From: Kamoltat Sirivadhna Date: Wed, 1 Mar 2023 11:54:48 -0500 Subject: [PATCH 4/6] Refractor to conform code to pep8 python format Signed-off-by: Kamoltat Sirivadhna --- src/config.py | 18 ++++++++---- src/main.py | 19 ++++++++++-- src/routes/kill.py | 14 +++++---- src/routes/login.py | 49 ++++++++++++++++++++++--------- src/routes/logout.py | 17 ++++++----- src/schemas/base.py | 6 +++- src/schemas/kill.py | 6 +++- src/schemas/schedule.py | 6 +++- src/schemas/suite.py | 65 ++++++++++++++++++++++++++++------------- src/services/helpers.py | 54 +++++++++++++++++++++------------- src/services/kill.py | 22 +++++++------- src/services/suite.py | 31 ++++++++++---------- 12 files changed, 201 insertions(+), 106 deletions(-) diff --git a/src/config.py b/src/config.py index 2915b0d..ca76a18 100644 --- a/src/config.py +++ b/src/config.py @@ -1,23 +1,31 @@ from functools import lru_cache -from typing import List -import os from pydantic import BaseSettings class APISettings(BaseSettings): + """ + Class for API settings. + """ PADDLES_URL: str = "http://paddles:8080" # TODO: team names need to be changed below when created - admin_team:str = "ceph" # ceph's github team with *sudo* access to sepia - teuth_team: str = "teuth" # ceph's github team with access to sepia + admin_team: str = "ceph" # ceph's github team with *sudo* access to sepia + teuth_team: str = "teuth" # ceph's github team with access to sepia class Config: + """ + Class for Config. + """ + # pylint: disable=too-few-public-methods env_file = ".env" env_file_encoding = 'utf-8' @lru_cache() def get_api_settings() -> APISettings: + """ + Returns the API settings. + """ return APISettings() # reads variables from environment -settings = get_api_settings() \ No newline at end of file +settings = get_api_settings() diff --git a/src/main.py b/src/main.py index bee1ee8..0908d91 100644 --- a/src/main.py +++ b/src/main.py @@ -1,25 +1,38 @@ +import logging +import os from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware from starlette.middleware.sessions import SessionMiddleware from routes import suite, kill, login, logout from dotenv import load_dotenv -import logging -import os load_dotenv() +DEPLOYMENT = os.getenv('DEPLOYMENT') SESSION_SECRET_KEY = os.getenv('SESSION_SECRET_KEY') log = logging.getLogger(__name__) app = FastAPI() - @app.get("/") def read_root(request: Request): + """ + GET route for root. + """ return { "root": "success", "session": request.session.get('user', None) } +if DEPLOYMENT == 'development': + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET_KEY) app.include_router(suite.router) app.include_router(kill.router) diff --git a/src/routes/kill.py b/src/routes/kill.py index 75185a9..3a2d070 100644 --- a/src/routes/kill.py +++ b/src/routes/kill.py @@ -1,8 +1,8 @@ -from fastapi import APIRouter, HTTPException, Depends, Request +import logging +from fastapi import APIRouter, Depends, Request from services.kill import run from services.helpers import get_token from schemas.kill import KillArgs -import logging log = logging.getLogger(__name__) @@ -18,8 +18,12 @@ def create_run( logs: bool = False, access_token: str = Depends(get_token), ): - # Note: I needed to put `request` before `args` - # or else it will SyntaxError: non-dafault - # argument follows default argument error. + """ + POST route for killing a run or a job. + + Note: I needed to put `request` before `args` + or else it will SyntaxError: non-dafault + argument follows default argument error. + """ args = args.dict(by_alias=True) return run(args, logs, access_token, request) diff --git a/src/routes/login.py b/src/routes/login.py index d64070a..872c4e8 100644 --- a/src/routes/login.py +++ b/src/routes/login.py @@ -1,9 +1,9 @@ -from fastapi import APIRouter, HTTPException, Request, Depends -from starlette.responses import RedirectResponse, HTMLResponse -from dotenv import load_dotenv +import logging import os +from fastapi import APIRouter, HTTPException, Request +from starlette.responses import RedirectResponse +from dotenv import load_dotenv import httpx -import logging load_dotenv() @@ -20,34 +20,55 @@ responses={404: {"description": "Not found"}}, ) + @router.get("/", status_code=200) -async def github_login(request: Request): +async def github_login(): + """ + GET route for /login, (If first time) will redirect to github login page + where you should authorize the app to gain access. + """ scope = 'read:org' - return RedirectResponse(f'{GH_AUTHORIZATION_BASE_URL}?client_id={GH_CLIENT_ID}&scope={scope}', status_code=302) + return RedirectResponse( + f'{GH_AUTHORIZATION_BASE_URL}?client_id={GH_CLIENT_ID}&scope={scope}', + status_code=302 + ) @router.get("/callback", status_code=200) async def handle_callback(code: str, request: Request): + """ + Call back route after user login & authorize the app + for access. + """ params = { 'client_id': GH_CLIENT_ID, 'client_secret': GH_CLIENT_SECRET, 'code': code } - headers = {'Accept' : 'application/json'} + headers = {'Accept': 'application/json'} async with httpx.AsyncClient() as client: response_token = await client.post(url=GH_TOKEN_URL, params=params, headers=headers) log.info(response_token.json()) response_token_dic = dict(response_token.json()) - token = response_token_dic.get('access_token') + token = response_token_dic.get('access_token') if response_token_dic.get('error') or not token: - raise HTTPException(status_code=401, detail="The code passed is incorrect or expired.") - log.info(token) + log.error("The code is incorrect or expired.") + raise HTTPException( + status_code=401, detail="The code is incorrect or expired.") headers = {'Authorization': 'token ' + token} response_org = await client.get(url=GH_FETCH_MEMBERSHIP_URL, headers=headers) log.info(response_org.json()) if response_org.status_code == 404: - raise HTTPException(status_code=404, detail="User is not part of the Ceph Organization, please contact ") - elif response_org.status_code == 403: - raise HTTPException(status_code=403, detail="The application doesn't have permission to view github org") + log.error("User is not part of the Ceph Organization") + raise HTTPException( + status_code=404, + detail="User is not part of the Ceph Organization, please contact " + ) + if response_org.status_code == 403: + log.error("The application doesn't have permission to view github org") + raise HTTPException( + status_code=403, + detail="The application doesn't have permission to view github org" + ) response_org_dic = dict(response_org.json()) data = { "id": response_org_dic.get('user', {}).get('id'), @@ -57,4 +78,4 @@ async def handle_callback(code: str, request: Request): "access_token": token, } request.session['user'] = data - return RedirectResponse(url='/') \ No newline at end of file + return RedirectResponse(url='/') diff --git a/src/routes/logout.py b/src/routes/logout.py index 6690e8e..7ccc509 100644 --- a/src/routes/logout.py +++ b/src/routes/logout.py @@ -1,6 +1,5 @@ -from fastapi import APIRouter, HTTPException, Request -from starlette.responses import RedirectResponse import logging +from fastapi import APIRouter, HTTPException, Request log = logging.getLogger(__name__) router = APIRouter( @@ -11,13 +10,15 @@ @router.get('/', status_code=200) def logout(request: Request): + """ + GET route for logging out. + """ user = request.session.get('user') if user: request.session.pop('user', None) return {"logout": "success"} - else: - log.warning("No session found, probably already logged out.") - raise HTTPException( - status_code=204, - detail="No session found, probably already logged out." - ) + log.warning("No session found, probably already logged out.") + raise HTTPException( + status_code=204, + detail="No session found, probably already logged out." + ) diff --git a/src/schemas/base.py b/src/schemas/base.py index 81581c6..5a624fa 100644 --- a/src/schemas/base.py +++ b/src/schemas/base.py @@ -1,8 +1,12 @@ -from typing import Union, Optional +from typing import Union from pydantic import BaseModel, Field class BaseArgs(BaseModel): + # pylint: disable=too-few-public-methods + """ + Class for Base Args. + """ dry_run: Union[bool, None] = Field(default=False, alias="--dry-run") non_interactive: Union[bool, None] = Field(default=False, alias="--non-interactive") verbose: Union[int, None] = Field(default=1, alias="--verbose") diff --git a/src/schemas/kill.py b/src/schemas/kill.py index 69941ad..1587b57 100644 --- a/src/schemas/kill.py +++ b/src/schemas/kill.py @@ -1,10 +1,14 @@ -from typing import Union, Optional +from typing import Union from pydantic import Field from .base import BaseArgs class KillArgs(BaseArgs): + # pylint: disable=too-few-public-methods + """ + Class for KillArgs. + """ owner: Union[str, None] = Field(default=None, alias="--owner") run: Union[str, None] = Field(default=None, alias="--run") preserve_queue: Union[bool, None] = Field(default=None, alias="--preserve-queue") diff --git a/src/schemas/schedule.py b/src/schemas/schedule.py index e9ddcdb..ffb0c46 100644 --- a/src/schemas/schedule.py +++ b/src/schemas/schedule.py @@ -1,10 +1,14 @@ -from typing import Union, Optional +from typing import Union from pydantic import Field from .base import BaseArgs class SchedulerArgs(BaseArgs): + # pylint: disable=too-few-public-methods + """ + Class for SchedulerArgs. + """ owner: Union[str, None] = Field(default=None, alias="--owner") num: Union[str, None] = Field(default='1', alias="--num") priority: Union[str, None] = Field(default='70', alias="--priority") diff --git a/src/schemas/suite.py b/src/schemas/suite.py index f1d24e0..6e017f8 100644 --- a/src/schemas/suite.py +++ b/src/schemas/suite.py @@ -1,51 +1,74 @@ -from typing import Union, Optional +from typing import Union from pydantic import Field from .base import BaseArgs class SuiteArgs(BaseArgs): - # Standard arguments + # pylint: disable=too-few-public-methods + """ + Standard arguments + """ arch: Union[str, None] = Field(default=None, alias="--arch") ceph: Union[str, None] = Field(default='main', alias="--ceph") - ceph_repo: Union[str, None] = Field(default='https://github.com/ceph/ceph-ci.git', alias="--ceph-repo") + ceph_repo: Union[str, None] = Field( + default='https://github.com/ceph/ceph-ci.git', alias="--ceph-repo") distro: Union[str, None] = Field(default=None, alias="--distro") - distro_version: Union[str, None] = Field(default=None, alias="--distro-version") + distro_version: Union[str, None] = Field( + default=None, alias="--distro-version") email: Union[str, None] = Field(default=None, alias="--email") flavor: Union[str, None] = Field(default='default', alias="--flavor") kernel: Union[str, None] = Field(default='distro', alias="--kernel") - machine_type: Union[str, None] = Field(default='smithi', alias="--machine-type") + machine_type: Union[str, None] = Field( + default='smithi', alias="--machine-type") newest: Union[str, None] = Field(default='0', alias="--newest") - rerun_status: Union[bool, None] = Field(default=False, alias="--rerun-status.") - rerun_statuses: Union[str, None] = Field(default='fail,dead', alias="--rerun-statuses") + rerun_status: Union[bool, None] = Field( + default=False, alias="--rerun-status.") + rerun_statuses: Union[str, None] = Field( + default='fail,dead', alias="--rerun-statuses") sha1: Union[str, None] = Field(default=None, alias="--sha1") - sleep_before_teardown: Union[str, None] = Field(default='0', alias="--sleep-before-teardown") + sleep_before_teardown: Union[str, None] = Field( + default='0', alias="--sleep-before-teardown") suite: str = Field(alias="--suite") - suite_branch: Union[str, None] = Field(default=None, alias="--suite-branch") + suite_branch: Union[str, None] = Field( + default=None, alias="--suite-branch") suite_dir: Union[str, None] = Field(default=None, alias="--suite-dir") - suite_relpath: Union[str, None] = Field(default='qa', alias="--suite-relpath") - suite_repo: Union[str, None] = Field(default='https://github.com/ceph/ceph-ci.git', alias="--suite_repo") - teuthology_branch: Union[str, None] = Field(default='main', alias="--teuthology-branch") - validate_sha1: Union[str, None] = Field(default='true', alias="--validate-sha1") + suite_relpath: Union[str, None] = Field( + default='qa', alias="--suite-relpath") + suite_repo: Union[str, None] = Field( + default='https://github.com/ceph/ceph-ci.git', alias="--suite_repo") + teuthology_branch: Union[str, None] = Field( + default='main', alias="--teuthology-branch") + validate_sha1: Union[str, None] = Field( + default='true', alias="--validate-sha1") wait: Union[bool, None] = Field(default=False, alias="--wait") config_yaml: Union[list, None] = Field(default=[], alias="") - # Scheduler arguments + """ + Scheduler arguments + """ owner: Union[str, None] = Field(default=None, alias="--owner") num: Union[str, None] = Field(default='1', alias="--num") priority: Union[str, None] = Field(default='70', alias="--priority") - queue_backend: Union[str, None] = Field(default=None, alias="--queue-backend") + queue_backend: Union[str, None] = Field( + default=None, alias="--queue-backend") rerun: Union[str, None] = Field(default=None, alias="--rerun") seed: Union[str, None] = Field(default='-1', alias="--seed") - force_priority: Union[bool, None] = Field(default=False, alias="--force-priority") - no_nested_subset: Union[bool, None] = Field(default=False, alias="--no-nested-subset") - job_threshold: Union[str, None] = Field(default='500', alias="--job-threshold") - archive_upload: Union[str, None] = Field(default=None, alias="--archive-upload") - archive_upload_url: Union[str, None] = Field(default=None, alias="--archive-upload-url") + force_priority: Union[bool, None] = Field( + default=False, alias="--force-priority") + no_nested_subset: Union[bool, None] = Field( + default=False, alias="--no-nested-subset") + job_threshold: Union[str, None] = Field( + default='500', alias="--job-threshold") + archive_upload: Union[str, None] = Field( + default=None, alias="--archive-upload") + archive_upload_url: Union[str, None] = Field( + default=None, alias="--archive-upload-url") throttle: Union[str, None] = Field(default=None, alias="--throttle") filter: Union[str, None] = Field(default=None, alias="--filter") filter_out: Union[str, None] = Field(default=None, alias="--filter-out") filter_all: Union[str, None] = Field(default=None, alias="--filter-all") - filter_fragments: Union[str, None] = Field(default='false', alias="--filter-fragments") + filter_fragments: Union[str, None] = Field( + default='false', alias="--filter-fragments") subset: Union[str, None] = Field(default=None, alias="--subset") timeout: Union[str, None] = Field(default='43200', alias="--timeout") rocketchat: Union[str, None] = Field(default=None, alias="--rocketchat") diff --git a/src/services/helpers.py b/src/services/helpers.py index 6e18abf..a4e0efb 100644 --- a/src/services/helpers.py +++ b/src/services/helpers.py @@ -1,34 +1,39 @@ from multiprocessing import Process +import logging +import os +import uuid from config import settings from fastapi import HTTPException, Request -import teuthology -import uuid, os, requests, logging # Note: import requests after teuthology from requests.exceptions import HTTPError +import teuthology +import requests # Note: import requests after teuthology PADDLES_URL = settings.PADDLES_URL log = logging.getLogger(__name__) + def logs_run(func, args): """ Run the command function in a seperate process (to isolate logs), and return logs printed during the execution of the function. """ - id = str(uuid.uuid4()) - log_file = f'/tmp/{id}.log' + _id = str(uuid.uuid4()) + log_file = f'/archive_dir/{_id}.log' teuthology_process = Process( - target=_execute_with_logs, args=(func, args, log_file,)) + target=_execute_with_logs, args=(func, args, log_file)) teuthology_process.start() teuthology_process.join() logs = "" - with open(log_file) as f: - logs = f.readlines() - if os.path.isfile(log_file): + with open(log_file, encoding="utf-8") as file: + logs = file.readlines() + if os.path.isfile(log_file): os.remove(log_file) return logs + def _execute_with_logs(func, args, log_file): """ To store logs, set a new FileHandler for teuthology root logger @@ -37,6 +42,7 @@ def _execute_with_logs(func, args, log_file): teuthology.setup_log_file(log_file) func(args) + def get_run_details(run_name: str): """ Queries paddles to look if run is created. @@ -51,34 +57,40 @@ def get_run_details(run_name: str): raise HTTPException( status_code=http_err.response.status_code, detail=repr(http_err) - ) + ) from http_err except Exception as err: log.error(err) raise HTTPException( status_code=500, detail=repr(err) - ) + ) from err + def get_username(request: Request): + """ + Get username from request.session + """ username = request.session.get('user', {}).get('username') if username: return username - else: - log.error("username empty, user probably is not logged in.") - raise HTTPException( - status_code=401, - detail="You need to be logged in", - headers={"WWW-Authenticate": "Bearer"}, - ) + log.error("username empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) + def get_token(request: Request): + """ + Get access token from request.session + """ token = request.session.get('user', {}).get('access_token') if token: return {"access_token": token, "token_type": "bearer"} - else: - log.error("access_token empty, user probably is not logged in.") - raise HTTPException( + log.error("access_token empty, user probably is not logged in.") + raise HTTPException( status_code=401, detail="You need to be logged in", headers={"WWW-Authenticate": "Bearer"}, - ) \ No newline at end of file + ) diff --git a/src/services/kill.py b/src/services/kill.py index 7810b07..534fbd5 100644 --- a/src/services/kill.py +++ b/src/services/kill.py @@ -1,7 +1,7 @@ from fastapi import HTTPException, Request from services.helpers import logs_run, get_username, get_run_details import teuthology.kill -import logging, requests # Note: import requests after teuthology +import logging log = logging.getLogger(__name__) @@ -25,18 +25,20 @@ def run(args, send_logs: bool, access_token: str, request: Request): run_username = run_details.get("user") else: log.error("teuthology-kill is missing --run") - raise HTTPException(status_code=400, detail="--run is a required argument") - #TODO if user has admin priviledge, then they can kill any run/job. + raise HTTPException( + status_code=400, detail="--run is a required argument") + # TODO if user has admin priviledge, then they can kill any run/job. if run_username != username: - log.error("%s doesn't have permission to kill a job scheduled by: %s" % (username, run_username)) - raise HTTPException(status_code=401, detail="You don't have permission to kill this run/job") + log.error("%s doesn't have permission to kill a job scheduled by: %s", + username, run_username) + raise HTTPException( + status_code=401, detail="You don't have permission to kill this run/job") try: if send_logs: logs = logs_run(teuthology.kill.main, args) - return { "logs": logs } - else: - teuthology.kill.main(args) + return {"logs": logs} + teuthology.kill.main(args) return {"kill": "success"} except Exception as exc: - log.error("teuthology.suite.main failed with the error: " + repr(exc)) - raise HTTPException(status_code=500, detail=repr(exc)) \ No newline at end of file + log.error("teuthology.suite.main failed with the error: %s", repr(exc)) + raise HTTPException(status_code=500, detail=repr(exc)) from exc diff --git a/src/services/suite.py b/src/services/suite.py index 88fd084..34e53bb 100644 --- a/src/services/suite.py +++ b/src/services/suite.py @@ -1,9 +1,8 @@ -from fastapi import HTTPException -from services.helpers import logs_run, get_run_details from datetime import datetime -from config import settings -import teuthology.suite import logging +import teuthology.suite +from fastapi import HTTPException +from services.helpers import logs_run, get_run_details log = logging.getLogger(__name__) @@ -45,26 +44,26 @@ def run(args, dry_run: bool, send_logs: bool, access_token: str): run_details = get_run_details(run_name) return { "run": run_details, "logs": logs } except Exception as exc: - log.error("teuthology.suite.main failed with the error: " + repr(exc)) - raise HTTPException(status_code=500, detail=repr(exc)) + log.error("teuthology.suite.main failed with the error: %s", repr(exc)) + raise HTTPException(status_code=500, detail=repr(exc)) from exc -def make_run_name(run): +def make_run_name(run_dic): """ Generate a run name. A run name looks like: - teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plana + teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plan """ - if "," in run["machine_type"]: + if "," in run_dic["machine_type"]: worker = "multi" else: - worker = run["machine_type"] + worker = run_dic["machine_type"] return '-'.join( [ - run["user"], - str(run["timestamp"]), - run["suite"], - run["ceph_branch"], - run["kernel_branch"] or '-', - run["flavor"], worker + run_dic["user"], + str(run_dic["timestamp"]), + run_dic["suite"], + run_dic["ceph_branch"], + run_dic["kernel_branch"] or '-', + run_dic["flavor"], worker ] ).replace('/', ':') From 81229a9edf9709ee0c43afbd5c0c33be29bb2d4e Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Mon, 13 Mar 2023 15:22:47 +0530 Subject: [PATCH 5/6] Kill schema: --job needs to be iterable since https://github.com/ceph/teuthology/blob/main/teuthology/kill.py#L37 Signed-off-by: Vallari Agrawal --- src/schemas/kill.py | 2 +- src/services/kill.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/schemas/kill.py b/src/schemas/kill.py index 1587b57..7e4a75e 100644 --- a/src/schemas/kill.py +++ b/src/schemas/kill.py @@ -12,7 +12,7 @@ class KillArgs(BaseArgs): owner: Union[str, None] = Field(default=None, alias="--owner") run: Union[str, None] = Field(default=None, alias="--run") preserve_queue: Union[bool, None] = Field(default=None, alias="--preserve-queue") - job: Union[int, None] = Field(default=None, alias="--job") + job: Union[list, None] = Field(default=None, alias="--job") jobspec: Union[str, None] = Field(default=None, alias="--jobspec") machine_type: Union[str, None] = Field(default='default', alias="--machine-type") archive: Union[str, None] = Field(default=None, alias="--archive") diff --git a/src/services/kill.py b/src/services/kill.py index 534fbd5..29f16a2 100644 --- a/src/services/kill.py +++ b/src/services/kill.py @@ -40,5 +40,5 @@ def run(args, send_logs: bool, access_token: str, request: Request): teuthology.kill.main(args) return {"kill": "success"} except Exception as exc: - log.error("teuthology.suite.main failed with the error: %s", repr(exc)) + log.error("teuthology.kill.main failed with the error: %s", repr(exc)) raise HTTPException(status_code=500, detail=repr(exc)) from exc From ae5ed4b5979142b5fcf38cdfc2bd451ac631f6af Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Mon, 13 Mar 2023 16:00:59 +0530 Subject: [PATCH 6/6] readme: add initial login instruction Signed-off-by: Vallari Agrawal --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 038ba8b..39ceafd 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,14 @@ A REST API to execute [teuthology commands](https://docs.ceph.com/projects/teuth The documentation can be accessed at http://localhost:8082/docs after running the application. +Note: To run commands, authenticate by visiting `http://localhost:8082/login` through browser and follow the github authentication steps (this stores the auth token in browser cookies). + ### Route `/` ``` curl http://localhost:8082/ ``` -Returns `{"root": "success"}`. +Returns `{"root": "success", "session": { }}`. ### Route `/suite`