diff --git a/Dockerfile b/Dockerfile index db4abde..a8cfcd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && \ apt-get clean all COPY .teuthology.yaml /root - +COPY .env* /teuthology_api/ WORKDIR /teuthology_api COPY requirements.txt /teuthology_api/ RUN pip3 install -r requirements.txt diff --git a/README.md b/README.md index 038ba8b..39ceafd 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,14 @@ A REST API to execute [teuthology commands](https://docs.ceph.com/projects/teuth The documentation can be accessed at http://localhost:8082/docs after running the application. +Note: To run commands, authenticate by visiting `http://localhost:8082/login` through browser and follow the github authentication steps (this stores the auth token in browser cookies). + ### Route `/` ``` curl http://localhost:8082/ ``` -Returns `{"root": "success"}`. +Returns `{"root": "success", "session": { }}`. ### Route `/suite` diff --git a/requirements.txt b/requirements.txt index 6b12856..5a33b97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ fastapi uvicorn[standard] gunicorn +httpx +itsdangerous # Temporarily, using teuthology without monkey patching the thread git+https://github.com/VallariAg/teuthology@teuth-api#egg=teuthology[test] # Original: git+https://github.com/ceph/teuthology#egg=teuthology[test] diff --git a/src/config.py b/src/config.py index 82b0c5e..ca76a18 100644 --- a/src/config.py +++ b/src/config.py @@ -1,20 +1,31 @@ from functools import lru_cache -from typing import List -import os from pydantic import BaseSettings class APISettings(BaseSettings): + """ + Class for API settings. + """ PADDLES_URL: str = "http://paddles:8080" + # TODO: team names need to be changed below when created + admin_team: str = "ceph" # ceph's github team with *sudo* access to sepia + teuth_team: str = "teuth" # ceph's github team with access to sepia class Config: + """ + Class for Config. + """ + # pylint: disable=too-few-public-methods env_file = ".env" env_file_encoding = 'utf-8' @lru_cache() def get_api_settings() -> APISettings: + """ + Returns the API settings. + """ return APISettings() # reads variables from environment -settings = get_api_settings() \ No newline at end of file +settings = get_api_settings() diff --git a/src/main.py b/src/main.py index 42b47ef..0908d91 100644 --- a/src/main.py +++ b/src/main.py @@ -1,14 +1,40 @@ import logging -from fastapi import FastAPI +import os +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from starlette.middleware.sessions import SessionMiddleware +from routes import suite, kill, login, logout +from dotenv import load_dotenv -from routes import suite +load_dotenv() + +DEPLOYMENT = os.getenv('DEPLOYMENT') +SESSION_SECRET_KEY = os.getenv('SESSION_SECRET_KEY') log = logging.getLogger(__name__) app = FastAPI() - @app.get("/") -def read_root(): - return {"root": "success"} +def read_root(request: Request): + """ + GET route for root. + """ + return { + "root": "success", + "session": request.session.get('user', None) + } + +if DEPLOYMENT == 'development': + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) +app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET_KEY) app.include_router(suite.router) +app.include_router(kill.router) +app.include_router(login.router) +app.include_router(logout.router) diff --git a/src/routes/kill.py b/src/routes/kill.py new file mode 100644 index 0000000..3a2d070 --- /dev/null +++ b/src/routes/kill.py @@ -0,0 +1,29 @@ +import logging +from fastapi import APIRouter, Depends, Request +from services.kill import run +from services.helpers import get_token +from schemas.kill import KillArgs + +log = logging.getLogger(__name__) + +router = APIRouter( + prefix="/kill", + tags=["kill"], +) + +@router.post("/", status_code=200) +def create_run( + request: Request, + args: KillArgs, + logs: bool = False, + access_token: str = Depends(get_token), +): + """ + POST route for killing a run or a job. + + Note: I needed to put `request` before `args` + or else it will SyntaxError: non-dafault + argument follows default argument error. + """ + args = args.dict(by_alias=True) + return run(args, logs, access_token, request) diff --git a/src/routes/login.py b/src/routes/login.py new file mode 100644 index 0000000..872c4e8 --- /dev/null +++ b/src/routes/login.py @@ -0,0 +1,81 @@ +import logging +import os +from fastapi import APIRouter, HTTPException, Request +from starlette.responses import RedirectResponse +from dotenv import load_dotenv +import httpx + +load_dotenv() + +GH_CLIENT_ID = os.getenv('GH_CLIENT_ID') +GH_CLIENT_SECRET = os.getenv('GH_CLIENT_SECRET') +GH_AUTHORIZATION_BASE_URL = os.getenv('GH_AUTHORIZATION_BASE_URL') +GH_TOKEN_URL = os.getenv('GH_TOKEN_URL') +GH_FETCH_MEMBERSHIP_URL = os.getenv('GH_FETCH_MEMBERSHIP_URL') + +log = logging.getLogger(__name__) +router = APIRouter( + prefix="/login", + tags=["login"], + responses={404: {"description": "Not found"}}, +) + + +@router.get("/", status_code=200) +async def github_login(): + """ + GET route for /login, (If first time) will redirect to github login page + where you should authorize the app to gain access. + """ + scope = 'read:org' + return RedirectResponse( + f'{GH_AUTHORIZATION_BASE_URL}?client_id={GH_CLIENT_ID}&scope={scope}', + status_code=302 + ) + +@router.get("/callback", status_code=200) +async def handle_callback(code: str, request: Request): + """ + Call back route after user login & authorize the app + for access. + """ + params = { + 'client_id': GH_CLIENT_ID, + 'client_secret': GH_CLIENT_SECRET, + 'code': code + } + headers = {'Accept': 'application/json'} + async with httpx.AsyncClient() as client: + response_token = await client.post(url=GH_TOKEN_URL, params=params, headers=headers) + log.info(response_token.json()) + response_token_dic = dict(response_token.json()) + token = response_token_dic.get('access_token') + if response_token_dic.get('error') or not token: + log.error("The code is incorrect or expired.") + raise HTTPException( + status_code=401, detail="The code is incorrect or expired.") + headers = {'Authorization': 'token ' + token} + response_org = await client.get(url=GH_FETCH_MEMBERSHIP_URL, headers=headers) + log.info(response_org.json()) + if response_org.status_code == 404: + log.error("User is not part of the Ceph Organization") + raise HTTPException( + status_code=404, + detail="User is not part of the Ceph Organization, please contact " + ) + if response_org.status_code == 403: + log.error("The application doesn't have permission to view github org") + raise HTTPException( + status_code=403, + detail="The application doesn't have permission to view github org" + ) + response_org_dic = dict(response_org.json()) + data = { + "id": response_org_dic.get('user', {}).get('id'), + "username": response_org_dic.get('user', {}).get('login'), + "state": response_org_dic.get('state'), + "role": response_org_dic.get('role'), + "access_token": token, + } + request.session['user'] = data + return RedirectResponse(url='/') diff --git a/src/routes/logout.py b/src/routes/logout.py new file mode 100644 index 0000000..7ccc509 --- /dev/null +++ b/src/routes/logout.py @@ -0,0 +1,24 @@ +import logging +from fastapi import APIRouter, HTTPException, Request + +log = logging.getLogger(__name__) +router = APIRouter( + prefix="/logout", + tags=["logout"], + responses={404: {"description": "Not found"}}, +) + +@router.get('/', status_code=200) +def logout(request: Request): + """ + GET route for logging out. + """ + user = request.session.get('user') + if user: + request.session.pop('user', None) + return {"logout": "success"} + log.warning("No session found, probably already logged out.") + raise HTTPException( + status_code=204, + detail="No session found, probably already logged out." + ) diff --git a/src/routes/suite.py b/src/routes/suite.py index ec0ff78..290a3af 100644 --- a/src/routes/suite.py +++ b/src/routes/suite.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Depends from services.suite import run +from services.helpers import get_token from schemas.suite import SuiteArgs import logging @@ -12,10 +13,10 @@ ) @router.post("/", status_code=200) -def create_run(args: SuiteArgs, dry_run: bool = False, logs: bool = False): - try: - args = args.dict(by_alias=True) - results = run(args, dry_run, logs) - return results - except Exception as exc: - raise HTTPException(status_code=404, detail=repr(exc)) +def create_run(args: SuiteArgs, + access_token: str = Depends(get_token), + dry_run: bool = False, + logs: bool = False +): + args = args.dict(by_alias=True) + return run(args, dry_run, logs, access_token) diff --git a/src/schemas/base.py b/src/schemas/base.py index 81581c6..5a624fa 100644 --- a/src/schemas/base.py +++ b/src/schemas/base.py @@ -1,8 +1,12 @@ -from typing import Union, Optional +from typing import Union from pydantic import BaseModel, Field class BaseArgs(BaseModel): + # pylint: disable=too-few-public-methods + """ + Class for Base Args. + """ dry_run: Union[bool, None] = Field(default=False, alias="--dry-run") non_interactive: Union[bool, None] = Field(default=False, alias="--non-interactive") verbose: Union[int, None] = Field(default=1, alias="--verbose") diff --git a/src/schemas/kill.py b/src/schemas/kill.py new file mode 100644 index 0000000..7e4a75e --- /dev/null +++ b/src/schemas/kill.py @@ -0,0 +1,18 @@ +from typing import Union +from pydantic import Field + +from .base import BaseArgs + + +class KillArgs(BaseArgs): + # pylint: disable=too-few-public-methods + """ + Class for KillArgs. + """ + owner: Union[str, None] = Field(default=None, alias="--owner") + run: Union[str, None] = Field(default=None, alias="--run") + preserve_queue: Union[bool, None] = Field(default=None, alias="--preserve-queue") + job: Union[list, None] = Field(default=None, alias="--job") + jobspec: Union[str, None] = Field(default=None, alias="--jobspec") + machine_type: Union[str, None] = Field(default='default', alias="--machine-type") + archive: Union[str, None] = Field(default=None, alias="--archive") diff --git a/src/schemas/schedule.py b/src/schemas/schedule.py index e9ddcdb..ffb0c46 100644 --- a/src/schemas/schedule.py +++ b/src/schemas/schedule.py @@ -1,10 +1,14 @@ -from typing import Union, Optional +from typing import Union from pydantic import Field from .base import BaseArgs class SchedulerArgs(BaseArgs): + # pylint: disable=too-few-public-methods + """ + Class for SchedulerArgs. + """ owner: Union[str, None] = Field(default=None, alias="--owner") num: Union[str, None] = Field(default='1', alias="--num") priority: Union[str, None] = Field(default='70', alias="--priority") diff --git a/src/schemas/suite.py b/src/schemas/suite.py index f1d24e0..6e017f8 100644 --- a/src/schemas/suite.py +++ b/src/schemas/suite.py @@ -1,51 +1,74 @@ -from typing import Union, Optional +from typing import Union from pydantic import Field from .base import BaseArgs class SuiteArgs(BaseArgs): - # Standard arguments + # pylint: disable=too-few-public-methods + """ + Standard arguments + """ arch: Union[str, None] = Field(default=None, alias="--arch") ceph: Union[str, None] = Field(default='main', alias="--ceph") - ceph_repo: Union[str, None] = Field(default='https://github.com/ceph/ceph-ci.git', alias="--ceph-repo") + ceph_repo: Union[str, None] = Field( + default='https://github.com/ceph/ceph-ci.git', alias="--ceph-repo") distro: Union[str, None] = Field(default=None, alias="--distro") - distro_version: Union[str, None] = Field(default=None, alias="--distro-version") + distro_version: Union[str, None] = Field( + default=None, alias="--distro-version") email: Union[str, None] = Field(default=None, alias="--email") flavor: Union[str, None] = Field(default='default', alias="--flavor") kernel: Union[str, None] = Field(default='distro', alias="--kernel") - machine_type: Union[str, None] = Field(default='smithi', alias="--machine-type") + machine_type: Union[str, None] = Field( + default='smithi', alias="--machine-type") newest: Union[str, None] = Field(default='0', alias="--newest") - rerun_status: Union[bool, None] = Field(default=False, alias="--rerun-status.") - rerun_statuses: Union[str, None] = Field(default='fail,dead', alias="--rerun-statuses") + rerun_status: Union[bool, None] = Field( + default=False, alias="--rerun-status.") + rerun_statuses: Union[str, None] = Field( + default='fail,dead', alias="--rerun-statuses") sha1: Union[str, None] = Field(default=None, alias="--sha1") - sleep_before_teardown: Union[str, None] = Field(default='0', alias="--sleep-before-teardown") + sleep_before_teardown: Union[str, None] = Field( + default='0', alias="--sleep-before-teardown") suite: str = Field(alias="--suite") - suite_branch: Union[str, None] = Field(default=None, alias="--suite-branch") + suite_branch: Union[str, None] = Field( + default=None, alias="--suite-branch") suite_dir: Union[str, None] = Field(default=None, alias="--suite-dir") - suite_relpath: Union[str, None] = Field(default='qa', alias="--suite-relpath") - suite_repo: Union[str, None] = Field(default='https://github.com/ceph/ceph-ci.git', alias="--suite_repo") - teuthology_branch: Union[str, None] = Field(default='main', alias="--teuthology-branch") - validate_sha1: Union[str, None] = Field(default='true', alias="--validate-sha1") + suite_relpath: Union[str, None] = Field( + default='qa', alias="--suite-relpath") + suite_repo: Union[str, None] = Field( + default='https://github.com/ceph/ceph-ci.git', alias="--suite_repo") + teuthology_branch: Union[str, None] = Field( + default='main', alias="--teuthology-branch") + validate_sha1: Union[str, None] = Field( + default='true', alias="--validate-sha1") wait: Union[bool, None] = Field(default=False, alias="--wait") config_yaml: Union[list, None] = Field(default=[], alias="") - # Scheduler arguments + """ + Scheduler arguments + """ owner: Union[str, None] = Field(default=None, alias="--owner") num: Union[str, None] = Field(default='1', alias="--num") priority: Union[str, None] = Field(default='70', alias="--priority") - queue_backend: Union[str, None] = Field(default=None, alias="--queue-backend") + queue_backend: Union[str, None] = Field( + default=None, alias="--queue-backend") rerun: Union[str, None] = Field(default=None, alias="--rerun") seed: Union[str, None] = Field(default='-1', alias="--seed") - force_priority: Union[bool, None] = Field(default=False, alias="--force-priority") - no_nested_subset: Union[bool, None] = Field(default=False, alias="--no-nested-subset") - job_threshold: Union[str, None] = Field(default='500', alias="--job-threshold") - archive_upload: Union[str, None] = Field(default=None, alias="--archive-upload") - archive_upload_url: Union[str, None] = Field(default=None, alias="--archive-upload-url") + force_priority: Union[bool, None] = Field( + default=False, alias="--force-priority") + no_nested_subset: Union[bool, None] = Field( + default=False, alias="--no-nested-subset") + job_threshold: Union[str, None] = Field( + default='500', alias="--job-threshold") + archive_upload: Union[str, None] = Field( + default=None, alias="--archive-upload") + archive_upload_url: Union[str, None] = Field( + default=None, alias="--archive-upload-url") throttle: Union[str, None] = Field(default=None, alias="--throttle") filter: Union[str, None] = Field(default=None, alias="--filter") filter_out: Union[str, None] = Field(default=None, alias="--filter-out") filter_all: Union[str, None] = Field(default=None, alias="--filter-all") - filter_fragments: Union[str, None] = Field(default='false', alias="--filter-fragments") + filter_fragments: Union[str, None] = Field( + default='false', alias="--filter-fragments") subset: Union[str, None] = Field(default=None, alias="--subset") timeout: Union[str, None] = Field(default='43200', alias="--timeout") rocketchat: Union[str, None] = Field(default=None, alias="--rocketchat") diff --git a/src/services/helpers.py b/src/services/helpers.py index a2ae497..a4e0efb 100644 --- a/src/services/helpers.py +++ b/src/services/helpers.py @@ -1,31 +1,96 @@ -import uuid, os from multiprocessing import Process +import logging +import os +import uuid +from config import settings +from fastapi import HTTPException, Request +from requests.exceptions import HTTPError import teuthology +import requests # Note: import requests after teuthology + +PADDLES_URL = settings.PADDLES_URL + +log = logging.getLogger(__name__) + def logs_run(func, args): """ Run the command function in a seperate process (to isolate logs), and return logs printed during the execution of the function. """ - id = str(uuid.uuid4()) - log_file = f'/tmp/{id}.log' + _id = str(uuid.uuid4()) + log_file = f'/archive_dir/{_id}.log' teuthology_process = Process( - target=_execute_with_logs, args=(func, args, log_file,)) + target=_execute_with_logs, args=(func, args, log_file)) teuthology_process.start() teuthology_process.join() logs = "" - with open(log_file) as f: - logs = f.readlines() - if os.path.isfile(log_file): + with open(log_file, encoding="utf-8") as file: + logs = file.readlines() + if os.path.isfile(log_file): os.remove(log_file) return logs + def _execute_with_logs(func, args, log_file): """ To store logs, set a new FileHandler for teuthology root logger and then execute the command function. """ teuthology.setup_log_file(log_file) - func(args) \ No newline at end of file + func(args) + + +def get_run_details(run_name: str): + """ + Queries paddles to look if run is created. + """ + url = f'{PADDLES_URL}/runs/{run_name}/' + try: + run_info = requests.get(url) + run_info.raise_for_status() + return run_info.json() + except HTTPError as http_err: + log.error(http_err) + raise HTTPException( + status_code=http_err.response.status_code, + detail=repr(http_err) + ) from http_err + except Exception as err: + log.error(err) + raise HTTPException( + status_code=500, + detail=repr(err) + ) from err + + +def get_username(request: Request): + """ + Get username from request.session + """ + username = request.session.get('user', {}).get('username') + if username: + return username + log.error("username empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def get_token(request: Request): + """ + Get access token from request.session + """ + token = request.session.get('user', {}).get('access_token') + if token: + return {"access_token": token, "token_type": "bearer"} + log.error("access_token empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) diff --git a/src/services/kill.py b/src/services/kill.py new file mode 100644 index 0000000..29f16a2 --- /dev/null +++ b/src/services/kill.py @@ -0,0 +1,44 @@ +from fastapi import HTTPException, Request +from services.helpers import logs_run, get_username, get_run_details +import teuthology.kill +import logging + + +log = logging.getLogger(__name__) + + +def run(args, send_logs: bool, access_token: str, request: Request): + """ + Kill running teuthology jobs. + """ + if not access_token: + log.error("access_token empty, user probably is not logged in.") + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) + username = get_username(request) + run_name = args.get("--run") + if run_name: + run_details = get_run_details(run_name) + run_username = run_details.get("user") + else: + log.error("teuthology-kill is missing --run") + raise HTTPException( + status_code=400, detail="--run is a required argument") + # TODO if user has admin priviledge, then they can kill any run/job. + if run_username != username: + log.error("%s doesn't have permission to kill a job scheduled by: %s", + username, run_username) + raise HTTPException( + status_code=401, detail="You don't have permission to kill this run/job") + try: + if send_logs: + logs = logs_run(teuthology.kill.main, args) + return {"logs": logs} + teuthology.kill.main(args) + return {"kill": "success"} + except Exception as exc: + log.error("teuthology.kill.main failed with the error: %s", repr(exc)) + raise HTTPException(status_code=500, detail=repr(exc)) from exc diff --git a/src/services/suite.py b/src/services/suite.py index ed07218..34e53bb 100644 --- a/src/services/suite.py +++ b/src/services/suite.py @@ -1,20 +1,23 @@ -import teuthology.suite -from services.helpers import logs_run -import logging, requests # Note: import requests after teuthology from datetime import datetime - -from config import settings - -PADDLES_URL = settings.PADDLES_URL +import logging +import teuthology.suite +from fastapi import HTTPException +from services.helpers import logs_run, get_run_details log = logging.getLogger(__name__) -def run(args, dry_run: bool, send_logs: bool): +def run(args, dry_run: bool, send_logs: bool, access_token: str): """ Schedule a suite. :returns: Run details (dict) and logs (list). """ + if not access_token: + raise HTTPException( + status_code=401, + detail="You need to be logged in", + headers={"WWW-Authenticate": "Bearer"}, + ) try: args["--timestamp"] = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') if dry_run: @@ -41,37 +44,26 @@ def run(args, dry_run: bool, send_logs: bool): run_details = get_run_details(run_name) return { "run": run_details, "logs": logs } except Exception as exc: - log.error("teuthology.suite.main failed with the error: " + repr(exc)) - raise - -def get_run_details(run_name): - """ - Queries paddles to look if run is created. - """ - try: - url = f'{PADDLES_URL}/runs/{run_name}/' - run_info = requests.get(url).json() - return run_info - except: - raise RuntimeError(f"Unable to find run `{run_name}` in paddles database.") + log.error("teuthology.suite.main failed with the error: %s", repr(exc)) + raise HTTPException(status_code=500, detail=repr(exc)) from exc -def make_run_name(run): +def make_run_name(run_dic): """ Generate a run name. A run name looks like: - teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plana + teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plan """ - if "," in run["machine_type"]: + if "," in run_dic["machine_type"]: worker = "multi" else: - worker = run["machine_type"] + worker = run_dic["machine_type"] return '-'.join( [ - run["user"], - str(run["timestamp"]), - run["suite"], - run["ceph_branch"], - run["kernel_branch"] or '-', - run["flavor"], worker + run_dic["user"], + str(run_dic["timestamp"]), + run_dic["suite"], + run_dic["ceph_branch"], + run_dic["kernel_branch"] or '-', + run_dic["flavor"], worker ] ).replace('/', ':')