Make ETOS API compatible with the kubernetes controller poc (#69)

* Make ETOS API compatible with the kubernetes controller poc

Add a new router for starting ETOS using the new TestRun
resource from the ETOS kubernetes controller.
Added a root path for the ETOS API python code so that we
can make a standard Ingress for the ETOS API.
Added StripPrefix to the SSE and LogArea services for so
that we can make a standard Ingress for the ETOS API.

internal/configs/base/config.go

@@ -25,6 +25,7 @@ import (
 type Config interface {
 	ServiceHost() string
 	ServicePort() string
+	StripPrefix() string
 	LogLevel() string
 	LogFilePath() string
 	ETOSNamespace() string
@@ -35,6 +36,7 @@ type Config interface {
 type cfg struct {
 	serviceHost   string
 	servicePort   string
+	stripPrefix   string
 	logLevel      string
 	logFilePath   string
 	etosNamespace string
@@ -48,6 +50,7 @@ func Get() Config {
 
 	flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on")
 	flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on")
+	flag.StringVar(&conf.stripPrefix, "stripprefix", EnvOrDefault("STRIP_PREFIX", ""), "Strip a URL prefix. Useful when a reverse proxy sets a subpath. I.e. reverse proxy sets /stream as prefix, making the etos API available at /stream/v1/events. In that case we want to set stripprefix to /stream")
 	flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).")
 	flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.")
 	flag.StringVar(&conf.etosNamespace, "etosnamespace", ReadNamespaceOrEnv("ETOS_NAMESPACE"), "Path, including filename, for the log files to create.")
@@ -68,6 +71,11 @@ func (c *cfg) ServicePort() string {
 	return c.servicePort
 }
 
+// StripPrefix returns the prefix to strip. Empty string if no prefix.
+func (c *cfg) StripPrefix() string {
+	return c.stripPrefix
+}
+
 // LogLevel returns the log level.
 func (c *cfg) LogLevel() string {
 	return c.logLevel

internal/configs/iut/config.go

@@ -25,6 +25,7 @@ import (
 type Config interface {
 	ServiceHost() string
 	ServicePort() string
+	StripPrefix() string
 	LogLevel() string
 	LogFilePath() string
 	ETOSNamespace() string
@@ -35,6 +36,7 @@ type Config interface {
 type cfg struct {
 	serviceHost   string
 	servicePort   string
+	stripPrefix   string
 	logLevel      string
 	logFilePath   string
 	etosNamespace string
@@ -48,6 +50,7 @@ func Get() Config {
 
 	flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on")
 	flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on")
+	flag.StringVar(&conf.stripPrefix, "stripprefix", EnvOrDefault("STRIP_PREFIX", ""), "Strip a URL prefix. Useful when a reverse proxy sets a subpath. I.e. reverse proxy sets /stream as prefix, making the etos API available at /stream/v1/events. In that case we want to set stripprefix to /stream")
 	flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).")
 	flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.")
 	flag.StringVar(&conf.databaseHost, "database_host", EnvOrDefault("ETOS_ETCD_HOST", "etcd-client"), "Host to ETOS database")
@@ -67,6 +70,11 @@ func (c *cfg) ServicePort() string {
 	return c.servicePort
 }
 
+// StripPrefix returns the prefix to strip. Empty string if no prefix.
+func (c *cfg) StripPrefix() string {
+	return c.stripPrefix
+}
+
 // LogLevel returns the log level.
 func (c *cfg) LogLevel() string {
 	return c.logLevel

internal/configs/logarea/config.go

@@ -25,6 +25,7 @@ import (
 type Config interface {
 	ServiceHost() string
 	ServicePort() string
+	StripPrefix() string
 	LogLevel() string
 	LogFilePath() string
 	ETOSNamespace() string
@@ -35,6 +36,7 @@ type Config interface {
 type cfg struct {
 	serviceHost   string
 	servicePort   string
+	stripPrefix   string
 	logLevel      string
 	logFilePath   string
 	etosNamespace string
@@ -48,6 +50,7 @@ func Get() Config {
 
 	flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on")
 	flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on")
+	flag.StringVar(&conf.stripPrefix, "stripprefix", EnvOrDefault("STRIP_PREFIX", ""), "Strip a URL prefix. Useful when a reverse proxy sets a subpath. I.e. reverse proxy sets /stream as prefix, making the etos API available at /stream/v1/events. In that case we want to set stripprefix to /stream")
 	flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).")
 	flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.")
 	flag.StringVar(&conf.etosNamespace, "etosnamespace", ReadNamespaceOrEnv("ETOS_NAMESPACE"), "Path, including filename, for the log files to create.")
@@ -68,6 +71,11 @@ func (c *cfg) ServicePort() string {
 	return c.servicePort
 }
 
+// StripPrefix returns the prefix to strip. Empty string if no prefix.
+func (c *cfg) StripPrefix() string {
+	return c.stripPrefix
+}
+
 // LogLevel returns the log level.
 func (c *cfg) LogLevel() string {
 	return c.logLevel

internal/configs/sse/config.go

@@ -25,6 +25,7 @@ import (
 type Config interface {
 	ServiceHost() string
 	ServicePort() string
+	StripPrefix() string
 	LogLevel() string
 	LogFilePath() string
 	ETOSNamespace() string
@@ -35,6 +36,7 @@ type Config interface {
 type cfg struct {
 	serviceHost   string
 	servicePort   string
+	stripPrefix   string
 	logLevel      string
 	logFilePath   string
 	etosNamespace string
@@ -48,6 +50,7 @@ func Get() Config {
 
 	flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on")
 	flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on")
+	flag.StringVar(&conf.stripPrefix, "stripprefix", EnvOrDefault("STRIP_PREFIX", ""), "Strip a URL prefix. Useful when a reverse proxy sets a subpath. I.e. reverse proxy sets /stream as prefix, making the etos API available at /stream/v1/events. In that case we want to set stripprefix to /stream")
 	flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).")
 	flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.")
 	flag.StringVar(&conf.etosNamespace, "etosnamespace", ReadNamespaceOrEnv("ETOS_NAMESPACE"), "Path, including filename, for the log files to create.")
@@ -68,6 +71,11 @@ func (c *cfg) ServicePort() string {
 	return c.servicePort
 }
 
+// StripPrefix returns the prefix to strip. Empty string if no prefix.
+func (c *cfg) StripPrefix() string {
+	return c.stripPrefix
+}
+
 // LogLevel returns the log level.
 func (c *cfg) LogLevel() string {
 	return c.logLevel

internal/server/server.go

@@ -39,6 +39,9 @@ type WebService struct {
 
 // NewWebService creates a new Server of the webservice type.
 func NewWebService(cfg config.Config, log *logrus.Entry, handler http.Handler) Server {
+	if cfg.StripPrefix() != "" {
+		handler = http.StripPrefix(cfg.StripPrefix(), handler)
+	}
 	webservice := &WebService{
 		server: &http.Server{
 			Addr:    fmt.Sprintf("%s:%s", cfg.ServiceHost(), cfg.ServicePort()),
@@ -52,7 +55,7 @@ func NewWebService(cfg config.Config, log *logrus.Entry, handler http.Handler) S
 
 // Start a webservice and block until closed or crashed.
 func (s *WebService) Start() error {
-	s.logger.Infof("Starting webservice listening on %s:%s", s.cfg.ServiceHost(), s.cfg.ServicePort())
+	s.logger.Infof("Starting webservice listening on %s:%s%s", s.cfg.ServiceHost(), s.cfg.ServicePort(), s.cfg.StripPrefix())
 	return s.server.ListenAndServe()
 }
 

manifests/base/role.yaml

@@ -19,6 +19,16 @@ rules:
     - delete
     - list
     - watch
+  - apiGroups:
+    - "etos.eiffel-community.github.io"
+    resources:
+    - testruns
+    verbs:
+    - create
+    - get
+    - delete
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:

pkg/application/application.go

@@ -15,7 +15,9 @@
 // limitations under the License.
 package application
 
-import "github.com/julienschmidt/httprouter"
+import (
+	"github.com/julienschmidt/httprouter"
+)
 
 type Application interface {
 	LoadRoutes(*httprouter.Router)

python/pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-    "etos_lib==4.3.1",
+    "etos_lib==4.4.1",
     "etcd3gw~=2.3",
     "uvicorn~=0.22",
     "fastapi~=0.109.1",
@@ -62,4 +62,4 @@ testpaths = ["tests"]
 root = ".."
 
 [tool.setuptools.packages]
-find = { where = ["src"], exclude = ["tests"] }
+find = { where = ["src"], exclude = ["tests"] }

python/src/etos_api/library/validator.py

@@ -18,8 +18,6 @@
 from typing import List, Union
 from uuid import UUID
 
-import requests
-
 # Pylint refrains from linting C extensions due to arbitrary code execution.
 from pydantic import BaseModel  # pylint:disable=no-name-in-module
 from pydantic import ValidationError, conlist, constr, field_validator
@@ -157,33 +155,14 @@ class SuiteValidator:
 
     logger = logging.getLogger(__name__)
 
-    async def _download_suite(self, test_suite_url):
-        """Attempt to download suite.
-
-        :param test_suite_url: URL to test suite to download.
-        :type test_suite_url: str
-        :return: Downloaded test suite as JSON.
-        :rtype: list
-        """
-        try:
-            suite = requests.get(test_suite_url, timeout=60)
-            suite.raise_for_status()
-        except Exception as exception:  # pylint:disable=broad-except
-            raise AssertionError(f"Unable to download suite from {test_suite_url}") from exception
-        return suite.json()
-
-    async def validate(self, test_suite_url):
+    async def validate(self, test_suite):
         """Validate the ETOS suite definition.
 
-        :param test_suite_url: URL to test suite that is being executed.
-        :type test_suite_url: str
+        :param test_suite: Test suite that is being executed.
+        :type test_suite: list
         :raises ValidationError: If the suite did not validate.
         """
-        downloaded_suite = await self._download_suite(test_suite_url)
-        assert (
-            len(downloaded_suite) > 0
-        ), "Suite definition validation unsuccessful - Reason: Empty Test suite definition list"
-        for suite_json in downloaded_suite:
+        for suite_json in test_suite:
             test_runners = set()
             suite = Suite(**suite_json)
             assert suite

python/src/etos_api/main.py

@@ -18,12 +18,12 @@
 
 from fastapi import FastAPI
 
-# from opentelemetry.sdk.trace import TracerProvider
 from starlette.responses import RedirectResponse
 
 from etos_api import routers
 
-APP = FastAPI()
+# This allows the path to start either at '/api' or '/'.
+APP = FastAPI(root_path="/api")
 LOGGER = logging.getLogger(__name__)
 
 
@@ -52,5 +52,6 @@ async def redirect_head_to_root():
 
 
 APP.include_router(routers.etos.ROUTER)
+APP.include_router(routers.testrun.ROUTER)
 APP.include_router(routers.selftest.ROUTER)
 APP.include_router(routers.logs.ROUTER)

python/src/etos_api/routers/__init__.py

@@ -14,4 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ETOS API routers module."""
-from . import etos, logs, selftest
+import os
+from kubernetes import config
+from . import etos, testrun, logs, selftest
+
+if os.getenv("RUNNING_TESTS") is None:
+    try:
+        config.load_incluster_config()
+    except config.ConfigException:
+        config.load_config()

python/src/etos_api/routers/etos/router.py

@@ -24,6 +24,7 @@
 from kubernetes import client
 from opentelemetry import trace
 from opentelemetry.trace import Span
+import requests
 
 from etos_api.library.environment import Configuration, configure_testrun
 from etos_api.library.utilities import sync_to_async
@@ -39,6 +40,20 @@
 logging.getLogger("pika").setLevel(logging.WARNING)
 
 
+async def download_suite(test_suite_url: str) -> dict:
+    """Attempt to download suite.
+
+    :param test_suite_url: URL to test suite to download.
+    :return: Downloaded test suite as JSON.
+    """
+    try:
+        suite = requests.get(test_suite_url, timeout=60)
+        suite.raise_for_status()
+    except Exception as exception:  # pylint:disable=broad-except
+        raise AssertionError(f"Unable to download suite from {test_suite_url}") from exception
+    return suite.json()
+
+
 async def validate_suite(test_suite_url: str) -> None:
     """Validate the ETOS test suite through the SuiteValidator.
 
@@ -47,7 +62,8 @@ async def validate_suite(test_suite_url: str) -> None:
     span = trace.get_current_span()
 
     try:
-        await SuiteValidator().validate(test_suite_url)
+        test_suite = await download_suite(test_suite_url)
+        await SuiteValidator().validate(test_suite)
     except AssertionError as exception:
         LOGGER.error("Test suite validation failed!")
         LOGGER.error(exception)

python/src/etos_api/routers/lib/kubernetes.py

@@ -17,19 +17,9 @@
 import logging
 import os
 
-from kubernetes import config
-
 NAMESPACE_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
 LOGGER = logging.getLogger(__name__)
 
-try:
-    config.load_incluster_config()
-except config.ConfigException:
-    try:
-        config.load_config()
-    except config.ConfigException:
-        LOGGER.warning("Could not load a Kubernetes config")
-
 
 def namespace() -> str:
     """Get current namespace if available."""

python/src/etos_api/routers/testrun/__init__.py

@@ -0,0 +1,18 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS API testrun module."""
+from .router import ROUTER
+from . import schemas