From ce0e0d47ecdc17cbab9100a8da4a70a03d34a408 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Mon, 11 May 2020 08:56:49 +0300 Subject: [PATCH] Bugfix: Add config/ fodler --- .gitignore | 1 - config/base/.env.gcp | 29 ++++++++ config/base/.env.local | 20 ++++++ config/base/docker-compose.yaml | 80 ++++++++++++++++++++++ config/base/docker/base/Dockerfile | 18 +++++ config/base/docker/base/requirements.txt | 12 ++++ config/base/docker/deploy/Dockerfile | 17 +++++ config/base/docker/deploy/requirements.txt | 4 ++ config/base/docker/projects/Dockerfile | 5 ++ config/base/docker/ui/Dockerfile | 17 +++++ config/base/docker/web/Dockerfile | 12 ++++ config/base/nginx/.gitignore | 1 + config/base/nginx/conf.d/cors.conf | 8 +++ config/base/nginx/nginx.conf | 40 +++++++++++ config/credentials/.gitignore | 1 + 15 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 config/base/.env.gcp create mode 100644 config/base/.env.local create mode 100644 config/base/docker-compose.yaml create mode 100644 config/base/docker/base/Dockerfile create mode 100644 config/base/docker/base/requirements.txt create mode 100644 config/base/docker/deploy/Dockerfile create mode 100644 config/base/docker/deploy/requirements.txt create mode 100644 config/base/docker/projects/Dockerfile create mode 100644 config/base/docker/ui/Dockerfile create mode 100644 config/base/docker/web/Dockerfile create mode 100644 config/base/nginx/.gitignore create mode 100644 config/base/nginx/conf.d/cors.conf create mode 100644 config/base/nginx/nginx.conf create mode 100644 config/credentials/.gitignore diff --git a/.gitignore b/.gitignore index 33d8d83..bb3e327 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ services/projects/examples/models/* ## configs .env -config/* ## credentials .htpasswd diff --git a/config/base/.env.gcp b/config/base/.env.gcp new file mode 100644 index 0000000..35fb0cb --- /dev/null +++ b/config/base/.env.gcp @@ -0,0 +1,29 @@ +# Common +WORKSPACE=[PATH-TO-WORKSPACE-FOLDER-ON-LOCAL-HOST] +HOST_IP=0.0.0.0 +LOGLEVEL=DEBUG + +# Database +PROJECTS_DB_NAME=projects_db +DEPLOY_DB_NAME=deploy_db +POSTGRES_USER=user +POSTGRES_PASSWORD=passw +DB_HOST=db +DB_PORT=5432 + +# Deploy +GOOGLE_APPLICATION_CREDENTIALS=/home/config/credentials/ +GCP_PROJECT= +GCP_ZONE=us-east1-b +GCP_MACHINE_TYPE=g1-small +GCP_OS_IMAGE=cos-dev-84-13078-0-0 +GCP_BUCKET= +MODEL_DEPLOY_DOCKER_IMAGE=mlrepa/mlpanel-deploy-web:v0.2 +MODEL_DEPLOY_DEFAULT_PORT=5000 +MODEL_DEPLOY_FIREWALL_RULE=mlflow-deploy +DEPLOY_SERVER_WORKERS=1 + +# Projects +ARTIFACT_STORE=[mlruns|gs://] +TRACKING_SERVER_PORTS=5000-5100 +TRACKING_SERVER_WORKERS=1 \ No newline at end of file diff --git a/config/base/.env.local b/config/base/.env.local new file mode 100644 index 0000000..b8e4e3a --- /dev/null +++ b/config/base/.env.local @@ -0,0 +1,20 @@ +# Common +WORKSPACE=[PATH-TO-WORKSPACE-FOLDER-ON-LOCAL-HOST] +HOST_IP=0.0.0.0 +LOGLEVEL=DEBUG + +# Database +PROJECTS_DB_NAME=projects_db +DEPLOY_DB_NAME=deploy_db +POSTGRES_USER=user +POSTGRES_PASSWORD=passw +DB_HOST=db +DB_PORT=5432 + +# Deploy +DEPLOY_SERVER_WORKERS=1 + +# Projects +ARTIFACT_STORE=mlruns +TRACKING_SERVER_PORTS=5000-5100 +TRACKING_SERVER_WORKERS=1 \ No newline at end of file diff --git a/config/base/docker-compose.yaml b/config/base/docker-compose.yaml new file mode 100644 index 0000000..ace9f0b --- /dev/null +++ b/config/base/docker-compose.yaml @@ -0,0 +1,80 @@ +version: '3.5' + +services: + + db: + environment: + POSTGRES_USER: $POSTGRES_USER + POSTGRES_PASSWORD: $POSTGRES_PASSWORD + PGDATA: "/var/lib/postgresql/data/pgdata" + image: postgres:12.2 + container_name: mlpanel-database + expose: + - $DB_PORT + volumes: + - $WORKSPACE/ps_data:/var/lib/postgresql/data + networks: + - mlpanel_default + + mlpanel: + image: mlrepa/mlpanel-base-web:v0.2 + container_name: mlpanel-base-web + ports: + - 8089:8089 + - 5000-5100:5000-5100 + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./nginx/conf.d:/etc/nginx/conf.d + depends_on: + - projects + networks: + - mlpanel_default + + projects: + env_file: + - .env + image: mlrepa/mlpanel-base-projects:v0.2 + container_name: mlpanel-base-projects + expose: + - 8080 + - 5000-5100 + volumes: + - ../../config/credentials:/home/config/credentials + - ../../services/projects:/home/projects + - ../../common:/home/common + - $WORKSPACE:$WORKSPACE + depends_on: + - db + networks: + - mlpanel_default + + deploy: + env_file: + - .env + image: mlrepa/mlpanel-base-deploy:v0.2 + container_name: mlpanel-base-deploy + expose: + - 9000 + volumes: + - ../../config/credentials:/home/config/credentials + - ../../services/deploy:/home/deploy + - ../../common:/home/common + - $WORKSPACE:$WORKSPACE + depends_on: + - db + networks: + - mlpanel_default + + ui: + environment: + HOST_IP: $HOST_IP + image: mlrepa/mlpanel-base-ui:v0.2 + container_name: mlpanel-base-ui + ports: + - 3000:3000 + networks: + - mlpanel_default + +networks: + mlpanel_default: + name: mlpanel_default diff --git a/config/base/docker/base/Dockerfile b/config/base/docker/base/Dockerfile new file mode 100644 index 0000000..dd01818 --- /dev/null +++ b/config/base/docker/base/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.7-slim + +RUN apt-get update && \ + apt-get install -y curl gcc sudo postgresql-client && \ + useradd -m user -u 1000 && \ + echo 'user:user' | chpasswd user && \ + echo "user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/user && \ + chmod 0440 /etc/sudoers.d/user && \ + chown -R user /home && \ + rm -rf /var/lib/apt/lists/* + +COPY ./requirements.txt /tmp/requirements.txt + +RUN pip install -r /tmp/requirements.txt + +ENV PYTHONPATH=/home + +USER user \ No newline at end of file diff --git a/config/base/docker/base/requirements.txt b/config/base/docker/base/requirements.txt new file mode 100644 index 0000000..8eb8e16 --- /dev/null +++ b/config/base/docker/base/requirements.txt @@ -0,0 +1,12 @@ +aiofiles==0.5.0 +email-validator==1.1.0 +fastapi==0.54.1 +google-cloud-storage==1.28.0 +jinja2==2.11.2 +mlflow==1.6.0 +psutil==5.7.0 +psycopg2-binary==2.8.5 +pytest==5.4.1 +python-multipart==0.0.5 +pyyaml==5.3.1 +uvicorn==0.11.5 \ No newline at end of file diff --git a/config/base/docker/deploy/Dockerfile b/config/base/docker/deploy/Dockerfile new file mode 100644 index 0000000..b4b107e --- /dev/null +++ b/config/base/docker/deploy/Dockerfile @@ -0,0 +1,17 @@ +FROM mlrepa/mlpanel-base:latest + +# Google Cloud SDK installation +RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz \ + && mkdir -p /home/user/gcloud \ + && tar -C /home/user/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ + && /home/user/gcloud/google-cloud-sdk/install.sh -q +# Adding the package path to local +ENV PATH $PATH:/home/user/gcloud/google-cloud-sdk/bin + + +COPY ./requirements.txt /tmp/requirements.txt +RUN sudo pip install -r /tmp/requirements.txt + +WORKDIR /home/deploy + +CMD chmod +x startup.sh && ./startup.sh diff --git a/config/base/docker/deploy/requirements.txt b/config/base/docker/deploy/requirements.txt new file mode 100644 index 0000000..7662c1d --- /dev/null +++ b/config/base/docker/deploy/requirements.txt @@ -0,0 +1,4 @@ +google-api-python-client==1.8.2 +oauth2client==3.0.0 +requests==2.23.0 +scikit-learn==0.22.2 diff --git a/config/base/docker/projects/Dockerfile b/config/base/docker/projects/Dockerfile new file mode 100644 index 0000000..c893ce8 --- /dev/null +++ b/config/base/docker/projects/Dockerfile @@ -0,0 +1,5 @@ +FROM mlrepa/mlpanel-base:latest + +WORKDIR /home/projects + +CMD chmod +x startup.sh && ./startup.sh \ No newline at end of file diff --git a/config/base/docker/ui/Dockerfile b/config/base/docker/ui/Dockerfile new file mode 100644 index 0000000..2cb6b04 --- /dev/null +++ b/config/base/docker/ui/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:16.04 + +# Install NodeJS +RUN apt-get update && \ + apt-get install -y git curl python3-software-properties && \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ + apt-get install -y nodejs + +# Clone UI repository +RUN cd /home && \ + git clone https://github.com/mlrepa/mlpanel-ui.git && \ + cd mlpanel-ui && \ + npm install + +WORKDIR /home/mlpanel-ui + +CMD git pull origin master && export REACT_APP_API_URL=http://${HOST_IP}:8089/ && npm start diff --git a/config/base/docker/web/Dockerfile b/config/base/docker/web/Dockerfile new file mode 100644 index 0000000..9467555 --- /dev/null +++ b/config/base/docker/web/Dockerfile @@ -0,0 +1,12 @@ +FROM ubuntu:16.04 + +RUN apt-get update && \ + apt-get dist-upgrade -y && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:nginx/stable && \ + apt-get update && \ + apt-get install -y nginx-extras && \ + rm -rf /var/cache/apt && rm -rf /var/lib/apt + + +CMD nginx -g 'daemon off;' \ No newline at end of file diff --git a/config/base/nginx/.gitignore b/config/base/nginx/.gitignore new file mode 100644 index 0000000..4529a17 --- /dev/null +++ b/config/base/nginx/.gitignore @@ -0,0 +1 @@ +!conf.d \ No newline at end of file diff --git a/config/base/nginx/conf.d/cors.conf b/config/base/nginx/conf.d/cors.conf new file mode 100644 index 0000000..8655cc1 --- /dev/null +++ b/config/base/nginx/conf.d/cors.conf @@ -0,0 +1,8 @@ +add_header "Access-Control-Allow-Origin" * always; +if ($request_method = 'OPTIONS') { + add_header "Access-Control-Allow-Origin" *; + add_header Access-Control-Allow-Headers "Cache-Control,If-Match,If-None-Match,If-Modified-Since,If-Unmodified-Since,If-Range,Range,Authorization,Content-Type,Link,Slug"; + add_header Access-Control-Allow-Methods "OPTIONS,HEAD,GET,PATCH,POST,PUT,DELETE,PROPFIND,PROPPATCH,MKCOL,COPY,MOVE,LOCK,UNLOCK"; + add_header Access-Control-Max-Age 60; + return 204; +} \ No newline at end of file diff --git a/config/base/nginx/nginx.conf b/config/base/nginx/nginx.conf new file mode 100644 index 0000000..cf33f41 --- /dev/null +++ b/config/base/nginx/nginx.conf @@ -0,0 +1,40 @@ +user www-data; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; +events {} + +env AUTH_REQUIRED; +error_log /dev/stdout info; + +http { + + access_log /dev/stdout; + + upstream backend_server { + server projects:8080; + } + + server { # main server + + listen 8089; + + location ~* ^/(projects|experiments|runs|registered-models|model-versions|deployments|artifacts) { + include /etc/nginx/conf.d/cors.conf; + proxy_pass http://backend_server; + } + + } + + server { + + listen 5000-5100; + + location / { + resolver 127.0.0.11 ipv6=off; + proxy_pass http://projects:$server_port; + } + + } + +} \ No newline at end of file diff --git a/config/credentials/.gitignore b/config/credentials/.gitignore new file mode 100644 index 0000000..b722e9e --- /dev/null +++ b/config/credentials/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file