Merge pull request #23 from mlrepa/dev

Bugfix: Add config/ fodler

.gitignore

@@ -21,7 +21,6 @@ services/projects/examples/models/*
 
 ## configs
 .env
-config/*
 
 ## credentials
 .htpasswd

config/base/.env.gcp

@@ -0,0 +1,29 @@
+# Common
+WORKSPACE=[PATH-TO-WORKSPACE-FOLDER-ON-LOCAL-HOST]
+HOST_IP=0.0.0.0
+LOGLEVEL=DEBUG
+
+# Database
+PROJECTS_DB_NAME=projects_db
+DEPLOY_DB_NAME=deploy_db
+POSTGRES_USER=user
+POSTGRES_PASSWORD=passw
+DB_HOST=db
+DB_PORT=5432
+
+# Deploy
+GOOGLE_APPLICATION_CREDENTIALS=/home/config/credentials/<credentials.json>
+GCP_PROJECT=<your-gcp-projects-id>
+GCP_ZONE=us-east1-b
+GCP_MACHINE_TYPE=g1-small
+GCP_OS_IMAGE=cos-dev-84-13078-0-0
+GCP_BUCKET=<bucket>
+MODEL_DEPLOY_DOCKER_IMAGE=mlrepa/mlpanel-deploy-web:v0.2
+MODEL_DEPLOY_DEFAULT_PORT=5000
+MODEL_DEPLOY_FIREWALL_RULE=mlflow-deploy
+DEPLOY_SERVER_WORKERS=1
+
+# Projects
+ARTIFACT_STORE=[mlruns|gs://<bucket>]
+TRACKING_SERVER_PORTS=5000-5100
+TRACKING_SERVER_WORKERS=1

config/base/.env.local

@@ -0,0 +1,20 @@
+# Common
+WORKSPACE=[PATH-TO-WORKSPACE-FOLDER-ON-LOCAL-HOST]
+HOST_IP=0.0.0.0
+LOGLEVEL=DEBUG
+
+# Database
+PROJECTS_DB_NAME=projects_db
+DEPLOY_DB_NAME=deploy_db
+POSTGRES_USER=user
+POSTGRES_PASSWORD=passw
+DB_HOST=db
+DB_PORT=5432
+
+# Deploy
+DEPLOY_SERVER_WORKERS=1
+
+# Projects
+ARTIFACT_STORE=mlruns
+TRACKING_SERVER_PORTS=5000-5100
+TRACKING_SERVER_WORKERS=1

config/base/docker-compose.yaml

@@ -0,0 +1,80 @@
+version: '3.5'
+
+services:
+
+  db:
+    environment:
+      POSTGRES_USER: $POSTGRES_USER
+      POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+      PGDATA: "/var/lib/postgresql/data/pgdata"
+    image: postgres:12.2
+    container_name: mlpanel-database
+    expose:
+      - $DB_PORT
+    volumes:
+      - $WORKSPACE/ps_data:/var/lib/postgresql/data
+    networks:
+      - mlpanel_default
+
+  mlpanel:
+    image: mlrepa/mlpanel-base-web:v0.2
+    container_name: mlpanel-base-web
+    ports:
+      - 8089:8089
+      - 5000-5100:5000-5100
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/conf.d:/etc/nginx/conf.d
+    depends_on:
+      - projects
+    networks:
+      - mlpanel_default
+
+  projects:
+    env_file:
+      - .env
+    image: mlrepa/mlpanel-base-projects:v0.2
+    container_name: mlpanel-base-projects
+    expose:
+      - 8080
+      - 5000-5100
+    volumes:
+      - ../../config/credentials:/home/config/credentials
+      - ../../services/projects:/home/projects
+      - ../../common:/home/common
+      - $WORKSPACE:$WORKSPACE
+    depends_on:
+      - db
+    networks:
+      - mlpanel_default
+
+  deploy:
+    env_file:
+      - .env
+    image: mlrepa/mlpanel-base-deploy:v0.2
+    container_name: mlpanel-base-deploy
+    expose:
+      - 9000
+    volumes:
+      - ../../config/credentials:/home/config/credentials
+      - ../../services/deploy:/home/deploy
+      - ../../common:/home/common
+      - $WORKSPACE:$WORKSPACE
+    depends_on:
+      - db
+    networks:
+      - mlpanel_default
+
+  ui:
+    environment:
+      HOST_IP: $HOST_IP
+    image: mlrepa/mlpanel-base-ui:v0.2
+    container_name: mlpanel-base-ui
+    ports:
+    - 3000:3000
+    networks:
+      - mlpanel_default
+
+networks:
+  mlpanel_default:
+    name: mlpanel_default

config/base/docker/base/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.7-slim
+
+RUN apt-get update && \
+    apt-get install -y curl gcc sudo postgresql-client && \
+    useradd -m user -u 1000 && \
+    echo 'user:user' | chpasswd user && \
+    echo "user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/user && \
+    chmod 0440 /etc/sudoers.d/user && \
+    chown -R user /home && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./requirements.txt /tmp/requirements.txt
+
+RUN pip install -r /tmp/requirements.txt
+
+ENV PYTHONPATH=/home
+
+USER user

config/base/docker/base/requirements.txt

@@ -0,0 +1,12 @@
+aiofiles==0.5.0
+email-validator==1.1.0
+fastapi==0.54.1
+google-cloud-storage==1.28.0
+jinja2==2.11.2
+mlflow==1.6.0
+psutil==5.7.0
+psycopg2-binary==2.8.5
+pytest==5.4.1
+python-multipart==0.0.5
+pyyaml==5.3.1
+uvicorn==0.11.5

config/base/docker/deploy/Dockerfile

@@ -0,0 +1,17 @@
+FROM mlrepa/mlpanel-base:latest
+
+# Google Cloud SDK installation
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz \
+  && mkdir -p /home/user/gcloud \
+  && tar -C /home/user/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+  && /home/user/gcloud/google-cloud-sdk/install.sh -q
+# Adding the package path to local
+ENV PATH $PATH:/home/user/gcloud/google-cloud-sdk/bin
+
+
+COPY ./requirements.txt /tmp/requirements.txt
+RUN sudo pip install -r /tmp/requirements.txt
+
+WORKDIR /home/deploy
+
+CMD chmod +x startup.sh && ./startup.sh

config/base/docker/deploy/requirements.txt

@@ -0,0 +1,4 @@
+google-api-python-client==1.8.2
+oauth2client==3.0.0
+requests==2.23.0
+scikit-learn==0.22.2

config/base/docker/projects/Dockerfile

@@ -0,0 +1,5 @@
+FROM mlrepa/mlpanel-base:latest
+
+WORKDIR /home/projects
+
+CMD chmod +x startup.sh && ./startup.sh

config/base/docker/ui/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:16.04
+
+# Install NodeJS
+RUN apt-get update && \
+    apt-get install -y git curl python3-software-properties && \
+    curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
+    apt-get install -y nodejs
+
+# Clone UI repository
+RUN cd /home && \
+    git clone https://github.com/mlrepa/mlpanel-ui.git && \
+    cd mlpanel-ui && \
+    npm install
+
+WORKDIR /home/mlpanel-ui
+
+CMD git pull origin master && export REACT_APP_API_URL=http://${HOST_IP}:8089/ && npm start

config/base/docker/web/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository -y ppa:nginx/stable && \
+    apt-get update && \
+    apt-get install -y nginx-extras && \
+    rm -rf /var/cache/apt && rm -rf /var/lib/apt
+
+
+CMD nginx -g 'daemon off;'

config/base/nginx/.gitignore

@@ -0,0 +1 @@
+!conf.d

config/base/nginx/conf.d/cors.conf

@@ -0,0 +1,8 @@
+add_header "Access-Control-Allow-Origin" * always;
+if ($request_method = 'OPTIONS') {
+    add_header "Access-Control-Allow-Origin" *;
+    add_header Access-Control-Allow-Headers "Cache-Control,If-Match,If-None-Match,If-Modified-Since,If-Unmodified-Since,If-Range,Range,Authorization,Content-Type,Link,Slug";
+    add_header Access-Control-Allow-Methods "OPTIONS,HEAD,GET,PATCH,POST,PUT,DELETE,PROPFIND,PROPPATCH,MKCOL,COPY,MOVE,LOCK,UNLOCK";
+    add_header Access-Control-Max-Age 60;
+    return 204;
+}

config/base/nginx/nginx.conf

@@ -0,0 +1,40 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+events {}
+
+env AUTH_REQUIRED;
+error_log /dev/stdout info;
+
+http {
+
+  access_log /dev/stdout;
+
+  upstream backend_server {
+    server projects:8080;
+  }
+
+  server {  # main server
+
+    listen 8089;
+
+    location ~* ^/(projects|experiments|runs|registered-models|model-versions|deployments|artifacts) {
+      include /etc/nginx/conf.d/cors.conf;
+      proxy_pass http://backend_server;
+    }
+
+  }
+
+  server {
+
+    listen 5000-5100;
+
+    location / {
+      resolver 127.0.0.11 ipv6=off;
+      proxy_pass http://projects:$server_port;
+    }
+
+  }
+
+}

config/credentials/.gitignore

@@ -0,0 +1 @@
+!.gitignore