LEAF-4487 - large queries local server and code

LEAF_Request_Portal/sources/Form.php

@@ -3038,6 +3038,34 @@ private function parseBooleanQuery(string $query): string
         return implode(' ', $words);
     }
 
+    /**
+     * @param array $query - this is the decoded string used for query
+     * @return bool
+     */
+    private function isLargeQuery(array $query): bool
+    {
+
+        $externalProcessQuery = false;
+        // No limit parameter set
+        if (isset($query['limit']) && is_numeric($query['limit'])) {
+
+            // Limit is > 10,000 records
+            if ($query['limit'] > 10000) {
+
+                $externalProcessQuery = true;
+
+            // Limit is > 1000 and <= 10,000 records AND more than 10 indicators are requested
+            } else if ($query['limit'] > 1000 && !empty($query['getData']) && count($query['getData']) >= 10) {
+                $externalProcessQuery = true;
+            }
+        } else {
+            // no limit parameter set
+            $externalProcessQuery = true;
+        }
+
+        return $externalProcessQuery;
+    }
+
     /**
      * query parses a JSON formatted user query defined in formQuery.js.
      *
@@ -3054,6 +3082,14 @@ public function query(string $inQuery): mixed
             return 'Invalid query';
         }
 
+        $externalProcessQuery = $this->isLargeQuery($query);
+        // is this a processes worthy of the external process and that we are on not wanting to actually run this query here.
+        if ($externalProcessQuery === true && !empty($_SERVER['LEAF_Large_Queries']) && $_SERVER['LEAF_Large_Queries'] == 'pass_onto_large_query_server') {
+            ob_end_clean();
+            http_response_code(503);
+            exit();
+        }
+
         $joinSearchAllData = false;
         $joinSearchOrgchartEmployeeData = false;
         $filterActionable = false;

docker/docker-compose.yml

@@ -71,7 +71,42 @@ services:
             - php-fpm
         networks:
             - leaf
-
+    php-fpm-api:
+        env_file:
+            - ./env_files/globals_local.env
+            - ./env_files/secrets.env
+            - ./env_files/globals_leaf20_local.env
+        build:
+            context: ../
+            dockerfile: docker/phpapi/leaf_monlith.dockerfile
+            args:
+                - BUILD_UID=${BUILD_UID}
+                - SMTP_HOST=smtp4dev
+                - MAIL_HUB=smtp4dev
+        volumes:
+            - ./env_files/leaf.key:/etc/ssl/certs/leaf.key
+            - ./env_files/leaf.pem:/etc/ssl/certs/leaf.pem
+            - ../LEAF_Request_Portal/vafavicon.ico:/var/www/html/favicon.ico
+            - ../LEAF_Request_Portal/vafavicon.ico:/var/www/html/LEAF_Nexus/vafavicon.ico
+            # - ../launchpad:/var/www/html/launchpad  # Dev should echo prod as exact as possible.  Will be added later
+            - ../LEAF_Nexus:/var/www/html/LEAF_NationalNexus
+            - ../LEAF_Nexus:/var/www/html/LEAF_Nexus
+            - ../LEAF_Nexus:/var/www/html/Test_Nexus
+            - ../LEAF_Request_Portal:/var/www/html/LEAF_Request_Portal
+            - ../LEAF_Request_Portal:/var/www/html/Test_Request_Portal
+            - ../scripts:/var/www/scripts
+            - ../health_checks:/var/www/html/health_checks
+            - ../app:/var/www/html/app
+            - ../libs:/var/www/html/libs # temp step.  Will shift to below later
+            - ../docker/mysql/db:/var/www/db
+            - ./php/dev:/var/www/html/dev
+        networks:
+            - leaf-sql
+            - leaf
+        environment:
+            - REMOTE_USER=${REMOTE_USER}
+            - APACHE_RUN_USER=build_user
+
     smtp4dev:
         image: rnwood/smtp4dev
         env_file:
@@ -117,7 +152,6 @@ services:
         volumes:
             - "../LEAF-Automated-Tests/api-test-helper/:/app"
             - "../LEAF-Automated-Tests/API-tests/:/API-tests"
-
         ports:
             - "8000:8000"
         env_file:

docker/env_files/nginx_local.env

@@ -1,5 +1,10 @@
+NGINX_POD=nginx
+NGINX_API_POD=nginx-api
 LEAF_POD=php-fpm
+LEAF_API_POD=php-fpm-api
 SEL_POD=selenium-api
 HOST=host.docker.internal
 HTTP_PORT=8080
+HTTP_API_PORT=8081
+HTTP_API_LQ_PORT=8082
 HTTPS_PORT=8443

docker/nginx/leaf_nginx.conf.template

@@ -1,3 +1,8 @@
+upstream backend_api_servers{
+  server ${NGINX_POD}:${HTTP_API_PORT} max_fails=1 fail_timeout=5;
+  server ${NGINX_POD}:${HTTP_API_LQ_PORT} backup;
+}
+
 server {
     listen ${HTTP_PORT} default_server;
     #listen [::]:${HTTP_PORT};
@@ -6,6 +11,144 @@ server {
     return 301 https://$server_name$request_uri;
 }
 
+# Large query NGINX Server
+server {
+
+    listen ${HTTP_API_LQ_PORT} ;
+
+    client_max_body_size 20m;
+
+    error_log  /var/log/nginx/error.log;
+    access_log /var/log/nginx/access.log;
+
+    #no ports in any redirect.
+    port_in_redirect off;
+
+    server_name ${HOST};
+    root /var/www/html;
+    index index.php index.html index.htm;
+
+    location /health-check-api {
+        access_log off;
+        return 200 "ok api";
+        add_header Content-Type text/plain;
+    }
+
+    location ~ \.php$ {
+        try_files $uri $uri/ /index.php =404;
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_index index.php;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        #fixes timeouts
+        fastcgi_read_timeout 600;
+        include fastcgi_params;
+        gzip on;
+    }
+
+    location ~ /api/(.*?)$ {
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_split_path_info ^(.+\/api\/)(.*)$;
+        fastcgi_index index.php;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        fastcgi_param PATH_INFO $fastcgi_path_info;
+        fastcgi_param QUERY_STRING $query_string;
+        fastcgi_param REQUEST_METHOD $request_method;
+        fastcgi_param CONTENT_TYPE $content_type;
+        fastcgi_param CONTENT_LENGTH $content_length;
+        #fixes timeouts
+        fastcgi_read_timeout 3000;
+        fastcgi_send_timeout 3000;
+
+        # Give this info to php
+        fastcgi_param LEAF_Large_Queries "process_ran_on_large_query_server";
+        # Give this info to the browser, need to see if this is actually needed or just a nice to have to see
+        add_header LEAF_Large_Queries "process_ran_on_large_query_server" always;
+
+        include fastcgi_params;
+        gzip on;
+
+        # debuging
+        add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
+        add_header x-debug-path-info "$fastcgi_path_info" always;
+        add_header x-debug-args-info "$args" always;
+    }
+
+}
+
+#API server
+server {
+
+    listen ${HTTP_API_PORT};
+
+    client_max_body_size 20m;
+
+    error_log  /var/log/nginx/error.log;
+    access_log /var/log/nginx/access.log;
+
+    #no ports in any redirect.
+    port_in_redirect off;
+
+    server_name ${HOST};
+    root /var/www/html;
+    index index.php index.html index.htm;
+
+    location /health-check-api {
+        access_log off;
+        return 200 "ok api";
+        add_header Content-Type text/plain;
+    }
+
+    location ~ \.php$ {
+        try_files $uri $uri/ /index.php =404;
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_index index.php;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        #fixes timeouts
+        fastcgi_read_timeout 600;
+        include fastcgi_params;
+        gzip on;
+    }
+
+    location ~ /api/(.*?)$ {
+        fastcgi_pass ${LEAF_POD}:9000;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_split_path_info ^(.+\/api\/)(.*)$;
+        fastcgi_index index.php;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        fastcgi_param PATH_INFO $fastcgi_path_info;
+        fastcgi_param QUERY_STRING $query_string;
+        fastcgi_param REQUEST_METHOD $request_method;
+        fastcgi_param CONTENT_TYPE $content_type;
+        fastcgi_param CONTENT_LENGTH $content_length;
+        #fixes timeouts
+        fastcgi_read_timeout 600;
+
+        # Give this info to php
+        fastcgi_param LEAF_Large_Queries "pass_onto_large_query_server";
+        # Give this info to the browser, need to see if this is actually needed or just a nice to have to see
+        add_header LEAF_Large_Queries "pass_onto_large_query_server" always;
+
+        include fastcgi_params;
+        gzip on;
+
+
+
+        # debuging
+        add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
+        add_header x-debug-path-info "$fastcgi_path_info" always;
+        add_header x-debug-args-info "$args" always;
+    }
+
+}
+
+
 server {
     listen ${HTTPS_PORT} ssl http2 default_server;
     #listen [::]:${HTTPS_PORT} ssl http2 default_server;
@@ -36,6 +179,12 @@ server {
 
     }
 
+    location /health-check-server {
+        access_log off;
+        return 200 "ok server";
+        add_header Content-Type text/plain;
+    }
+
     location ~ \.php$ {
         try_files $uri $uri/ /index.php =404;
         fastcgi_pass ${LEAF_POD}:9000;
@@ -47,6 +196,7 @@ server {
         fastcgi_read_timeout 600;
         include fastcgi_params;
         gzip on;
+
     }
 
     # static files browser caching
@@ -111,26 +261,27 @@ server {
     }
 
     location ~ /api/(.*?)$ {
-        fastcgi_pass ${LEAF_POD}:9000;
-        fastcgi_buffers 16 16k;
-        fastcgi_buffer_size 32k;
-        fastcgi_split_path_info ^(.+\/api\/)(.*)$;
-        fastcgi_index index.php;
-        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-        fastcgi_param PATH_INFO $fastcgi_path_info;
-        fastcgi_param QUERY_STRING $query_string;
-        fastcgi_param REQUEST_METHOD $request_method;
-        fastcgi_param CONTENT_TYPE $content_type;
-        fastcgi_param CONTENT_LENGTH $content_length;
-        #fixes timeouts
-        fastcgi_read_timeout 600;
-        include fastcgi_params;
-        gzip on;
+        proxy_pass http://backend_api_servers;
 
-        # debuging
-        add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
-        add_header x-debug-path-info "$fastcgi_path_info" always;
-        add_header x-debug-args-info "$args" always;
+        #this is what is used to tell nginx that we want the next server
+        proxy_next_upstream error timeout http_503;
+        proxy_next_upstream_tries 2;
+        proxy_intercept_errors on;
+
+        proxy_set_header Host $host;
+        proxy_redirect     off;
+        proxy_set_header   X-Real-IP  $remote_addr;
+        proxy_set_header   X-Scheme   $scheme;
+        proxy_set_header   X-Forwarded-Proto https;
+        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Proto on;
+        proxy_set_header   REFERER $http_referer;
+        #proxy_http_version 1.1;
+        proxy_connect_timeout 600;
+        proxy_read_timeout 600;
+        proxy_buffer_size 16k;
+        proxy_buffers 32 8k;
+        proxy_busy_buffers_size 64k;
     }
 
     location /selenium-tests {

docker/php/php-fpm.d/www.conf

@@ -145,7 +145,7 @@ pm.max_spare_servers = 3
 ; This can be useful to work around memory leaks in 3rd party libraries. For
 ; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS.
 ; Default Value: 0
-;pm.max_requests = 500
+pm.max_requests = 1
 
 ; The URI to view the FPM status page. If this value is not set, no URI will be
 ; recognized as a status page. It shows the following information:

docker/phpapi/conf.d/php.ini

@@ -0,0 +1 @@
+upload_max_filesize = 20M

docker/phpapi/dev/phpinfo.php

@@ -0,0 +1,3 @@
+<?php
+
+phpinfo();