LEAF-4487 - large queries local server and code

LEAF_Request_Portal/sources/Form.php

@@ -3016,6 +3016,34 @@ private function parseBooleanQuery(string $query): string
         return implode(' ', $words);
     }
 
+    /**
+     * @param array $query - this is the decoded string used for query
+     * @return bool
+     */
+    private function isLargeQuery(array $query): bool
+    {
+
+        $externalProcessQuery = false;
+        // No limit parameter set
+        if (isset($query['limit']) && is_numeric($query['limit'])) {
+
+            // Limit is > 10,000 records
+            if ($query['limit'] > 10000) {
+
+                $externalProcessQuery = true;
+
+            // Limit is > 1000 and <= 10,000 records AND more than 10 indicators are requested
+            } else if ($query['limit'] > 1000 && !empty($query['getData']) && count($query['getData']) > 10) {
+                $externalProcessQuery = true;
+            }
+        } else {
+            // no limit parameter set
+            $externalProcessQuery = true;
+        }
+
+        return $externalProcessQuery;
+    }
+
     /**
      * query parses a JSON formatted user query defined in formQuery.js.
      *
@@ -3032,6 +3060,14 @@ public function query(string $inQuery): mixed
             return 'Invalid query';
         }
 
+        $externalProcessQuery = $this->isLargeQuery($query);
+        // is this a processes worthy of the external process and that we are on not wanting to actually run this query here.
+        if ($externalProcessQuery === true && !empty($_SERVER['LEAF_Large_Queries']) && $_SERVER['LEAF_Large_Queries'] == 'pass_onto_large_query_server') {
+            ob_end_clean();
+            http_response_code(503);
+            exit();
+        }
+
         $joinSearchAllData = false;
         $joinSearchOrgchartEmployeeData = false;
         $filterActionable = false;

docker/docker-compose.yml

@@ -64,13 +64,89 @@ services:
             - ./nginx/leaf_nginx.conf.template:/etc/nginx/templates/default.conf.template
         volumes_from:
             - php-fpm
+        #ports:
+        #    - 8081:${HTTP_PORT}
+        #    - 443:${HTTPS_PORT}
+        depends_on:
+            - php-fpm
+        networks:
+            - leaf
+    php-fpm-api:
+        env_file:
+            - ./env_files/globals_local.env
+            - ./env_files/secrets.env
+            - ./env_files/globals_leaf20_local.env
+        build:
+            context: ../
+            dockerfile: docker/phpapi/leaf_monlith.dockerfile
+            args:
+                - BUILD_UID=${BUILD_UID}
+                - SMTP_HOST=smtp4dev
+                - MAIL_HUB=smtp4dev
+        volumes:
+            - ./env_files/leaf.key:/etc/ssl/certs/leaf.key
+            - ./env_files/leaf.pem:/etc/ssl/certs/leaf.pem
+            - ../LEAF_Request_Portal/vafavicon.ico:/var/www/html/favicon.ico
+            - ../LEAF_Request_Portal/vafavicon.ico:/var/www/html/LEAF_Nexus/vafavicon.ico
+            # - ../launchpad:/var/www/html/launchpad  # Dev should echo prod as exact as possible.  Will be added later
+            - ../LEAF_Nexus:/var/www/html/LEAF_NationalNexus
+            - ../LEAF_Nexus:/var/www/html/LEAF_Nexus
+            - ../LEAF_Nexus:/var/www/html/Test_Nexus
+            - ../LEAF_Request_Portal:/var/www/html/LEAF_Request_Portal
+            - ../LEAF_Request_Portal:/var/www/html/Test_Request_Portal
+            - ../scripts:/var/www/scripts
+            - ../health_checks:/var/www/html/health_checks
+            - ../app:/var/www/html/app
+            - ../libs:/var/www/html/libs # temp step.  Will shift to below later
+            - ../docker/mysql/db:/var/www/db
+            - ./php/dev:/var/www/html/dev
+        networks:
+            - leaf-sql
+            - leaf
+        environment:
+            - REMOTE_USER=${REMOTE_USER}
+            - APACHE_RUN_USER=build_user
+
+    nginx-api:
+        env_file:
+            - ./env_files/nginx_local.env
+        build:
+            context: ../
+            dockerfile: docker/nginxapi/leaf_nginx.dockerfile
+        volumes:
+            - ./env_files/leaf.key:/etc/ssl/certs/leaf.key
+            - ./env_files/leaf.pem:/etc/ssl/certs/leaf.pem
+            - ./nginx/src:/var/www/html
+            - ./nginxapi/leaf_nginx.conf.template:/etc/nginx/templates/default.conf.template
+        volumes_from:
+            - php-fpm
+        #ports:
+        #    - 8088:${HTTP_PORT}
+        #    - 443:${HTTPS_PORT}
+        depends_on:
+            - php-fpm
+            - nginx
+        networks:
+            - leaf
+    rpnginx:
+        env_file:
+            - ./env_files/nginx_local.env
+        build:
+            context: ../
+            dockerfile: docker/rpnginx/leaf_nginx.dockerfile
+        volumes:
+            - ./env_files/leaf.key:/etc/ssl/certs/leaf.key
+            - ./env_files/leaf.pem:/etc/ssl/certs/leaf.pem
+            - ./nginx/src:/var/www/html
+            - ./rpnginx/leaf_nginx.conf.template:/etc/nginx/templates/default.conf.template
+        volumes_from:
+            - php-fpm
         ports:
             - 80:${HTTP_PORT}
             - 443:${HTTPS_PORT}
-        depends_on:
-            - php-fpm
         networks:
             - leaf
+
 
     smtp4dev:
         image: rnwood/smtp4dev
@@ -117,6 +193,7 @@ services:
         volumes:
             - "../x-test/api-test-helper/:/app"
             - "../x-test/API-tests/:/API-tests"
+            - "../x-test/loadrandomdata/:/loadrandomdata"
         ports:
             - "8000:8000"
         env_file:

docker/env_files/nginx_local.env

@@ -1,4 +1,7 @@
+NGINX_POD=nginx
+NGINX_API_POD=nginx-api
 LEAF_POD=php-fpm
+LEAF_API_POD=php-fpm-api
 SEL_POD=selenium-api
 HOST=host.docker.internal
 HTTP_PORT=8080

docker/nginx/leaf_nginx.conf.template

@@ -1,19 +1,20 @@
-server {
-    listen ${HTTP_PORT} default_server;
-    #listen [::]:${HTTP_PORT};
-    server_name ${HOST};
-    root /var/www/html;
-    return 301 https://$server_name$request_uri;
-}
+#server {
+#    listen ${HTTP_PORT} default_server;
+#    #listen [::]:${HTTP_PORT};
+#    server_name ${HOST};
+#    root /var/www/html;
+#    return 301 https://$server_name$request_uri;
+#}
 
 server {
-    listen ${HTTPS_PORT} ssl http2 default_server;
+    #listen ${HTTPS_PORT} ssl http2 default_server;
     #listen [::]:${HTTPS_PORT} ssl http2 default_server;
+    listen ${HTTP_PORT} default_server;
 
-    ssl_certificate /etc/ssl/certs/leaf.pem;
-    ssl_certificate_key /etc/ssl/certs/leaf.key;
-    ssl_protocols TLSv1.3;
-    ssl_ciphers HIGH:!aNULL:!MD5;
+    #ssl_certificate /etc/ssl/certs/leaf.pem;
+    #ssl_certificate_key /etc/ssl/certs/leaf.key;
+    #ssl_protocols TLSv1.3;
+    #ssl_ciphers HIGH:!aNULL:!MD5;
 
     client_max_body_size 20m;
 
@@ -36,6 +37,12 @@ server {
 
     }
 
+    location /health-check-server {
+        access_log off;
+        return 200 "ok server";
+        add_header Content-Type text/plain;
+    }
+
     location ~ \.php$ {
         try_files $uri $uri/ /index.php =404;
         fastcgi_pass ${LEAF_POD}:9000;
@@ -47,6 +54,7 @@ server {
         fastcgi_read_timeout 600;
         include fastcgi_params;
         gzip on;
+
     }
 
     # static files browser caching
@@ -124,9 +132,17 @@ server {
         fastcgi_param CONTENT_LENGTH $content_length;
         #fixes timeouts
         fastcgi_read_timeout 600;
+
+        # Give this info to php
+        fastcgi_param LEAF_Large_Queries "pass_onto_large_query_server";
+        # Give this info to the browser, need to see if this is actually needed or just a nice to have to see
+        add_header LEAF_Large_Queries "pass_onto_large_query_server" always;
+
         include fastcgi_params;
         gzip on;
 
+
+
         # debuging
         add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
         add_header x-debug-path-info "$fastcgi_path_info" always;

docker/nginxapi/leaf_nginx.conf.template

@@ -0,0 +1,151 @@
+#server {
+#    listen ${HTTP_PORT} default_server;
+#    #listen [::]:${HTTP_PORT};
+#    server_name ${HOST};
+#    root /var/www/html;
+#    return 301 https://$server_name$request_uri;
+#}
+
+server {
+    #listen ${HTTPS_PORT} ssl http2 default_server;
+    #listen [::]:${HTTPS_PORT} ssl http2 default_server;
+    listen ${HTTP_PORT} default_server;
+
+    #ssl_certificate /etc/ssl/certs/leaf.pem;
+    #ssl_certificate_key /etc/ssl/certs/leaf.key;
+    #ssl_protocols TLSv1.3;
+    #ssl_ciphers HIGH:!aNULL:!MD5;
+
+    client_max_body_size 20m;
+
+    error_log  /var/log/nginx/error.log;
+    access_log /var/log/nginx/access.log;
+
+    #no ports in any redirect.
+    port_in_redirect off;
+
+    server_name ${HOST};
+    root /var/www/html;
+    index index.php index.html index.htm;
+
+    #this is for everything except gallery.php and svgs since those need to load still
+    location ~* ^(.*)/libs/dynicons(?!(/gallery.php|/(.*).svg)){
+
+        #absolute_redirect off;
+
+        rewrite ^/(.*)/libs/dynicons(.*)$ /$1/dynicons;# permanent;
+
+    }
+
+    location /health-check-api {
+        access_log off;
+        return 200 "ok api";
+        add_header Content-Type text/plain;
+    }
+
+    location ~ \.php$ {
+        try_files $uri $uri/ /index.php =404;
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_index index.php;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        #fixes timeouts
+        fastcgi_read_timeout 600;
+        include fastcgi_params;
+        gzip on;
+    }
+
+    # static files browser caching
+    location ~* \.(jpg|jpeg|png|gif|ico)$ {
+        #how long a browser should wait on checking for new cache
+        expires 1m;
+        #leave these out of the access log, who cares if someone access the leaf logo at 1245 pm
+        access_log off;
+        add_header Vary Accept-Encoding;
+        #send the whole file
+        #tcp_nodelay off;
+        #If many changes to the file it can cause the file not to load, waiting the 45s between file changes keeps it
+        #from failing. This should not be too much of an issue on live and helps perf
+        #open_file_cache max=3000 inactive=120s;
+        #open_file_cache_valid 45s;
+        #open_file_cache_min_uses 2;
+        #open_file_cache_errors off;
+        #the magic sauce to bust a cache
+        etag on;
+    }
+
+    # static files browser caching that impact client runtime
+    location ~* \.(css|js)$ {
+        #leave these out of the access log, who cares if someone access the leaf logo at 1245 pm
+        access_log off;
+        add_header Vary Accept-Encoding;
+        #send the whole file
+        #tcp_nodelay off;
+        #If many changes to the file it can cause the file not to load, waiting the 45s between file changes keeps it
+        #from failing. This should not be too much of an issue on live and helps perf
+        #open_file_cache max=3000 inactive=120s;
+        #open_file_cache_valid 45s;
+        #open_file_cache_min_uses 2;
+        #open_file_cache_errors off;
+        #the magic sauce to bust a cache
+        etag on;
+        add_header Cache-Control "max-age=0, must-revalidate";
+    }
+
+    location ~ /api/v2/(.*?)$ {
+        set $api_directory /var/www/html/app/Api/v2;
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_split_path_info ^(.+\/api\/v2\/)(.*)$;
+        fastcgi_index index.php;
+        fastcgi_param SCRIPT_FILENAME $api_directory/index.php;
+        fastcgi_param PATH_INFO $fastcgi_path_info;
+        fastcgi_param QUERY_STRING $query_string;
+        fastcgi_param REQUEST_METHOD $request_method;
+        fastcgi_param CONTENT_TYPE $content_type;
+        fastcgi_param CONTENT_LENGTH $content_length;
+        #fixes timeouts
+        fastcgi_read_timeout 3000;
+        fastcgi_send_timeout 3000;
+        include fastcgi_params;
+        gzip on;
+
+        # debuging
+        add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
+        add_header x-debug-path-info "$fastcgi_path_info" always;
+        add_header x-debug-args-info "$args" always;
+    }
+
+    location ~ /api/(.*?)$ {
+        fastcgi_pass ${LEAF_API_POD}:9000;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_split_path_info ^(.+\/api\/)(.*)$;
+        fastcgi_index index.php;
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        fastcgi_param PATH_INFO $fastcgi_path_info;
+        fastcgi_param QUERY_STRING $query_string;
+        fastcgi_param REQUEST_METHOD $request_method;
+        fastcgi_param CONTENT_TYPE $content_type;
+        fastcgi_param CONTENT_LENGTH $content_length;
+        #fixes timeouts
+        fastcgi_read_timeout 3000;
+        fastcgi_send_timeout 3000;
+
+        # Give this info to php
+        fastcgi_param LEAF_Large_Queries "process_ran_on_large_query_server";
+        # Give this info to the browser, need to see if this is actually needed or just a nice to have to see
+        add_header LEAF_Large_Queries "process_ran_on_large_query_server" always;
+
+        include fastcgi_params;
+        gzip on;
+
+        # debuging
+        add_header x-debug-root-script "$document_root$fastcgi_script_name" always;
+        add_header x-debug-path-info "$fastcgi_path_info" always;
+        add_header x-debug-args-info "$args" always;
+    }
+
+}

docker/nginxapi/leaf_nginx.dockerfile

@@ -0,0 +1,5 @@
+FROM nginxinc/nginx-unprivileged:1.22-alpine
+COPY ./docker/nginxapi/leaf_nginx.conf.template /etc/nginx/templates/default.conf.template
+COPY ./docker/nginx/src/index.html /var/www/html/index.html
+
+USER 1001

docker/php/php-fpm.d/www.conf

@@ -145,7 +145,7 @@ pm.max_spare_servers = 3
 ; This can be useful to work around memory leaks in 3rd party libraries. For
 ; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS.
 ; Default Value: 0
-;pm.max_requests = 500
+pm.max_requests = 1
 
 ; The URI to view the FPM status page. If this value is not set, no URI will be
 ; recognized as a status page. It shows the following information: