Fork

.github/workflows/docker-publish.yml

@@ -15,38 +15,38 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v4
 
-      - name: Build and potentially push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          # ensure latest base image is used
-          pull: true
-          push: false
-          tags: adaguc-server
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/[email protected]
-        with:
-          image-ref: 'adaguc-server'
-          format: 'table'
-          exit-code: '1'
-          ignore-unfixed: true
-          vuln-type: 'os,library'
-          trivyignores: .trivyignore
-#          severity: 'CRITICAL,HIGH'
-  docker-build-and-publish:
-    name: Build Docker image and potentially push to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
+#       - name: Build and potentially push Docker image
+#         uses: docker/build-push-action@v5
+#         with:
+#           context: .
+#           # ensure latest base image is used
+#           pull: true
+#           push: false
+#           tags: adaguc-server
+#       - name: Run Trivy vulnerability scanner
+#         uses: aquasecurity/[email protected]
+#         with:
+#           image-ref: 'adaguc-server'
+#           format: 'table'
+#           exit-code: '1'
+#           ignore-unfixed: true
+#           vuln-type: 'os,library'
+#           trivyignores: .trivyignore
+# #          severity: 'CRITICAL,HIGH'
+#   docker-build-and-publish:
+#     name: Build Docker image and potentially push to Docker Hub
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Check out the repo
+#         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: 'arm64'
+#       - name: Set up QEMU
+#         uses: docker/setup-qemu-action@v3
+#         with:
+#           platforms: 'arm64'
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+#       - name: Set up Docker Buildx
+#         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
         if: github.repository == 'KNMI/adaguc-server'
@@ -65,9 +65,9 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           # ensure latest base image is used
           pull: true
           push: ${{ github.repository == 'KNMI/adaguc-server' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.meta.outputs.tags }}-20240911T1446
           labels: ${{ steps.meta.outputs.labels }}

Docker/start.sh

@@ -26,7 +26,7 @@ done
 
 export ADAGUC_PATH=/adaguc/adaguc-server-master/ && \
 export ADAGUC_TMP=/tmp && \
-/adaguc/adaguc-server-master/bin/adagucserver --updatedb \
+env -u ADAGUC_FORK /adaguc/adaguc-server-master/bin/adagucserver --updatedb \
   --config /adaguc/adaguc-server-config.xml,baselayers.xml
 
 if [ $? -ne 0 ]

Docker/supervisord/adaguc-pgbouncer.conf

@@ -9,6 +9,17 @@ stdout_logfile_maxbytes=0
 redirect_stderr=true
 command=/adaguc/start.sh
 
+[program:adagucbin]
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+environment=
+    ADAGUC_FORK=1,
+    ADAGUC_PATH=/adaguc/adaguc-server-master/,
+    ADAGUC_TMP=/tmp,
+    ADAGUC_CONFIG=/adaguc/adaguc-server-master/python/lib/adaguc/adaguc-server-config-python-postgres.xml
+command=/adaguc/adaguc-server-master/bin/adagucserver
+
 [program:pgbouncer]
 stdout_logfile=/dev/fd/2
 stdout_logfile_maxbytes=0

Docker/supervisord/adaguc.conf

@@ -3,6 +3,17 @@ nodaemon=true
 logfile=/dev/null
 logfile_maxbytes=0
 
+[program:adagucbin]
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+environment=
+    ADAGUC_FORK=1,
+    ADAGUC_PATH=/adaguc/adaguc-server-master/,
+    ADAGUC_TMP=/tmp,
+    ADAGUC_CONFIG=/adaguc/adaguc-server-master/python/lib/adaguc/adaguc-server-config-python-postgres.xml
+command=/adaguc/adaguc-server-master/bin/adagucserver
+
 [program:adaguc]
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0

Dockerfile

@@ -139,12 +139,12 @@ ENV PYTHONPATH=${ADAGUC_PATH}/python/python_fastapi_server
 # Build and test adaguc python support
 WORKDIR /adaguc/adaguc-server-master/python/lib/
 RUN python3 setup.py install
-RUN bash -c "python3 /adaguc/adaguc-server-master/python/examples/runautowms/run.py && ls result.png"
+# RUN bash -c "python3 /adaguc/adaguc-server-master/python/examples/runautowms/run.py && ls result.png"
 
 WORKDIR /adaguc/adaguc-server-master
 
 # This checks if the test stage has ran without issues.
-COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt
+# COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt
 
 USER adaguc
 

adagucserverEC/CImageDataWriter.cpp

@@ -2563,6 +2563,9 @@ int CImageDataWriter::end() {
       resetErrors();
 
       printf("%s", resultHTML.c_str());
+
+      fflush(stdout);
+      fflush(stderr);
     } /*End of text html */
 
     /* Text XML */

adagucserverEC/CMakeLists.txt

@@ -182,6 +182,8 @@ add_library(
     Types/ProjectionStore.h
     Types/ProjectionStore.cpp 
     testadagucserver.cpp
+    fork_server.h
+    fork_server.cpp
 )
 
 target_include_directories(adagucserverEC PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${Cairo_INCLUDE_DIRS} ${FREETYPE_INCLUDE_DIRS} ${PostgreSQL_INCLUDE_DIRS} ${GDAL_INCLUDE_DIRS} ${PROJ_INCLUDE_DIR})

adagucserverEC/CRequest.cpp

@@ -802,6 +802,9 @@ int CRequest::process_wms_getcap_request() {
     printf("%s", XMLdocument.c_str());
   }
 
+  fflush(stdout);
+  fflush(stderr);
+
   return 0;
 }
 

adagucserverEC/adagucserver.cpp

@@ -30,6 +30,13 @@
 #include <getopt.h>
 
 #include "ProjCache.h"
+#include "fork_server.h"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <stdio.h>
+#include <signal.h>
+#include <unistd.h>
 
 DEF_ERRORMAIN();
 
@@ -197,9 +204,9 @@ int _main(int argc, char **argv, char **) {
   /* Initialize error functions */
   seterrormode(EXCEPTIONS_PLAINTEXT);
   setStatusCode(HTTP_STATUSCODE_200_OK);
-  setErrorFunction(serverLogFunctionCMDLine);
-  setWarningFunction(serverLogFunctionCMDLine);
-  setDebugFunction(serverLogFunctionCMDLine);
+  // setErrorFunction(serverLogFunctionCMDLine);
+  // setWarningFunction(serverLogFunctionCMDLine);
+  // setDebugFunction(serverLogFunctionCMDLine);
 
   int opt;
   int scanFlags = 0;
@@ -375,25 +382,25 @@ int _main(int argc, char **argv, char **) {
   }
 
   /* Process the OGC request */
-  setErrorFunction(serverErrorFunction);
-  setWarningFunction(serverWarningFunction);
-  setDebugFunction(serverDebugFunction);
+  // setErrorFunction(serverErrorFunction);
+  // setWarningFunction(serverWarningFunction);
+  // setDebugFunction(serverDebugFunction);
 
-#ifdef MEASURETIME
+  // #ifdef MEASURETIME
   StopWatch_Start();
-#endif
+  // #endif
 
   status = runRequest();
   /* Display errors if any */
   readyerror();
-#ifdef MEASURETIME
+  // #ifdef MEASURETIME
   StopWatch_Stop("Ready!!!");
-#endif
+  // #endif
 
   return getStatusCode();
 }
 
-int main(int argc, char **argv, char **envp) {
+int do_work(int argc, char **argv, char **envp) {
   /* Check if ADAGUC_LOGFILE is set */
   const char *ADAGUC_LOGFILE = getenv("ADAGUC_LOGFILE");
   if (ADAGUC_LOGFILE != NULL) {
@@ -458,6 +465,23 @@ int main(int argc, char **argv, char **envp) {
     fclose(pLogDebugFile);
     pLogDebugFile = NULL;
   }
+  close(1);
+  close(2);
 
   return status;
 }
+
+int main(int argc, char **argv, char **envp) {
+  // If these lines are commented out, the calls the /edr/collections/instances/<my-instance> fail to return data
+  // because the call to `request=getreferencetimes` does not contain useful output
+  setvbuf(stdout, NULL, _IONBF, 0); // turn off buffering
+  setvbuf(stderr, NULL, _IONBF, 0); // turn off buffering
+
+  const char *ADAGUC_FORK = getenv("ADAGUC_FORK");
+  if (ADAGUC_FORK != NULL) {
+    return run_server(do_work, argc, argv, envp);
+  } else {
+    // normal flow without unix socket server/fork
+    return do_work(argc, argv, envp);
+  }
+}

adagucserverEC/fork_server.cpp

@@ -0,0 +1,122 @@
+#include <map>
+#include <signal.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "CTString.h"
+#include "fork_server.h"
+
+static std::map<pid_t, int> child_sockets;
+static const unsigned int max_pending_connections = 5;
+
+void handle_client(int client_socket, int (*do_work)(int, char **, char **), int argc, char **argv, char **envp) {
+  int recv_buf_len = 4096;
+  char recv_buf[recv_buf_len];
+  memset(recv_buf, 0, recv_buf_len * sizeof(char));
+
+  int data_recv = recv(client_socket, recv_buf, recv_buf_len, 0);
+  if (data_recv > 0) {
+    // The child stdout should end up in the client socket
+    dup2(client_socket, STDOUT_FILENO);
+
+    setenv("QUERY_STRING", recv_buf, 1);
+
+    int status = do_work(argc, argv, envp);
+    // fprintf(stderr, "exiting, status=%d", status);
+
+    // fflush(stdout);
+    // fflush(stderr);
+
+    exit(0);
+  }
+}
+
+void on_child_exit(int child_signal) {
+  /*
+  This function gets executed once a child exits/signals (through SIGCHLD)
+  Note: the kernel does not queue signals. If a child exits during the handling of another signal, the exit/signal gets dropped.
+  */
+
+  int stat_val;
+  pid_t child_pid;
+
+  // Loop over all exited children. WNOHANG makes the call non-blocking.
+  while ((child_pid = waitpid(-1, &stat_val, WNOHANG)) > 0) {
+    // If child exited normally
+    if (WIFEXITED(stat_val)) {
+      int child_status = WEXITSTATUS(stat_val);
+
+      int child_sock = child_sockets[child_pid];
+      fprintf(stderr, "@ on_child_exit [%d] [%d] [%d] [%d]\n", child_pid, child_sock, child_signal, child_status);
+
+      // Write the status code from the child pid into the unix socket back to python
+      write(child_sock, reinterpret_cast<const char *>(&child_status), sizeof(child_status));
+      close(child_sock);
+
+      child_sockets.erase(child_pid);
+    } else if (WIFSIGNALED(stat_val)) {
+      int child_signal2 = WTERMSIG(stat_val);
+      fprintf(stderr, "@ on_child_signal [%d] [%d]\n", child_pid, child_signal2);
+    }
+  }
+}
+
+int run_server(int (*do_work)(int, char **, char **), int argc, char **argv, char **envp) {
+  int client_socket = 0;
+
+  signal(SIGCHLD, on_child_exit);
+
+  struct sockaddr_un local, remote;
+  int len = 0;
+
+  int listen_socket = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (-1 == listen_socket) {
+    printf("Error on socket() call \n");
+    return 1;
+  }
+
+  local.sun_family = AF_UNIX;
+
+  CT::string socket_path(getenv("ADAGUC_PATH"));
+  socket_path.concat("/adaguc.socket");
+
+  strncpy(local.sun_path, socket_path.c_str(), sizeof(local.sun_path));
+  local.sun_path[sizeof(local.sun_path) - 1] = '\0';
+
+  // Remove old adaguc.socket file
+  unlink(local.sun_path);
+
+  // Bind name to the "local" socket
+  len = strlen(local.sun_path) + sizeof(local.sun_family) + 1;
+  if (bind(listen_socket, (struct sockaddr *)&local, len) != 0) {
+    printf("Error on binding socket \n");
+    return 1;
+  }
+
+  if (listen(listen_socket, max_pending_connections) != 0) {
+    printf("Error on listen call \n");
+    return 1;
+  }
+
+  while (1) {
+    unsigned int sock_len = 0;
+
+    // Once someone connects to the unix socket, immediately fork and execute the client request in `handle_client`
+    if ((client_socket = accept(listen_socket, (struct sockaddr *)&remote, &sock_len)) == -1) {
+      printf("Error on accept() call \n");
+      return 1;
+    }
+
+    pid_t pid = fork();
+    if (pid == 0) {
+      close(listen_socket);
+      handle_client(client_socket, do_work, argc, argv, envp);
+    } else {
+      child_sockets[pid] = client_socket;
+    }
+  }
+
+  return 0;
+}

adagucserverEC/fork_server.h

@@ -0,0 +1,8 @@
+#ifndef ADAGUC_SERVER_FORK_SERVER_H
+#define ADAGUC_SERVER_FORK_SERVER_H
+
+void handle_client(int client_socket, int (*do_work)(int, char **, char **), int argc, char **argv, char **envp);
+void on_child_exit(int child_signal);
+int run_server(int (*do_work)(int, char **, char **), int argc, char **argv, char **envp);
+
+#endif // ADAGUC_SERVER_FORK_SERVER_H

hclasses/CDebugger.cpp

@@ -49,7 +49,8 @@ void printErrorStream(const char *message) { _printErrorStreamPointer(message);
 
 void _printErrorStream(const char *pszMessage) { fprintf(stderr, "%s", pszMessage); }
 void _printWarningStream(const char *pszMessage) { fprintf(stderr, "%s", pszMessage); }
-void _printDebugStream(const char *pszMessage) { printf("%s", pszMessage); }
+// void _printDebugStream(const char *pszMessage) { printf("%s", pszMessage); }
+void _printDebugStream(const char *pszMessage) { fprintf(stderr, "%s", pszMessage); }
 
 void _printDebugLine(const char *pszMessage, ...) {
   logMessageNumber++;