From 914b9449cc389b4afcd5b10622f1d6c640972bf8 Mon Sep 17 00:00:00 2001
From: TechieNK <nk2indian@gmail.com>
Date: Tue, 25 Jun 2024 21:54:48 +0530
Subject: [PATCH 1/2] Moving some code into the new files functions.sh and
 run-maintenance-scripts.sh

---
 _sources/scripts/functions.sh               |  57 ++++
 _sources/scripts/run-all.sh                 | 142 +---------
 _sources/scripts/run-maintenance-scripts.sh | 280 ++++++++++++++++++++
 _sources/scripts/update-docker-gateway.sh   |  20 +-
 4 files changed, 342 insertions(+), 157 deletions(-)
 create mode 100644 _sources/scripts/functions.sh
 create mode 100644 _sources/scripts/run-maintenance-scripts.sh

diff --git a/_sources/scripts/functions.sh b/_sources/scripts/functions.sh
new file mode 100644
index 00000000..ab62fd2f
--- /dev/null
+++ b/_sources/scripts/functions.sh
@@ -0,0 +1,57 @@
+# read variables from LocalSettings.php
+get_mediawiki_variable() {
+    php /getMediawikiSettings.php --variable="$1" --format="${2:-string}"
+}
+
+get_mediawiki_db_var() {
+    case $1 in
+        "wgDBtype")
+            I="type"
+            ;;
+        "wgDBserver")
+            I="host"
+            ;;
+        "wgDBname")
+            I="dbname"
+            ;;
+        "wgDBuser")
+            I="user"
+            ;;
+        "wgDBpassword")
+            I="password"
+            ;;
+        *)
+            echo "Unexpected variable name passed to the get_mediawiki_db_var() function: $1"
+            return
+    esac
+    VALUE=$(php /getMediawikiSettings.php --variable=wgDBservers --variableArrayIndex="[0,\"$I\"]" --format=string)
+    if [ -z "$VALUE" ]; then
+        VALUE=$(get_mediawiki_variable "$1")
+    fi
+    echo "$VALUE"
+}
+
+isTrue() {
+    case $1 in
+        "True" | "TRUE" | "true" | 1)
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+get_hostname_with_port() {
+    port=$(echo "$1" | grep ":" | cut -d":" -f2)
+    echo "$1:${port:-$2}"
+}
+
+make_dir_writable() {
+    find "$@" '(' -type f -o -type d ')' \
+       -not '(' '(' -user "$WWW_USER" -perm -u=w ')' -o \
+           '(' -group "$WWW_GROUP" -perm -g=w ')' -o \
+           '(' -perm -o=w ')' \
+         ')' \
+         -exec chgrp "$WWW_GROUP" {} \; -exec chmod g=rwX {} \;
+}
diff --git a/_sources/scripts/run-all.sh b/_sources/scripts/run-all.sh
index be1f0af2..429971d3 100755
--- a/_sources/scripts/run-all.sh
+++ b/_sources/scripts/run-all.sh
@@ -2,55 +2,7 @@
 
 set -x
 
-WG_CIRRUS_SEARCH_SERVER=$(get_mediawiki_cirrus_search_server)
-
-isTrue() {
-    case $1 in
-        "True" | "TRUE" | "true" | 1)
-            return 0
-            ;;
-        *)
-            return 1
-            ;;
-    esac
-}
-
-isFalse() {
-    case $1 in
-        "True" | "TRUE" | "true" | 1)
-            return 1
-            ;;
-        *)
-            return 0
-            ;;
-    esac
-}
-
-dir_is_writable() {
-  # Use -L to get information about the target of a symlink,
-  # not the link itself, as pointed out in the comments
-  INFO=( $(stat -L -c "0%a %G %U" "$1") )
-  PERM=${INFO[0]}
-  GROUP=${INFO[1]}
-  OWNER=${INFO[2]}
-
-  if (( ($PERM & 0002) != 0 )); then
-      # Everyone has write access
-      return 0
-  elif (( ($PERM & 0020) != 0 )); then
-      # Some group has write access.
-      # Is user in that group?
-      if [[ $GROUP == $WWW_GROUP ]]; then
-          return 0
-      fi
-  elif (( ($PERM & 0200) != 0 )); then
-      # The owner has write access.
-      # Does the user own the file?
-      [[ $WWW_USER == $OWNER ]] && return 0
-  fi
-
-  return 1
-}
+. /functions.sh
 
 # Symlink all extensions and skins (both bundled and user)
 /create-symlinks.sh
@@ -94,88 +46,6 @@ else
   chmod -R g=rwX $APACHE_LOG_DIR
 fi
 
-run_maintenance_scripts() {
-  # Iterate through all the .sh files in /maintenance-scripts/ directory
-  for maintenance_script in $(find /maintenance-scripts/ -maxdepth 1 -mindepth 1 -type f -name "*.sh"); do
-    script_name=$(basename "$maintenance_script")
-
-    # If the script's name starts with "mw_", run it with the run_mw_script function
-    if [[ "$script_name" == mw* ]]; then
-      run_mw_script "$script_name" &
-    else
-      # If the script's name doesn't start with "mw"
-      echo "Running $script_name with user $WWW_USER..."
-      nice -n 20 runuser -c "/maintenance-scripts/$script_name" -s /bin/bash "$WWW_USER" &
-    fi
-  done
-}
-
-# Naming convention:
-# Scripts with names starting with "mw_" have corresponding enable variables.
-# The enable variable is formed by converting the script's name to uppercase and replacing the first underscore with "_ENABLE_".
-# For example, the enable variable for "mw_sitemap_generator.sh" would be "MW_ENABLE_SITEMAP_GENERATOR".
-
-run_mw_script() {
-  sleep 3
-
-  # Process the script name and create the corresponding enable variable
-  local script_name="$1"
-  script_name_no_ext="${script_name%.*}"
-  script_name_upper=$(basename "$script_name_no_ext" | tr '[:lower:]' '[:upper:]')
-  local MW_ENABLE_VAR="${script_name_upper/_/_ENABLE_}"
-
-  if isTrue "${!MW_ENABLE_VAR}"; then
-    echo "Running $script_name with user $WWW_USER..."
-    nice -n 20 runuser -c "/maintenance-scripts/$script_name" -s /bin/bash "$WWW_USER"
-  else
-    echo >&2 "$script_name is disabled."
-  fi
-}
-
-
-waitdatabase() {
-  if isFalse "$USE_EXTERNAL_DB"; then
-    /wait-for-it.sh -t 60 db:3306
-  fi
-}
-
-waitelastic() {
-    if [ -n "$es_started" ]; then
-        return 0; # already started
-    fi
-
-    echo >&2 'Waiting for elasticsearch to start'
-    /wait-for-it.sh -t 60 "$WG_CIRRUS_SEARCH_SERVER"
-
-    for i in {300..0}; do
-        result=0
-        output=$(wget --timeout=1 -q -O - "http://$WG_CIRRUS_SEARCH_SERVER/_cat/health") || result=$?
-        if [[ "$result" = 0 && $(echo "$output"|awk '{ print $4 }') = "green" ]]; then
-            break
-        fi
-        if [ "$result" = 0 ]; then
-            echo >&2 "Waiting for elasticsearch health status changed from [$(echo "$output"|awk '{ print $4 }')] to [green]..."
-        else
-            echo >&2 'Waiting for elasticsearch to start...'
-        fi
-        sleep 1
-    done
-    if [ "$i" = 0 ]; then
-        echo >&2 'Elasticsearch is not ready for use'
-        echo "$output"
-        return 1
-    fi
-    echo >&2 'Elasticsearch started successfully'
-    es_started="1"
-    return 0
-}
-
-run_autoupdate () {
-    echo "Running auto-update..."
-    runuser -c "php maintenance/update.php --quick" -s /bin/bash "$WWW_USER"
-    echo "Auto-update completed"
-}
-
 config_subdir_wikis() {
     echo "Configuring subdirectory wikis..."
     /config-subdir-wikis.sh
@@ -202,12 +72,6 @@ check_mount_points () {
   fi
 }
 
-# Wait db
-waitdatabase
-
-# Pause setup until ElasticSearch starts running
-waitelastic
-
 # Check for `user-` prefixed mounts and bow out if not found
 check_mount_points
 
@@ -227,7 +91,9 @@ fi
 
 echo "Starting services..."
 
-run_maintenance_scripts &
+# Run maintenance scripts in background.
+touch "$WWW_ROOT/.maintenance"
+/run-maintenance-scripts.sh &
 
 echo "Checking permissions of $MW_VOLUME/sitemap..."
 if dir_is_writable "$MW_VOLUME/sitemap"; then
diff --git a/_sources/scripts/run-maintenance-scripts.sh b/_sources/scripts/run-maintenance-scripts.sh
new file mode 100644
index 00000000..89553489
--- /dev/null
+++ b/_sources/scripts/run-maintenance-scripts.sh
@@ -0,0 +1,280 @@
+#!/bin/bash
+
+sleep 0.02
+printf "\n\n===== run-maintenance-script.sh =====\n\n\n"
+
+set -x
+
+. /functions.sh
+
+WG_DB_TYPE=$(get_mediawiki_db_var wgDBtype)
+WG_DB_SERVER=$(get_mediawiki_db_var wgDBserver)
+WG_DB_NAME=$(get_mediawiki_db_var wgDBname)
+WG_DB_USER=$(get_mediawiki_db_var wgDBuser)
+WG_DB_PASSWORD=$(get_mediawiki_db_var wgDBpassword)
+WG_SQLITE_DATA_DIR=$(get_mediawiki_variable wgSQLiteDataDir)
+WG_SEARCH_TYPE=$(get_mediawiki_variable wgSearchType)
+WG_CIRRUS_SEARCH_SERVER=$(get_mediawiki_cirrus_search_server)
+VERSION_HASH=$(php /getMediawikiSettings.php --versions --format=md5)
+
+waitdatabase() {
+    if [ -n "$db_started" ]; then
+        return 0; # already started
+    fi
+
+    if [ "$WG_DB_TYPE" = "sqlite" ]; then
+        echo >&2 "SQLite database used"
+        db_started="3"
+        return 0
+    fi
+
+    if [ "$WG_DB_TYPE" != "mysql" ]; then
+        echo >&2 "Unsupported database type ($WG_DB_TYPE)"
+        rm "$WWW_ROOT/.maintenance"
+        exit 123
+    fi
+
+    echo >&2 "Waiting for database to start"
+    /wait-for-it.sh -t 86400 "$WG_DB_SERVER:3306"
+
+    mysql=( mysql -h "$WG_DB_SERVER" -u"$WG_DB_USER" -p"$WG_DB_PASSWORD" )
+
+    for i in {60..0}; do
+        if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then
+            db_started="1"
+            break
+        fi
+        echo >&2 'Waiting for database to start...'
+        sleep 1
+    done
+    if [ "$i" = 0 ]; then
+        echo >&2 'Could not connect to the database.'
+        return 1
+    fi
+    echo >&2 'Successfully connected to the database.'
+    return 0
+}
+
+# Pause setup until ElasticSearch starts running
+waitelastic() {
+    if [ -n "$es_started" ]; then
+        return 0; # already started
+    fi
+
+    echo >&2 'Waiting for elasticsearch to start'
+    /wait-for-it.sh -t 60 "$WG_CIRRUS_SEARCH_SERVER"
+
+    for i in {300..0}; do
+        result=0
+        output=$(wget --timeout=1 -q -O - "http://$WG_CIRRUS_SEARCH_SERVER/_cat/health") || result=$?
+        if [[ "$result" = 0 && $(echo "$output"|awk '{ print $4 }') = "green" ]]; then
+            break
+        fi
+        if [ "$result" = 0 ]; then
+            echo >&2 "Waiting for elasticsearch health status changed from [$(echo "$output"|awk '{ print $4 }')] to [green]..."
+        else
+            echo >&2 'Waiting for elasticsearch to start...'
+        fi
+        sleep 1
+    done
+    if [ "$i" = 0 ]; then
+        echo >&2 'Elasticsearch is not ready for use'
+        echo "$output"
+        return 1
+    fi
+    echo >&2 'Elasticsearch started successfully'
+    es_started="1"
+    return 0
+}
+
+get_tables_count() {
+    waitdatabase || {
+        return $?
+    }
+
+    if [ "3" = "$db_started" ]; then
+        # sqlite
+        find "$WG_SQLITE_DATA_DIR" -type f | wc -l
+        return 0
+    elif [ "1" = "$db_started" ]; then
+        db_user="$WG_DB_USER"
+        db_password="$WG_DB_PASSWORD"
+    else
+        db_user="$MW_DB_INSTALLDB_USER"
+        db_password="$MW_DB_INSTALLDB_PASS"
+    fi
+    mysql -h "$WG_DB_SERVER" -u"$db_user" -p"$db_password" -e "SELECT count(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '$WG_DB_NAME'" | sed -n 2p
+}
+
+run_maintenance_script_if_needed () {
+    if [ -f "$MW_VOLUME/$1.info" ]; then
+        update_info="$(cat "$MW_VOLUME/$1.info" 2>/dev/null)"
+    else
+        update_info=""
+    fi
+
+    if [[ "$update_info" != "$2" && -n "$2" || "$2" == "always" ]]; then
+        waitdatabase || {
+            return $?
+        }
+        if [[ "$1" == *CirrusSearch* ]]; then
+            waitelastic || {
+                return $?
+            }
+        fi
+
+        i=3
+        while [ -n "${!i}" ]
+        do
+            if [ ! -f "$(echo "${!i}" | awk '{print $1}')" ]; then
+                echo >&2 "Maintenance script does not exit: ${!i}"
+                return 0;
+            fi
+            echo >&2 "Run maintenance script: ${!i}"
+            runuser -c "php ${!i}" -s /bin/bash "$WWW_USER" || {
+                echo >&2 "An error occurred when the maintenance script ${!i} was running"
+                return $?
+            }
+            i=$((i+1))
+        done
+
+        echo >&2 "Successful updated: $2"
+        echo "$2" > "$MW_VOLUME/$1.info"
+    else
+        echo >&2 "$1 is up to date: $2."
+    fi
+}
+
+run_autoupdate () {
+    echo >&2 'Check for the need to run maintenance scripts'
+    ### maintenance/update.php
+    SMW_UPGRADE_KEY=$(php /getMediawikiSettings.php --SMWUpgradeKey)
+    run_maintenance_script_if_needed 'maintenance_update' "$MW_VERSION-$MW_CORE_VERSION-$MW_MAINTENANCE_UPDATE-$VERSION_HASH-$SMW_UPGRADE_KEY" \
+        'maintenance/update.php --quick' || {
+            echo >&2 "An error occurred when auto-update script was running"
+            return $?
+        }
+    # The SMW upgrade key can be changes after running update.php
+    NEW_SMW_UPGRADE_KEY=$(php /getMediawikiSettings.php --SMWUpgradeKey)
+    if [ "$SMW_UPGRADE_KEY" != "$NEW_SMW_UPGRADE_KEY" ]; then
+        SMW_UPGRADE_KEY="$NEW_SMW_UPGRADE_KEY"
+        # update the key without running the maintenance script
+        run_maintenance_script_if_needed 'maintenance_update' "$MW_VERSION-$MW_CORE_VERSION-$MW_MAINTENANCE_UPDATE-$VERSION_HASH-$SMW_UPGRADE_KEY"
+    fi
+
+    # Run incomplete SemanticMediawiki setup tasks
+    SMW_INCOMPLETE_TASKS=$(php /getMediawikiSettings.php --SWMIncompleteSetupTasks --format=space)
+    for task in $SMW_INCOMPLETE_TASKS
+    do
+        case $task in
+            smw-updateentitycollation-incomplete)
+                run_maintenance_script_if_needed 'maintenance_semantic_updateEntityCollation' "always" \
+                    'extensions/SemanticMediaWiki/maintenance/updateEntityCollation.php'
+                ;;
+            smw-updateentitycountmap-incomplete)
+                run_maintenance_script_if_needed 'maintenance_semantic_updateEntityCountMap' "always" \
+                    'extensions/SemanticMediaWiki/maintenance/updateEntityCountMap.php'
+                ;;
+            *)
+                echo >&2 "######## Unknown SMW maintenance setup task - $task ########"
+                ;;
+        esac
+    done
+
+    ### CirrusSearch
+    if [ "$WG_SEARCH_TYPE" == 'CirrusSearch' ]; then
+        run_maintenance_script_if_needed 'maintenance_CirrusSearch_updateConfig' "${EXTRA_MW_MAINTENANCE_CIRRUSSEARCH_UPDATECONFIG}${MW_MAINTENANCE_CIRRUSSEARCH_UPDATECONFIG}${MW_VERSION}" \
+            'extensions/CirrusSearch/maintenance/UpdateSearchIndexConfig.php --reindexAndRemoveOk --indexIdentifier now' && \
+        run_maintenance_script_if_needed 'maintenance_CirrusSearch_forceIndex' "${EXTRA_MW_MAINTENANCE_CIRRUSSEARCH_FORCEINDEX}${MW_MAINTENANCE_CIRRUSSEARCH_FORCEINDEX}${MW_VERSION}" \
+            'extensions/CirrusSearch/maintenance/ForceSearchIndex.php --skipLinks --indexOnSkip' \
+            'extensions/CirrusSearch/maintenance/ForceSearchIndex.php --skipParse'
+    fi
+
+    echo >&2 "Auto-update completed"
+}
+
+run_maintenance_scripts() {
+  # Iterate through all the .sh files in /maintenance-scripts/ directory
+  for maintenance_script in $(find /maintenance-scripts/ -maxdepth 1 -mindepth 1 -type f -name "*.sh"); do
+    script_name=$(basename "$maintenance_script")
+
+    # If the script's name starts with "mw_", run it with the run_mw_script function
+    if [[ "$script_name" == mw* ]]; then
+      run_mw_script "$script_name" &
+    else
+      # If the script's name doesn't start with "mw"
+      echo "Running $script_name with user $WWW_USER..."
+      nice -n 20 runuser -c "/maintenance-scripts/$script_name" -s /bin/bash "$WWW_USER" &
+    fi
+  done
+}
+
+# Naming convention:
+# Scripts with names starting with "mw_" have corresponding enable variables.
+# The enable variable is formed by converting the script's name to uppercase and replacing the first underscore with "_ENABLE_".
+# For example, the enable variable for "mw_sitemap_generator.sh" would be "MW_ENABLE_SITEMAP_GENERATOR".
+
+run_mw_script() {
+  sleep 3
+
+  # Process the script name and create the corresponding enable variable
+  local script_name="$1"
+  script_name_no_ext="${script_name%.*}"
+  script_name_upper=$(basename "$script_name_no_ext" | tr '[:lower:]' '[:upper:]')
+  local MW_ENABLE_VAR="${script_name_upper/_/_ENABLE_}"
+
+  if isTrue "${!MW_ENABLE_VAR}"; then
+    echo "Running $script_name with user $WWW_USER..."
+    nice -n 20 runuser -c "/maintenance-scripts/$script_name" -s /bin/bash "$WWW_USER"
+  else
+    echo >&2 "$script_name is disabled."
+  fi
+}
+
+########## Run maintenance scripts ##########
+echo "Checking for LocalSettings..."
+if [ -e "$MW_VOLUME/config/LocalSettings.php" ] || [ -e "$MW_VOLUME/config/CommonSettings.php" ]; then
+  if isTrue "$MW_AUTOUPDATE"; then
+      waitdatabase
+      rm "$WWW_ROOT/.maintenance"
+      run_autoupdate
+  else
+      rm "$WWW_ROOT/.maintenance"
+      echo "Auto update script is disabled, MW_AUTOUPDATE is $MW_AUTOUPDATE";
+  fi
+  run_maintenance_scripts
+else
+    rm "$WWW_ROOT/.maintenance"
+    set +x
+    echo "There is no LocalSettings.php/CommonSettings.php file"
+    n=6
+    while [ ! -e "$MW_VOLUME/config/LocalSettings.php" ] && [ ! -e "$MW_VOLUME/config/CommonSettings.php" ]; do
+        echo -n "#"
+        if [ $n -eq 0 ]; then
+            echo " There is no LocalSettings.php/CommonSettings.php file..."
+            n=6
+        else
+            ((n--))
+        fi
+        sleep 10
+    done
+
+    echo
+    echo "Found LocalSettings.php/CommonSettings.php file"
+    set -x
+    # reload variables
+    WG_DB_TYPE=$(get_mediawiki_db_var wgDBtype)
+    WG_DB_SERVER=$(get_mediawiki_db_var wgDBserver)
+    WG_DB_NAME=$(get_mediawiki_db_var wgDBname)
+    WG_DB_USER=$(get_mediawiki_db_var wgDBuser)
+    WG_DB_PASSWORD=$(get_mediawiki_db_var wgDBpassword)
+    WG_SQLITE_DATA_DIR=$(get_mediawiki_variable wgSQLiteDataDir)
+    WG_SEARCH_TYPE=$(get_mediawiki_variable wgSearchType)
+    WG_CIRRUS_SEARCH_SERVER=$(get_mediawiki_cirrus_search_server)
+    VERSION_HASH=$(php /getMediawikiSettings.php --versions --format=md5)
+
+    run_maintenance_scripts
+fi
+
+sleep 4
+printf "\n\n>>>>> run-maintenance-script.sh <<<<<\n\n\n"
\ No newline at end of file
diff --git a/_sources/scripts/update-docker-gateway.sh b/_sources/scripts/update-docker-gateway.sh
index 191bf8ac..a3fbca99 100644
--- a/_sources/scripts/update-docker-gateway.sh
+++ b/_sources/scripts/update-docker-gateway.sh
@@ -2,25 +2,7 @@
 
 set -x
 
-# read variables from LocalSettings.php
-get_mediawiki_variable () {
-    php /getMediawikiSettings.php --variable="$1" --format="${2:-string}"
-}
-
-get_docker_gateway () {
-  getent hosts "gateway.docker.internal" | awk '{ print $1 }'
-}
-
-isTrue() {
-    case $1 in
-        "True" | "TRUE" | "true" | 1)
-            return 0
-            ;;
-        *)
-            return 1
-            ;;
-    esac
-}
+. /functions.sh
 
 # Try to fetch gateway IP from extra host
 DOCKER_GATEWAY=$(get_docker_gateway)

From 02e0ac976cd9a8791418a2f40459ef593cb6e882 Mon Sep 17 00:00:00 2001
From: TechieNK <nk2indian@gmail.com>
Date: Tue, 2 Jul 2024 20:33:04 +0530
Subject: [PATCH 2/2] Fix typo

---
 _sources/scripts/run-maintenance-scripts.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/_sources/scripts/run-maintenance-scripts.sh b/_sources/scripts/run-maintenance-scripts.sh
index 89553489..522e4678 100644
--- a/_sources/scripts/run-maintenance-scripts.sh
+++ b/_sources/scripts/run-maintenance-scripts.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 sleep 0.02
-printf "\n\n===== run-maintenance-script.sh =====\n\n\n"
+printf "\n\n===== run-maintenance-scripts.sh =====\n\n\n"
 
 set -x
 
@@ -277,4 +277,4 @@ else
 fi
 
 sleep 4
-printf "\n\n>>>>> run-maintenance-script.sh <<<<<\n\n\n"
\ No newline at end of file
+printf "\n\n>>>>> run-maintenance-scripts.sh <<<<<\n\n\n"
\ No newline at end of file