fix(rootfs): refactor database init process

When you complete a recovery of the database for the first time, a new log timeline
is started with an ID of 2. When you restore again, the timeline used when the last
backup occurred will be replayed. Because of this, if you restored the database and
did not perform a backup, all data from that successful recovery will be lost because
only WAL logs from the first timeline (the timeline that the database was last backed
up) will be restored.

In order to fix this, after completing a database recovery we create a fresh backup
in order to establish a new recovery baseline. That way we can now replay from
timeline 2.

Other fixes that this refactor includes:

 - database attempting to start halfway through recovery
 - /bin/is_running identifying the database as running during recovery
 - cleaning up database boot script

rootfs/bin/backup

@@ -3,6 +3,8 @@
 export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-12h}
 export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
 
+until is_running; do sleep 1; done
+
 while true; do
     sleep "$BACKUP_FREQUENCY"
     echo "Performing a base backup..."

rootfs/bin/backup-initial

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+until is_running; do sleep 1; done
+
+while true; do
+    if [[ ! -f "$PGDATA/recovery.conf" ]] ; then
+        # Push a fresh backup so we have a new recovery baseline
+        echo "Performing an initial backup..."
+        envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"
+        echo "Backup has been completed."
+        break
+    fi
+    sleep 1
+done

rootfs/bin/is_running

@@ -3,5 +3,9 @@
 # fail fast
 set -e
 
-# check if database is running
-gosu postgres pg_ctl status
+if [[ -f "$PGDATA/recovery.conf" ]]; then
+  # postgres is in recovery mode, so we know it's not ready to accept incoming connections.
+  exit 1
+fi
+
+gosu postgres psql -l -t >/dev/null 2>&1

rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh

@@ -11,15 +11,10 @@ EOF
 chown -R postgres:postgres "$PGDATA"
 chmod 0700 "$PGDATA"
 
-# reboot the server for wal_level to be set before backing up
-echo "Rebooting postgres to enable archive mode"
-gosu postgres pg_ctl -D "$PGDATA" -w restart
-
 # check if there are any backups -- if so, let's restore
 # we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
 if [[ $(envdir "$WALE_ENVDIR" wal-e --terse backup-list | wc -l) -gt "1" ]]; then
   echo "Found backups. Restoring from backup..."
-  gosu postgres pg_ctl -D "$PGDATA" -w stop
   rm -rf "$PGDATA"
   envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA" LATEST
   cat << EOF > "$PGDATA/postgresql.conf"
@@ -48,17 +43,8 @@ host    all             all             0.0.0.0/0               md5
 EOF
   touch "$PGDATA/pg_ident.conf"
   echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA/recovery.conf"
-  chown -R postgres:postgres "$PGDATA"
-  chmod 0700 "$PGDATA"
-  gosu postgres pg_ctl -D "$PGDATA" \
-      -o "-c listen_addresses=''" \
-      -t 1200 \
-      -w start
 fi
 
-echo "Performing an initial backup..."
-gosu postgres envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"
-
 # ensure $PGDATA has the right permissions
 chown -R postgres:postgres "$PGDATA"
 chmod 0700 "$PGDATA"

rootfs/docker-entrypoint-initdb.d/004_run_backups.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
 
+# Create a fresh backup as a starting point
+gosu postgres backup-initial &
+
 # Run periodic backups in the background
 gosu postgres backup &

rootfs/docker-entrypoint.sh

@@ -80,21 +80,17 @@ if [ "$1" = 'postgres' ]; then
 		EOSQL
 		echo
 
+		gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
+
 		echo
 		for f in /docker-entrypoint-initdb.d/*; do
 			case "$f" in
 				*.sh)  echo "$0: running $f"; . "$f" ;;
-				*.sql)
-					echo "$0: running $f";
-					psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < "$f"
-					echo
-					;;
 				*)     echo "$0: ignoring $f" ;;
 			esac
 			echo
 		done
 
-		gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
 		set_listen_addresses '*'
 
 		echo