Add pgvectorscale

CHANGELOG.md

@@ -9,6 +9,9 @@ These are changes that will probably be included in the next release.
 
 ## [future release]
 
+* Include pgvectorscale
+* Include pgai
+
 ## [v1.7.1] - 2023-04-27
 
 * [Refactor base HA image](https://github.com/timescale/timescaledb-docker-ha/pull/355)

Dockerfile

@@ -304,6 +304,11 @@ RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do
 
 RUN chown -R postgres:postgres /usr/local/cargo
 
+# required to install dbgsym packages
+RUN set -ex; \
+    chgrp -R postgres /usr/lib/debug; \
+    chmod -R g+w /usr/lib/debug
+
 USER postgres
 
 ENV MAKEFLAGS=-j4
@@ -382,6 +387,13 @@ RUN set -ex; \
         TOOLKIT_VERSIONS="${TOOLKIT_VERSIONS}" \
         /build/scripts/install_extensions rust
 
+ARG PGVECTORSCALE_VERSIONS
+RUN set -ex; \
+    OSS_ONLY="${OSS_ONLY}" \
+    RUST_RELEASE="${RUST_RELEASE}" \
+    PGVECTORSCALE_VERSIONS="${PGVECTORSCALE_VERSIONS}" \
+    /build/scripts/install_extensions pgvectorscale
+
 USER root
 
 # All the tools that were built in the previous steps have their ownership set to postgres
@@ -471,6 +483,7 @@ RUN /build/scripts/install_extensions versions > /.image_config; \
     echo "PGBOUNCER_EXPORTER_VERSION=\"${PGBOUNCER_EXPORTER_VERSION}\"" >> /.image_config; \
     echo "PGBACKREST_EXPORTER_VERSION=\"${PGBACKREST_EXPORTER_VERSION}\"" >> /.image_config; \
     echo "PGAI_VERSION=\"${PGAI_VERSION}\"" >> /.image_config; \
+    echo "PGVECTORSCALE_VERSIONS=\"${PGVECTORSCALE_VERSIONS}\"" >> /.image_config; \
     echo "PG_MAJOR=\"${PG_MAJOR}\"" >> /.image_config; \
     echo "PG_VERSIONS=\"${PG_VERSIONS}\"" >> /.image_config; \
     echo "FROM=\"${DOCKER_FROM}\"" >> /.image_config; \

Makefile

@@ -12,6 +12,7 @@ PG_VERSIONS?=
 
 # Additional PostgreSQL extensions we want to include with specific version/commit tags
 PGAI_VERSION?=v0.1.0
+PGVECTORSCALE_VERSIONS?=all
 POSTGIS_VERSIONS?=3
 PG_AUTH_MON?=v2.0
 PG_STAT_MONITOR?=2.0.3
@@ -116,6 +117,7 @@ DOCKER_BUILD_COMMAND=docker build \
 					 --build-arg GITHUB_TIMESCALEDB_DOCKER_REF="$(GITHUB_TIMESCALEDB_DOCKER_REF)" \
 					 --build-arg INSTALL_METHOD="$(INSTALL_METHOD)" \
 					 --build-arg PGAI_VERSION="$(PGAI_VERSION)" \
+					 --build-arg PGVECTORSCALE_VERSIONS="$(PGVECTORSCALE_VERSIONS)" \
 					 --build-arg PG_AUTH_MON="$(PG_AUTH_MON)" \
 					 --build-arg PG_LOGERRORS="$(PG_LOGERRORS)" \
 					 --build-arg PG_MAJOR=$(PG_MAJOR) \
@@ -152,13 +154,15 @@ fast: PG_VERSIONS=16
 fast: POSTGIS_VERSIONS=
 fast: TOOLKIT_VERSIONS=
 fast: PROMSCALE_VERSIONS=
+fast: PGVECTORSCALE_VERSIONS=
 fast: build
 
 .PHONY: latest
 latest: ALL_VERSIONS=false
 latest: TIMESCALEDB_VERSIONS=latest
 latest: TOOLKIT_VERSIONS=latest
 latest: PROMSCALE_VERSIONS=latest
+latest: PGVECTORSCALE_VERSIONS=latest
 latest: build
 
 prune: # docker system prune -af

README.md

@@ -47,7 +47,7 @@ POSTGIS_VERSIONS="" make build
 
 For further environment variables that can be set, we point you to the [Makefile](Makefile) itself.
 
-For updating changes in versions for timescaledb, promscale, or toolkit, update `build_scripts/versions.yaml`
+For updating changes in versions for timescaledb, promscale, pgvectorscale, or toolkit, update `build_scripts/versions.yaml`
 
 ## Verify your work
 

build_scripts/install_extensions

@@ -19,8 +19,8 @@ SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
 
 what="$1"
 [ -z "$what" ] && what=all
-if [[ ! "$what" =~ ^versions|timescaledb|rust|all|promscale|toolkit$ ]]; then
-    echo "usage: $0 [-n] [versions|timescaledb|rust|all|promscale|toolkit]" >&2
+if [[ ! "$what" =~ ^versions|timescaledb|rust|all|promscale|pgvectorscale|toolkit$ ]]; then
+    echo "usage: $0 [-n] [versions|timescaledb|rust|all|promscale|pgvectorscale|toolkit]" >&2
     exit 1
 fi
 
@@ -30,6 +30,7 @@ versions)
     echo "TIMESCALEDB_VERSIONS=\"$TIMESCALEDB_VERSIONS\""
     echo "TOOLKIT_VERSIONS=\"$TOOLKIT_VERSIONS\""
     echo "PROMSCALE_VERSIONS=\"$PROMSCALE_VERSIONS\""
+    echo "PGVECTORSCALE_VERSIONS=\"$PGVECTORSCALE_VERSIONS\""
     ;;
 
 timescaledb | all)
@@ -39,6 +40,12 @@ timescaledb | all)
     if [ "$DRYRUN" != true ]; then timescaledb_post_install; fi
     ;;& # fallthrough to get rust as well if we're called with 'all'
 
+pgvectorscale | all)
+    for ver in $PGVECTORSCALE_VERSIONS; do
+        install_pgvectorscale "$ver"
+    done
+    ;;& # fallthrough to get rust as well if we're called with 'all'
+
 rust | all) install_rust_extensions;;
 
 promscale) TOOLKIT_VERSIONS="" install_rust_extensions;;

build_scripts/shared_install.sh

@@ -243,3 +243,36 @@ __SQL__
         done # for file
     done     # for pg
 }
+
+install_pgvectorscale() {
+    local version="$1" pg pkg=pgvectorscale unsupported_reason
+
+    for pg in $(available_pg_versions); do
+        unsupported_reason="$(supported_pgvectorscale "$pg" "$version")"
+        if [ -n "$unsupported_reason" ]; then
+            log "$pkg-$version: $unsupported_reason"
+            continue
+        fi
+
+        log "building $pkg-$version for pg$pg"
+
+        [[ "$DRYRUN" = true ]] && continue
+
+        (
+            set -ex
+
+            rm -rf /build/pgvectorscale
+            mkdir /build/pgvectorscale
+            cd /build/pgvectorscale
+
+            curl --silent \
+                 --fail \
+                 --location \
+                 --output artifact.zip \
+                 "https://github.com/timescale/pgvectorscale/releases/download/$version/pgvectorscale-$version-pg${pg}.zip"
+
+            unzip artifact.zip
+            dpkg --install --log=/build/pgvectorscale/dpkg.log --admindir=/build/pgvectorscale/ --force-depends --force-not-root --force-overwrite pgvectorscale*amd64.deb
+        )
+    done
+}

build_scripts/shared_versions.sh

@@ -179,6 +179,19 @@ supported_promscale() {
     version_is_supported promscale "$pg" "$ver"
 }
 
+supported_pgvectorscale() {
+    local pg="$1" ver="$2"
+
+    if [ "$ARCH" != amd64 ]; then echo "unsupported arch $ARCH"; return; fi
+
+    # just attempt the build for main/master/or other branch build
+    if [[ "$ver" = main || "$ver" = master || "$ver" =~ [a-z_-]*/[A-Za-z0-9_-]* ]]; then
+        return
+    fi
+
+    version_is_supported pgvectorscale "$pg" "$ver"
+}
+
 require_supported_arch() {
     if [[ "$ARCH" != amd64 && "$ARCH" != aarch64 ]]; then
         echo "unsupported architecture: $ARCH" >&2
@@ -189,3 +202,4 @@ require_supported_arch() {
 TIMESCALEDB_VERSIONS="$(requested_pkg_versions timescaledb "$TIMESCALEDB_VERSIONS")"
 TOOLKIT_VERSIONS="$(requested_pkg_versions toolkit "$TOOLKIT_VERSIONS")"
 PROMSCALE_VERSIONS="$(requested_pkg_versions promscale "$PROMSCALE_VERSIONS")"
+PGVECTORSCALE_VERSIONS="$(requested_pkg_versions pgvectorscale "$PGVECTORSCALE_VERSIONS")"

build_scripts/versions.yaml

@@ -182,3 +182,9 @@ toolkit:
   1.18.0:
     cargo-pgrx: 0.10.2
     pg-max: 16
+
+pgvectorscale:
+  0.2.0:
+    arch: amd64
+    pg-min: 15
+    pg-max: 16

cicd/shared.sh

@@ -61,6 +61,7 @@ check_base_components() {
 
     check_timescaledb "$pg" "$lib"
     check_promscale "$pg" "$lib"
+    check_pgvectorscale "$pg" "$lib"
     check_toolkit "$pg" "$lib"
     check_oss_extensions "$pg" "$lib"
     check_others "$pg" "$lib"
@@ -196,6 +197,41 @@ check_toolkit() {
     if [ "$found" = false ]; then error "no toolkit versions found for pg$pg"; fi
 }
 
+check_pgvectorscale() {
+    if [ -z "$PGVECTORSCALE_VERSIONS" ]; then return; fi
+    local pg="$1" lib="$2" found=false
+
+    if [ "$ARCH" != amd64 ]; then
+        # no arm64 pgvectorscale packages
+        return
+    fi
+
+    # record an empty version so we'll get an empty table row if we don't have any versions
+    record_ext_version pgvectorscale "$pg" ""
+
+    for ver in $PGVECTORSCALE_VERSIONS; do
+        if [[ "$ver" = master || "$ver" = main ]]; then
+            log "skipping looking for vectorscale-$ver"
+            continue
+        fi
+
+        if [ -s "$lib/vectorscale-$ver.so" ]; then
+            found=true
+            record_ext_version pgvectorscale "$pg" "$ver"
+        else
+            unsupported_reason="$(supported_pgvectorscale "$pg" "$ver")"
+            if [ -n "$unsupported_reason" ]; then
+                log "skipped: pgvectorscale-$ver: $unsupported_reason"
+            else
+                error "pgvectorscale-$ver not found for pg$pg"
+            fi
+        fi
+    done
+
+
+    if [[ "$found" = false && "$pg" -gt 14 ]]; then error "no pgvectorscale versions found for pg$pg"; fi
+}
+
 # this checks for other extensions that should always exist
 check_others() {
     local pg="$1" lib="$2" version status

cicd/version_info.sql

@@ -9,7 +9,7 @@ WITH versions(name, version) AS (
     FROM
         pg_available_extensions
     WHERE
-        name IN ('timescaledb', 'postgis', 'pg_prometheus', 'timescale_prometheus_extra', 'promscale', 'timescaledb_toolkit', 'timescale_analytics')
+        name IN ('timescaledb', 'postgis', 'pg_prometheus', 'timescale_prometheus_extra', 'promscale', 'vectorscale', 'ai', 'timescaledb_toolkit', 'timescale_analytics')
     UNION ALL
     SELECT
         'postgresql.version',
@@ -41,6 +41,14 @@ WITH versions(name, version) AS (
     WHERE
         name = 'promscale'
     UNION ALL
+    SELECT
+        'vectorscale.available_versions',
+        string_agg(version, ',' ORDER BY version)
+    FROM
+        pg_available_extension_versions
+    WHERE
+        name = 'vectorscale'
+    UNION ALL
     SELECT
         'timescaledb_toolkit.available_versions',
         string_agg(version, ',' ORDER BY version)