From e017cfc7b4602610401a96b32f627421d9fb53b6 Mon Sep 17 00:00:00 2001 From: Matvey Arye Date: Thu, 6 Jun 2024 10:27:31 -0400 Subject: [PATCH] Add pgvectorscale --- CHANGELOG.md | 3 +++ Dockerfile | 13 ++++++++++++ Makefile | 4 ++++ README.md | 2 +- build_scripts/install_extensions | 11 ++++++++-- build_scripts/shared_install.sh | 33 +++++++++++++++++++++++++++++ build_scripts/shared_versions.sh | 14 +++++++++++++ build_scripts/versions.yaml | 6 ++++++ cicd/shared.sh | 36 ++++++++++++++++++++++++++++++++ cicd/version_info.sql | 10 ++++++++- 10 files changed, 128 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce07a4d..e92bde7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ These are changes that will probably be included in the next release. ## [future release] +* Include pgvectorscale +* Include pgai + ## [v1.7.1] - 2023-04-27 * [Refactor base HA image](https://github.com/timescale/timescaledb-docker-ha/pull/355) diff --git a/Dockerfile b/Dockerfile index 5b98e0ee..dddf2f10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -304,6 +304,11 @@ RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do RUN chown -R postgres:postgres /usr/local/cargo +# required to install dbgsym packages +RUN set -ex; \ + chgrp -R postgres /usr/lib/debug; \ + chmod -R g+w /usr/lib/debug + USER postgres ENV MAKEFLAGS=-j4 @@ -382,6 +387,13 @@ RUN set -ex; \ TOOLKIT_VERSIONS="${TOOLKIT_VERSIONS}" \ /build/scripts/install_extensions rust +ARG PGVECTORSCALE_VERSIONS +RUN set -ex; \ + OSS_ONLY="${OSS_ONLY}" \ + RUST_RELEASE="${RUST_RELEASE}" \ + PGVECTORSCALE_VERSIONS="${PGVECTORSCALE_VERSIONS}" \ + /build/scripts/install_extensions pgvectorscale + USER root # All the tools that were built in the previous steps have their ownership set to postgres @@ -471,6 +483,7 @@ RUN /build/scripts/install_extensions versions > /.image_config; \ echo "PGBOUNCER_EXPORTER_VERSION=\"${PGBOUNCER_EXPORTER_VERSION}\"" >> /.image_config; \ echo "PGBACKREST_EXPORTER_VERSION=\"${PGBACKREST_EXPORTER_VERSION}\"" >> /.image_config; \ echo "PGAI_VERSION=\"${PGAI_VERSION}\"" >> /.image_config; \ + echo "PGVECTORSCALE_VERSIONS=\"${PGVECTORSCALE_VERSIONS}\"" >> /.image_config; \ echo "PG_MAJOR=\"${PG_MAJOR}\"" >> /.image_config; \ echo "PG_VERSIONS=\"${PG_VERSIONS}\"" >> /.image_config; \ echo "FROM=\"${DOCKER_FROM}\"" >> /.image_config; \ diff --git a/Makefile b/Makefile index 2666d300..bfa17a2f 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ PG_VERSIONS?= # Additional PostgreSQL extensions we want to include with specific version/commit tags PGAI_VERSION?=v0.1.0 +PGVECTORSCALE_VERSIONS?=all POSTGIS_VERSIONS?=3 PG_AUTH_MON?=v2.0 PG_STAT_MONITOR?=2.0.3 @@ -116,6 +117,7 @@ DOCKER_BUILD_COMMAND=docker build \ --build-arg GITHUB_TIMESCALEDB_DOCKER_REF="$(GITHUB_TIMESCALEDB_DOCKER_REF)" \ --build-arg INSTALL_METHOD="$(INSTALL_METHOD)" \ --build-arg PGAI_VERSION="$(PGAI_VERSION)" \ + --build-arg PGVECTORSCALE_VERSIONS="$(PGVECTORSCALE_VERSIONS)" \ --build-arg PG_AUTH_MON="$(PG_AUTH_MON)" \ --build-arg PG_LOGERRORS="$(PG_LOGERRORS)" \ --build-arg PG_MAJOR=$(PG_MAJOR) \ @@ -152,6 +154,7 @@ fast: PG_VERSIONS=16 fast: POSTGIS_VERSIONS= fast: TOOLKIT_VERSIONS= fast: PROMSCALE_VERSIONS= +fast: PGVECTORSCALE_VERSIONS= fast: build .PHONY: latest @@ -159,6 +162,7 @@ latest: ALL_VERSIONS=false latest: TIMESCALEDB_VERSIONS=latest latest: TOOLKIT_VERSIONS=latest latest: PROMSCALE_VERSIONS=latest +latest: PGVECTORSCALE_VERSIONS=latest latest: build prune: # docker system prune -af diff --git a/README.md b/README.md index ef115a13..25916e06 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ POSTGIS_VERSIONS="" make build For further environment variables that can be set, we point you to the [Makefile](Makefile) itself. -For updating changes in versions for timescaledb, promscale, or toolkit, update `build_scripts/versions.yaml` +For updating changes in versions for timescaledb, promscale, pgvectorscale, or toolkit, update `build_scripts/versions.yaml` ## Verify your work diff --git a/build_scripts/install_extensions b/build_scripts/install_extensions index 6fd91acd..f53bb7f3 100755 --- a/build_scripts/install_extensions +++ b/build_scripts/install_extensions @@ -19,8 +19,8 @@ SCRIPT_DIR="${BASH_SOURCE[0]%/*}" what="$1" [ -z "$what" ] && what=all -if [[ ! "$what" =~ ^versions|timescaledb|rust|all|promscale|toolkit$ ]]; then - echo "usage: $0 [-n] [versions|timescaledb|rust|all|promscale|toolkit]" >&2 +if [[ ! "$what" =~ ^versions|timescaledb|rust|all|promscale|pgvectorscale|toolkit$ ]]; then + echo "usage: $0 [-n] [versions|timescaledb|rust|all|promscale|pgvectorscale|toolkit]" >&2 exit 1 fi @@ -30,6 +30,7 @@ versions) echo "TIMESCALEDB_VERSIONS=\"$TIMESCALEDB_VERSIONS\"" echo "TOOLKIT_VERSIONS=\"$TOOLKIT_VERSIONS\"" echo "PROMSCALE_VERSIONS=\"$PROMSCALE_VERSIONS\"" + echo "PGVECTORSCALE_VERSIONS=\"$PGVECTORSCALE_VERSIONS\"" ;; timescaledb | all) @@ -39,6 +40,12 @@ timescaledb | all) if [ "$DRYRUN" != true ]; then timescaledb_post_install; fi ;;& # fallthrough to get rust as well if we're called with 'all' +pgvectorscale | all) + for ver in $PGVECTORSCALE_VERSIONS; do + install_pgvectorscale "$ver" + done + ;;& # fallthrough to get rust as well if we're called with 'all' + rust | all) install_rust_extensions;; promscale) TOOLKIT_VERSIONS="" install_rust_extensions;; diff --git a/build_scripts/shared_install.sh b/build_scripts/shared_install.sh index 1b128f57..b3fd9a5d 100644 --- a/build_scripts/shared_install.sh +++ b/build_scripts/shared_install.sh @@ -243,3 +243,36 @@ __SQL__ done # for file done # for pg } + +install_pgvectorscale() { + local version="$1" pg pkg=pgvectorscale unsupported_reason + + for pg in $(available_pg_versions); do + unsupported_reason="$(supported_pgvectorscale "$pg" "$version")" + if [ -n "$unsupported_reason" ]; then + log "$pkg-$version: $unsupported_reason" + continue + fi + + log "building $pkg-$version for pg$pg" + + [[ "$DRYRUN" = true ]] && continue + + ( + set -ex + + rm -rf /build/pgvectorscale + mkdir /build/pgvectorscale + cd /build/pgvectorscale + + curl --silent \ + --fail \ + --location \ + --output artifact.zip \ + "https://github.com/timescale/pgvectorscale/releases/download/$version/pgvectorscale-$version-pg${pg}.zip" + + unzip artifact.zip + dpkg --install --log=/build/pgvectorscale/dpkg.log --admindir=/build/pgvectorscale/ --force-depends --force-not-root --force-overwrite pgvectorscale*amd64.deb + ) + done +} \ No newline at end of file diff --git a/build_scripts/shared_versions.sh b/build_scripts/shared_versions.sh index ff16287f..8fdfa4a3 100644 --- a/build_scripts/shared_versions.sh +++ b/build_scripts/shared_versions.sh @@ -179,6 +179,19 @@ supported_promscale() { version_is_supported promscale "$pg" "$ver" } +supported_pgvectorscale() { + local pg="$1" ver="$2" + + if [ "$ARCH" != amd64 ]; then echo "unsupported arch $ARCH"; return; fi + + # just attempt the build for main/master/or other branch build + if [[ "$ver" = main || "$ver" = master || "$ver" =~ [a-z_-]*/[A-Za-z0-9_-]* ]]; then + return + fi + + version_is_supported pgvectorscale "$pg" "$ver" +} + require_supported_arch() { if [[ "$ARCH" != amd64 && "$ARCH" != aarch64 ]]; then echo "unsupported architecture: $ARCH" >&2 @@ -189,3 +202,4 @@ require_supported_arch() { TIMESCALEDB_VERSIONS="$(requested_pkg_versions timescaledb "$TIMESCALEDB_VERSIONS")" TOOLKIT_VERSIONS="$(requested_pkg_versions toolkit "$TOOLKIT_VERSIONS")" PROMSCALE_VERSIONS="$(requested_pkg_versions promscale "$PROMSCALE_VERSIONS")" +PGVECTORSCALE_VERSIONS="$(requested_pkg_versions pgvectorscale "$PGVECTORSCALE_VERSIONS")" diff --git a/build_scripts/versions.yaml b/build_scripts/versions.yaml index 19051776..f311eb9c 100644 --- a/build_scripts/versions.yaml +++ b/build_scripts/versions.yaml @@ -182,3 +182,9 @@ toolkit: 1.18.0: cargo-pgrx: 0.10.2 pg-max: 16 + +pgvectorscale: + 0.2.0: + arch: amd64 + pg-min: 15 + pg-max: 16 diff --git a/cicd/shared.sh b/cicd/shared.sh index b689900a..8f6735e1 100644 --- a/cicd/shared.sh +++ b/cicd/shared.sh @@ -61,6 +61,7 @@ check_base_components() { check_timescaledb "$pg" "$lib" check_promscale "$pg" "$lib" + check_pgvectorscale "$pg" "$lib" check_toolkit "$pg" "$lib" check_oss_extensions "$pg" "$lib" check_others "$pg" "$lib" @@ -196,6 +197,41 @@ check_toolkit() { if [ "$found" = false ]; then error "no toolkit versions found for pg$pg"; fi } +check_pgvectorscale() { + if [ -z "$PGVECTORSCALE_VERSIONS" ]; then return; fi + local pg="$1" lib="$2" found=false + + if [ "$ARCH" != amd64 ]; then + # no arm64 pgvectorscale packages + return + fi + + # record an empty version so we'll get an empty table row if we don't have any versions + record_ext_version pgvectorscale "$pg" "" + + for ver in $PGVECTORSCALE_VERSIONS; do + if [[ "$ver" = master || "$ver" = main ]]; then + log "skipping looking for vectorscale-$ver" + continue + fi + + if [ -s "$lib/vectorscale-$ver.so" ]; then + found=true + record_ext_version pgvectorscale "$pg" "$ver" + else + unsupported_reason="$(supported_pgvectorscale "$pg" "$ver")" + if [ -n "$unsupported_reason" ]; then + log "skipped: pgvectorscale-$ver: $unsupported_reason" + else + error "pgvectorscale-$ver not found for pg$pg" + fi + fi + done + + + if [[ "$found" = false && "$pg" -gt 14 ]]; then error "no pgvectorscale versions found for pg$pg"; fi +} + # this checks for other extensions that should always exist check_others() { local pg="$1" lib="$2" version status diff --git a/cicd/version_info.sql b/cicd/version_info.sql index e98e8010..1d734ac6 100644 --- a/cicd/version_info.sql +++ b/cicd/version_info.sql @@ -9,7 +9,7 @@ WITH versions(name, version) AS ( FROM pg_available_extensions WHERE - name IN ('timescaledb', 'postgis', 'pg_prometheus', 'timescale_prometheus_extra', 'promscale', 'timescaledb_toolkit', 'timescale_analytics') + name IN ('timescaledb', 'postgis', 'pg_prometheus', 'timescale_prometheus_extra', 'promscale', 'vectorscale', 'ai', 'timescaledb_toolkit', 'timescale_analytics') UNION ALL SELECT 'postgresql.version', @@ -41,6 +41,14 @@ WITH versions(name, version) AS ( WHERE name = 'promscale' UNION ALL + SELECT + 'vectorscale.available_versions', + string_agg(version, ',' ORDER BY version) + FROM + pg_available_extension_versions + WHERE + name = 'vectorscale' + UNION ALL SELECT 'timescaledb_toolkit.available_versions', string_agg(version, ',' ORDER BY version)