Skip to content

Commit

Permalink
feat: add sqlite helper and some integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
brianmcgee committed Oct 1, 2024
1 parent 280b403 commit 31e10db
Show file tree
Hide file tree
Showing 8 changed files with 216 additions and 52 deletions.
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ module github.com/nix-community/go-nix
go 1.20

require (
github.com/adrg/xdg v0.5.0
github.com/alecthomas/kong v0.5.0
github.com/dgraph-io/badger/v3 v3.2103.2
github.com/mattn/go-sqlite3 v1.14.23
github.com/multiformats/go-multihash v0.2.1
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.9.0
)

require (
Expand All @@ -24,7 +26,6 @@ require (
github.com/google/flatbuffers v1.12.1 // indirect
github.com/klauspost/compress v1.12.3 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/mattn/go-sqlite3 v1.14.23 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
Expand All @@ -34,7 +35,7 @@ require (
go.opencensus.io v0.22.5 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.15.0 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.6 // indirect
)
13 changes: 8 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY=
github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4=
github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
Expand Down Expand Up @@ -91,8 +93,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -135,8 +138,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand All @@ -162,8 +165,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
6 changes: 2 additions & 4 deletions pkg/sqlite/binary_cache_v6/query.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ on conflict (url)
do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5
returning id;

-- todo should this be :many?
-- name: QueryCache :one
-- name: QueryCache :many
select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?;

-- name: InsertNar :exec
Expand All @@ -18,8 +17,7 @@ insert or replace into NARs(
-- name: InsertMissingNAR :exec
insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0);

-- todo should this be :many?
-- name: QueryNar :one
-- name: QueryNar :many
select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs
where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?));

Expand Down
94 changes: 62 additions & 32 deletions pkg/sqlite/binary_cache_v6/query.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/sqlite/fetcher_cache_v2/query.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- name: UpsertCache :exec
insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?);

-- name: QueryCache :one
-- name: QueryCache :many
select value, timestamp from Cache where domain = ? and key = ?;
28 changes: 22 additions & 6 deletions pkg/sqlite/fetcher_cache_v2/query.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 45 additions & 0 deletions pkg/sqlite/sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package sqlite

import (
"database/sql"
"fmt"

// enable the sqlite3 driver.
_ "github.com/mattn/go-sqlite3"
"github.com/nix-community/go-nix/pkg/sqlite/binary_cache_v6"
"github.com/nix-community/go-nix/pkg/sqlite/eval_cache_v5"
"github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2"
"github.com/nix-community/go-nix/pkg/sqlite/nix_v10"
)

func BinaryCacheV6(dsn string) (*sql.DB, *binary_cache_v6.Queries, error) {
db, err := sql.Open("sqlite3", dsn)
if err != nil {
return nil, nil, fmt.Errorf("failed to open database: %w", err)
}
return db, binary_cache_v6.New(db), nil
}

func EvalCacheV5(dsn string) (*sql.DB, *eval_cache_v5.Queries, error) {
db, err := sql.Open("sqlite3", dsn)
if err != nil {
return nil, nil, fmt.Errorf("failed to open database: %w", err)
}
return db, eval_cache_v5.New(db), nil
}

func FetcherCacheV2(dsn string) (*sql.DB, *fetcher_cache_v2.Queries, error) {
db, err := sql.Open("sqlite3", dsn)
if err != nil {
return nil, nil, fmt.Errorf("failed to open database: %w", err)
}
return db, fetcher_cache_v2.New(db), nil
}

func NixV10(dsn string) (*sql.DB, *nix_v10.Queries, error) {
db, err := sql.Open("sqlite3", dsn)
if err != nil {
return nil, nil, fmt.Errorf("failed to open database: %w", err)
}
return db, nix_v10.New(db), nil
}
71 changes: 71 additions & 0 deletions pkg/sqlite/sqlite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
//go:build integration

package sqlite

import (
"context"
"fmt"
"os/exec"
"testing"

"github.com/adrg/xdg"
"github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2"

"github.com/stretchr/testify/require"
)

func TestBinaryCacheV6(t *testing.T) {
as := require.New(t)

// open our user-specific binary cache db
path, err := xdg.CacheFile("nix/binary-cache-v6.sqlite")
as.NoError(err, "failed to resolve binary cache file")
as.FileExists(path)

// open the sqlite db
db, queries, err := BinaryCacheV6(fmt.Sprintf("file:%s?mode=ro", path))
as.NoError(err)
defer db.Close()

// perform a basic query, we aren't interested in the result
_, err = queries.QueryLastPurge(context.Background())
as.NoError(err)
}

func TestFetcherCacheV2(t *testing.T) {
as := require.New(t)

// open our user-specific binary cache db
path, err := xdg.CacheFile("nix/fetcher-cache-v2.sqlite")
as.NoError(err, "failed to resolve fetcher cache file")
as.FileExists(path)

// open the sqlite db
db, queries, err := FetcherCacheV2(fmt.Sprintf("file:%s?mode=ro", path))
as.NoError(err)
defer db.Close()

// perform a basic query, we aren't interested in the result
_, err = queries.QueryCache(context.Background(), fetcher_cache_v2.QueryCacheParams{})
as.NoError(err)
}

func TestNixV10(t *testing.T) {
as := require.New(t)

// pull down a known path
path := "/nix/store/kz5clxh7s1n0fnx6d37c1wc2cs9qm53q-hello-2.12.1"
as.NoError(exec.Command("nix", "build", "--no-link", "--refresh", path).Run(), "failed to pull hello path")

// open the sqlite db
db, queries, err := NixV10("file:/nix/var/nix/db/db.sqlite?mode=ro")
as.NoError(err)
defer db.Close()

// query the path we just pulled down
info, err := queries.QueryPathInfo(context.Background(), path)
as.NoError(err)
as.Equal("sha256:f8340af15f7996faded748bea9e2d0b82a6f7c96417b03f7fa8e1a6a873748e8", info.Hash)
as.Equal("/nix/store/qnavcbp5ydyd12asgz7rpr7is7hlswaz-hello-2.12.1.drv", info.Deriver.String)
as.Equal(int64(226560), info.Narsize.Int64)
}

0 comments on commit 31e10db

Please sign in to comment.