diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..c370f1f --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +watch_file devshell.nix + +if nix flake metadata &>/dev/null; then + use flake +else + use nix +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c66077..b87ad82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,40 +1,40 @@ name: CI on: - push: - branches: - - main - pull_request: + push: + branches: + - main + pull_request: jobs: - fixtures: - name: fixtures-up-to-date - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v22 - with: - install_url: https://releases.nixos.org/nix/nix-2.12.1/install - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - name: Build fixtures - run: bash -c 'cd test/testdata && ./build-fixtures.go' - - name: Diff fixtures - run: git diff --exit-code test/testdata + fixtures: + name: fixtures-up-to-date + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 + with: + install_url: https://releases.nixos.org/nix/nix-2.12.1/install + - uses: actions/setup-go@v4 + with: + go-version: "1.21" + - name: Build fixtures + run: bash -c 'cd test/testdata && ./build-fixtures.go' + - name: Diff fixtures + run: git diff --exit-code test/testdata - build: - strategy: - matrix: - os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - go: [ '1.20', '1.21' ] - runs-on: ${{ matrix.os }} + build: + strategy: + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + go: ["1.20", "1.21"] + runs-on: ${{ matrix.os }} - name: Build (Go ${{ matrix.go }}, OS ${{ matrix.os }}) - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go }} - - name: go test -race -bench='.+' -v ./... - run: go test -race -bench='.+' -v ./... + name: Build (Go ${{ matrix.go }}, OS ${{ matrix.os }}) + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go }} + - name: go test -race -bench='.+' -v ./... + run: go test -race -bench='.+' -v ./... diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 3446c84..eda0e12 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -1,41 +1,41 @@ name: golangci-lint on: - push: - tags: - - v* - branches: - - main - pull_request: + push: + tags: + - v* + branches: + - main + pull_request: permissions: - contents: read - # Optional: allow read access to pull request. Use with `only-new-issues` option. - # pull-requests: read + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read jobs: - golangci: - name: lint - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 + golangci: + name: lint + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 - # Optional: working directory, useful for monorepos - # working-directory: somedir + # Optional: working directory, useful for monorepos + # working-directory: somedir - # Optional: golangci-lint command line arguments. - args: --timeout=10m + # Optional: golangci-lint command line arguments. + args: --timeout=10m - # Optional: show only new issues if it's a pull request. The default value is `false`. - # only-new-issues: true + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true - # Optional: if set to true then the action will use pre-installed Go. - # skip-go-installation: true + # Optional: if set to true then the action will use pre-installed Go. + # skip-go-installation: true - # Optional: if set to true then the action don't cache or restore ~/go/pkg. - # skip-pkg-cache: true + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true - # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. - # skip-build-cache: true + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true diff --git a/.golangci.yml b/.golangci.yml index 22a0f00..7cdee4b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,38 +1,38 @@ linters: - enable: - - errname - - exhaustive - - gci - - gochecknoglobals - - gochecknoinits - - goconst - - godot - - gofumpt - - goheader - - goimports - - gosec - - importas - - ireturn - - lll - - makezero - - misspell - - nakedret - - nestif - - nilerr - - nilnil - - nlreturn - - noctx - - nolintlint - - prealloc - - predeclared - - revive - - rowserrcheck - - stylecheck - - tagliatelle - - tenv - - testpackage - - unconvert - - unparam - - wastedassign - - whitespace - - wsl \ No newline at end of file + enable: + - errname + - exhaustive + - gci + - gochecknoglobals + - gochecknoinits + - goconst + - godot + - gofumpt + - goheader + - goimports + - gosec + - importas + - ireturn + - lll + - makezero + - misspell + - nakedret + - nestif + - nilerr + - nilnil + - nlreturn + - noctx + - nolintlint + - prealloc + - predeclared + - revive + - rowserrcheck + - stylecheck + - tagliatelle + - tenv + - testpackage + - unconvert + - unparam + - wastedassign + - whitespace + - wsl diff --git a/README.md b/README.md index 3ade6df..4d84641 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # go-nix - Nix experiments written in go -*STATUS*: experimental +_STATUS_: experimental This repository holds a bunch of experiments written in Go. ## `cmd/gonix` + A command line entrypoint called `gonix`, currently implementing the nar {cat,dump-path,ls} commands. @@ -12,36 +13,68 @@ They're not meant to be 100% compatible, but are documented in the `--help` output. ## `pkg/derivation` + A parser for Nix `.drv` files. Functions to calculate derivation paths and output hashes. ## `pkg/derivation/store` + A Structure to hold derivation graphs. ## `pkg/hash` + Methods to serialize and deserialize some of the hashes used in nix code and `.narinfo` files. ## `pkg/nar` + A Nix ARchive (NAR) file Reader and Writer, with an interface similar to `archive/tar` from the stdlib, as well as a `DumpPath` method, which will assemble a NAR representation of a local file system path. ## `pkg/nar/ls` + A parser for .ls files (providing an index for .nar files) ## `pkg/nar/narinfo` + A parser and generator for `.narinfo` files. ## `pkg/nixbase32` + An implementation of the slightly odd "base32" encoding that's used in Nix, providing some of the functions in `encoding/base32.Encoding`. ## `pkg/storepath` + A parser and regexes for Nix Store Paths. ## `pkg/storepath/references` + A Nix Store path reference scanner. +## `pkg/sqlite` + +A collection of interfaces and utilities for writing to and querying various `sqlite` databases that Nix uses. + +[sqlc]: https://github.com/sqlc-dev/sqlc + +## `pkg/sqlite/binary_cache_v6` + +[SQLC] generated code for querying the Nar Info Disk Cache, typically located at `$XDG_CACHE_HOME/nix/binary-cache-v6.sqlite`. + +## `pkg/sqlite/eval_cache_v5` + +[SQLC] generated code for querying an instance of the Eval Cache, typically located at `$XDG_CACHE_HOME/nix/eval-cache-v5/*.sqlite`. + +## `pkg/sqlite/fetcher_cache_v2` + +[SQLC] generated code for querying the fetcher cache, typically located in `$XDG_CACHE_HOME/nix/fetcher-cache-v2.sqlite`. + +## `pkg/sqlite/nix_v10` + +[SQLC] generated code for querying the main Nix database, typically located in `/nix/var/nix/db.sqlite`. + ## `pkg/wire` + Methods to parse and produce fields used in the low-level Nix wire protocol. diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..3c05e8c --- /dev/null +++ b/default.nix @@ -0,0 +1,26 @@ +# This file provides backward compatibility to nix < 2.4 clients +{ + system ? builtins.currentSystem, +}: +let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + + root = lock.nodes.${lock.root}; + inherit (lock.nodes.${root.inputs.flake-compat}.locked) + owner + repo + rev + narHash + ; + + flake-compat = fetchTarball { + url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; + sha256 = narHash; + }; + + flake = import flake-compat { + inherit system; + src = ./.; + }; +in +flake.defaultNix diff --git a/devshell.nix b/devshell.nix new file mode 100644 index 0000000..3f604f7 --- /dev/null +++ b/devshell.nix @@ -0,0 +1,22 @@ +{ + perSystem, + pkgs, + ... +}: +pkgs.mkShell { + env.GOROOT = "${pkgs.go}/share/go"; + + packages = + (with pkgs; [ + delve + pprof + go + gotools + golangci-lint + lazysql + sqlc + ]) + ++ (with perSystem; [ + gomod2nix.default + ]); +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..f8a104a --- /dev/null +++ b/flake.lock @@ -0,0 +1,162 @@ +{ + "nodes": { + "blueprint": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "systems": [ + "systems" + ] + }, + "locked": { + "lastModified": 1727084436, + "narHash": "sha256-H5rbzYDlQD/lmTKvvfyohnhB+zdoZfykghjFHi2rS7o=", + "owner": "numtide", + "repo": "blueprint", + "rev": "77e32417d97959e3d81d22211cba7c8ba44c0079", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "blueprint", + "type": "github" + } + }, + "flake-compat": { + "locked": { + "lastModified": 1717312683, + "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=", + "owner": "nix-community", + "repo": "flake-compat", + "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1727668934, + "narHash": "sha256-nPpQ/4k6Fjaq2CHNSdO6j1ikiuWApuk/S6lU6ISp5SQ=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "066e0dd2afde263f547cb0905b77cea00521d86c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "gomod2nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1727348695, + "narHash": "sha256-J+PeFKSDV+pHL7ukkfpVzCOO7mBSrrpJ3svwBFABbhI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "1925c603f17fc89f4c8f6bf6f631a802ad85d784", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "blueprint": "blueprint", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "gomod2nix": "gomod2nix", + "nixpkgs": "nixpkgs", + "systems": "systems_2", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1727431250, + "narHash": "sha256-uGRlRT47ecicF9iLD1G3g43jn2e+b5KaMptb59LHnvM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "879b29ae9a0378904fbbefe0dadaed43c8905754", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..2df475e --- /dev/null +++ b/flake.nix @@ -0,0 +1,31 @@ +{ + description = "Elements of Nix re-implemented as Go libraries"; + + inputs = { + blueprint = { + url = "github:numtide/blueprint"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.systems.follows = "systems"; + }; + gomod2nix = { + url = "github:nix-community/gomod2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; + systems.url = "github:nix-systems/default"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + treefmt-nix = { + url = "github:numtide/treefmt-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-compat.url = "github:nix-community/flake-compat"; + }; + + # Keep the magic invocations to minimum. + outputs = + inputs: + inputs.blueprint { + inherit inputs; + }; +} diff --git a/formatter.nix b/formatter.nix new file mode 100644 index 0000000..89ecd08 --- /dev/null +++ b/formatter.nix @@ -0,0 +1,40 @@ +{ + pkgs, + inputs, + ... +}: +inputs.treefmt-nix.lib.mkWrapper pkgs { + projectRootFile = ".git/config"; + + programs = { + nixfmt.enable = true; + deadnix.enable = true; + gofumpt.enable = true; + prettier.enable = true; + statix.enable = true; + }; + + settings = { + global.excludes = [ + "LICENSE" + ".gitattributes" + "test/testdata/*" + # unsupported extensions + "*.{gif,png,svg,tape,mts,lock,mod,sum,toml,env,envrc,gitignore,sql}" + ]; + + formatter = { + deadnix.priority = 1; + statix.priority = 2; + nixfmt.priority = 3; + + prettier = { + options = [ + "--tab-width" + "4" + ]; + includes = [ "*.{css,html,js,json,jsx,md,mdx,scss,ts,yaml}" ]; + }; + }; + }; +} diff --git a/go.mod b/go.mod index 0e6fe60..98dbe83 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module github.com/nix-community/go-nix go 1.20 require ( + github.com/adrg/xdg v0.5.0 github.com/alecthomas/kong v0.5.0 github.com/dgraph-io/badger/v3 v3.2103.2 + github.com/mattn/go-sqlite3 v1.14.23 github.com/multiformats/go-multihash v0.2.1 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.9.0 ) require ( @@ -33,7 +35,7 @@ require ( go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect + golang.org/x/sys v0.22.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.6 // indirect ) diff --git a/go.sum b/go.sum index 2be3d82..7724314 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= +github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE= github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0= github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= @@ -59,6 +61,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -89,8 +93,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -133,8 +138,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -160,8 +165,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/pkg/nar/header_mode_windows.go b/pkg/nar/header_mode_windows.go index 669777c..5f33411 100644 --- a/pkg/nar/header_mode_windows.go +++ b/pkg/nar/header_mode_windows.go @@ -21,5 +21,5 @@ func (fi headerFileInfo) Mode() fs.FileMode { mode = fs.ModeSymlink } - return mode & ^fs.FileMode(0200) + return mode & ^fs.FileMode(0o200) } diff --git a/pkg/sqlite/README.md b/pkg/sqlite/README.md new file mode 100644 index 0000000..f8a5fff --- /dev/null +++ b/pkg/sqlite/README.md @@ -0,0 +1,11 @@ +# Sqlite Packages + +Each subpackage targets a specific version of an internal Nix [sqlite] database. + +The user should only edit `schema.sql` and `query.sql`. All other files are generated by [sqlc] via `sqlc generate` from +the root of the repository. + +> Note: when adding a new package, you must edit the `sqlc.yml` file at the root of the repository. + +[sqlite]: https://www.sqlite.org/ +[sqlc]: https://github.com/sqlc-dev/sqlc diff --git a/pkg/sqlite/binary_cache_v6/db.go b/pkg/sqlite/binary_cache_v6/db.go new file mode 100644 index 0000000..85452a6 --- /dev/null +++ b/pkg/sqlite/binary_cache_v6/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package binary_cache_v6 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/pkg/sqlite/binary_cache_v6/models.go b/pkg/sqlite/binary_cache_v6/models.go new file mode 100644 index 0000000..4356a1f --- /dev/null +++ b/pkg/sqlite/binary_cache_v6/models.go @@ -0,0 +1,53 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package binary_cache_v6 + +import ( + "database/sql" +) + +type BinaryCach struct { + ID int64 + Url string + Timestamp int64 + Storedir string + Wantmassquery int64 + Priority int64 +} + +type LastPurge struct { + Dummy string + Value sql.NullInt64 +} + +type NAR struct { + Cache int64 + Hashpart string + Namepart sql.NullString + Url sql.NullString + Compression sql.NullString + Filehash sql.NullString + Filesize sql.NullInt64 + Narhash sql.NullString + Narsize sql.NullInt64 + Refs sql.NullString + Deriver sql.NullString + Sigs sql.NullString + Ca sql.NullString + Timestamp int64 + Present int64 +} + +type Realisation struct { + Cache int64 + Outputid string + Content []byte + Timestamp int64 +} + +type SqliteSequence struct { + Name interface{} + Seq interface{} +} diff --git a/pkg/sqlite/binary_cache_v6/query.sql b/pkg/sqlite/binary_cache_v6/query.sql new file mode 100644 index 0000000..c5f83a4 --- /dev/null +++ b/pkg/sqlite/binary_cache_v6/query.sql @@ -0,0 +1,48 @@ +-- name: InsertCache :one +insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) +values (?1, ?2, ?3, ?4, ?5) +on conflict (url) +do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 +returning id; + +-- name: QueryCache :many +select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?; + +-- name: InsertNar :exec +insert or replace into NARs( + cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca, + timestamp, present +) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1); + +-- name: InsertMissingNAR :exec +insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0); + +-- name: QueryNar :many +select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs +where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?)); + +-- name: InsertRealisation :exec +insert or replace into Realisations(cache, outputId, content, timestamp) +values (?, ?, ?, ?); + +-- name: InsertMissingRealisation :exec +insert or replace into Realisations(cache, outputId, timestamp) +values (?, ?, ?); + +-- name: QueryRealisation :many +select content from Realisations +where cache = ? and outputId = ? and +( + (content is null and timestamp > ?) or + (content is not null and timestamp > ?) +); + +-- name: QueryLastPurge :one +select value from LastPurge; + +-- name: UpdateLastPurge :exec +insert or replace into LastPurge(dummy, value) values ('', ?); + +-- name: PurgeNars :exec +delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?)); + diff --git a/pkg/sqlite/binary_cache_v6/query.sql.go b/pkg/sqlite/binary_cache_v6/query.sql.go new file mode 100644 index 0000000..2d88084 --- /dev/null +++ b/pkg/sqlite/binary_cache_v6/query.sql.go @@ -0,0 +1,327 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: query.sql + +package binary_cache_v6 + +import ( + "context" + "database/sql" +) + +const insertCache = `-- name: InsertCache :one +insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) +values (?1, ?2, ?3, ?4, ?5) +on conflict (url) +do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 +returning id +` + +type InsertCacheParams struct { + Url string + Timestamp int64 + Storedir string + Wantmassquery int64 + Priority int64 +} + +func (q *Queries) InsertCache(ctx context.Context, arg InsertCacheParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertCache, + arg.Url, + arg.Timestamp, + arg.Storedir, + arg.Wantmassquery, + arg.Priority, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertMissingNAR = `-- name: InsertMissingNAR :exec +insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0) +` + +type InsertMissingNARParams struct { + Cache int64 + Hashpart string + Timestamp int64 +} + +func (q *Queries) InsertMissingNAR(ctx context.Context, arg InsertMissingNARParams) error { + _, err := q.db.ExecContext(ctx, insertMissingNAR, arg.Cache, arg.Hashpart, arg.Timestamp) + return err +} + +const insertMissingRealisation = `-- name: InsertMissingRealisation :exec +insert or replace into Realisations(cache, outputId, timestamp) +values (?, ?, ?) +` + +type InsertMissingRealisationParams struct { + Cache int64 + Outputid string + Timestamp int64 +} + +func (q *Queries) InsertMissingRealisation(ctx context.Context, arg InsertMissingRealisationParams) error { + _, err := q.db.ExecContext(ctx, insertMissingRealisation, arg.Cache, arg.Outputid, arg.Timestamp) + return err +} + +const insertNar = `-- name: InsertNar :exec +insert or replace into NARs( + cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca, + timestamp, present +) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1) +` + +type InsertNarParams struct { + Cache int64 + Hashpart string + Namepart sql.NullString + Url sql.NullString + Compression sql.NullString + Filehash sql.NullString + Filesize sql.NullInt64 + Narhash sql.NullString + Narsize sql.NullInt64 + Refs sql.NullString + Deriver sql.NullString + Sigs sql.NullString + Ca sql.NullString + Timestamp int64 +} + +func (q *Queries) InsertNar(ctx context.Context, arg InsertNarParams) error { + _, err := q.db.ExecContext(ctx, insertNar, + arg.Cache, + arg.Hashpart, + arg.Namepart, + arg.Url, + arg.Compression, + arg.Filehash, + arg.Filesize, + arg.Narhash, + arg.Narsize, + arg.Refs, + arg.Deriver, + arg.Sigs, + arg.Ca, + arg.Timestamp, + ) + return err +} + +const insertRealisation = `-- name: InsertRealisation :exec +insert or replace into Realisations(cache, outputId, content, timestamp) +values (?, ?, ?, ?) +` + +type InsertRealisationParams struct { + Cache int64 + Outputid string + Content []byte + Timestamp int64 +} + +func (q *Queries) InsertRealisation(ctx context.Context, arg InsertRealisationParams) error { + _, err := q.db.ExecContext(ctx, insertRealisation, + arg.Cache, + arg.Outputid, + arg.Content, + arg.Timestamp, + ) + return err +} + +const purgeNars = `-- name: PurgeNars :exec +delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?)) +` + +type PurgeNarsParams struct { + Timestamp int64 + Timestamp_2 int64 +} + +func (q *Queries) PurgeNars(ctx context.Context, arg PurgeNarsParams) error { + _, err := q.db.ExecContext(ctx, purgeNars, arg.Timestamp, arg.Timestamp_2) + return err +} + +const queryCache = `-- name: QueryCache :many +select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ? +` + +type QueryCacheParams struct { + Url string + Timestamp int64 +} + +type QueryCacheRow struct { + ID int64 + Storedir string + Wantmassquery int64 + Priority int64 +} + +func (q *Queries) QueryCache(ctx context.Context, arg QueryCacheParams) ([]QueryCacheRow, error) { + rows, err := q.db.QueryContext(ctx, queryCache, arg.Url, arg.Timestamp) + if err != nil { + return nil, err + } + defer rows.Close() + var items []QueryCacheRow + for rows.Next() { + var i QueryCacheRow + if err := rows.Scan( + &i.ID, + &i.Storedir, + &i.Wantmassquery, + &i.Priority, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queryLastPurge = `-- name: QueryLastPurge :one +select value from LastPurge +` + +func (q *Queries) QueryLastPurge(ctx context.Context) (sql.NullInt64, error) { + row := q.db.QueryRowContext(ctx, queryLastPurge) + var value sql.NullInt64 + err := row.Scan(&value) + return value, err +} + +const queryNar = `-- name: QueryNar :many +select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs +where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?)) +` + +type QueryNarParams struct { + Cache int64 + Hashpart string + Timestamp int64 + Timestamp_2 int64 +} + +type QueryNarRow struct { + Present int64 + Namepart sql.NullString + Url sql.NullString + Compression sql.NullString + Filehash sql.NullString + Filesize sql.NullInt64 + Narhash sql.NullString + Narsize sql.NullInt64 + Refs sql.NullString + Deriver sql.NullString + Sigs sql.NullString + Ca sql.NullString +} + +func (q *Queries) QueryNar(ctx context.Context, arg QueryNarParams) ([]QueryNarRow, error) { + rows, err := q.db.QueryContext(ctx, queryNar, + arg.Cache, + arg.Hashpart, + arg.Timestamp, + arg.Timestamp_2, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []QueryNarRow + for rows.Next() { + var i QueryNarRow + if err := rows.Scan( + &i.Present, + &i.Namepart, + &i.Url, + &i.Compression, + &i.Filehash, + &i.Filesize, + &i.Narhash, + &i.Narsize, + &i.Refs, + &i.Deriver, + &i.Sigs, + &i.Ca, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queryRealisation = `-- name: QueryRealisation :many +select content from Realisations +where cache = ? and outputId = ? and +( + (content is null and timestamp > ?) or + (content is not null and timestamp > ?) +) +` + +type QueryRealisationParams struct { + Cache int64 + Outputid string + Timestamp int64 + Timestamp_2 int64 +} + +func (q *Queries) QueryRealisation(ctx context.Context, arg QueryRealisationParams) ([][]byte, error) { + rows, err := q.db.QueryContext(ctx, queryRealisation, + arg.Cache, + arg.Outputid, + arg.Timestamp, + arg.Timestamp_2, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items [][]byte + for rows.Next() { + var content []byte + if err := rows.Scan(&content); err != nil { + return nil, err + } + items = append(items, content) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateLastPurge = `-- name: UpdateLastPurge :exec +insert or replace into LastPurge(dummy, value) values ('', ?) +` + +func (q *Queries) UpdateLastPurge(ctx context.Context, value sql.NullInt64) error { + _, err := q.db.ExecContext(ctx, updateLastPurge, value) + return err +} diff --git a/pkg/sqlite/binary_cache_v6/schema.sql b/pkg/sqlite/binary_cache_v6/schema.sql new file mode 100644 index 0000000..a242a20 --- /dev/null +++ b/pkg/sqlite/binary_cache_v6/schema.sql @@ -0,0 +1,40 @@ +CREATE TABLE BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); +CREATE TABLE sqlite_sequence(name,seq); +CREATE TABLE NARs ( + cache integer not null, + hashPart text not null, + namePart text, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + ca text, + timestamp integer not null, + present integer not null, + primary key (cache, hashPart), + foreign key (cache) references BinaryCaches(id) on delete cascade +); +CREATE TABLE Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or null if the realisation is absent + timestamp integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); +CREATE TABLE LastPurge ( + dummy text primary key, + value integer +); diff --git a/pkg/sqlite/eval_cache_v5/db.go b/pkg/sqlite/eval_cache_v5/db.go new file mode 100644 index 0000000..00014e6 --- /dev/null +++ b/pkg/sqlite/eval_cache_v5/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package eval_cache_v5 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/pkg/sqlite/eval_cache_v5/models.go b/pkg/sqlite/eval_cache_v5/models.go new file mode 100644 index 0000000..056e4df --- /dev/null +++ b/pkg/sqlite/eval_cache_v5/models.go @@ -0,0 +1,17 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package eval_cache_v5 + +import ( + "database/sql" +) + +type Attribute struct { + Parent int64 + Name sql.NullString + Type int64 + Value sql.NullString + Context sql.NullString +} diff --git a/pkg/sqlite/eval_cache_v5/query.sql b/pkg/sqlite/eval_cache_v5/query.sql new file mode 100644 index 0000000..6c26d9e --- /dev/null +++ b/pkg/sqlite/eval_cache_v5/query.sql @@ -0,0 +1,12 @@ +-- name: InsertAttribute :exec +insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?); + +-- name: InsertAttributeWithContext :exec +insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?); + +-- todo sqlc doesn't like the rowid column being included below +-- name: QueryAttribute :one +select type, value, context from Attributes where parent = ? and name = ?; + +-- name: QueryAttributes :many +select name from Attributes where parent = ?; \ No newline at end of file diff --git a/pkg/sqlite/eval_cache_v5/query.sql.go b/pkg/sqlite/eval_cache_v5/query.sql.go new file mode 100644 index 0000000..d28961a --- /dev/null +++ b/pkg/sqlite/eval_cache_v5/query.sql.go @@ -0,0 +1,105 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: query.sql + +package eval_cache_v5 + +import ( + "context" + "database/sql" +) + +const insertAttribute = `-- name: InsertAttribute :exec +insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?) +` + +type InsertAttributeParams struct { + Parent int64 + Name sql.NullString + Type int64 + Value sql.NullString +} + +func (q *Queries) InsertAttribute(ctx context.Context, arg InsertAttributeParams) error { + _, err := q.db.ExecContext(ctx, insertAttribute, + arg.Parent, + arg.Name, + arg.Type, + arg.Value, + ) + return err +} + +const insertAttributeWithContext = `-- name: InsertAttributeWithContext :exec +insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?) +` + +type InsertAttributeWithContextParams struct { + Parent int64 + Name sql.NullString + Type int64 + Value sql.NullString + Context sql.NullString +} + +func (q *Queries) InsertAttributeWithContext(ctx context.Context, arg InsertAttributeWithContextParams) error { + _, err := q.db.ExecContext(ctx, insertAttributeWithContext, + arg.Parent, + arg.Name, + arg.Type, + arg.Value, + arg.Context, + ) + return err +} + +const queryAttribute = `-- name: QueryAttribute :one +select type, value, context from Attributes where parent = ? and name = ? +` + +type QueryAttributeParams struct { + Parent int64 + Name sql.NullString +} + +type QueryAttributeRow struct { + Type int64 + Value sql.NullString + Context sql.NullString +} + +// todo sqlc doesn't like the rowid column being included below +func (q *Queries) QueryAttribute(ctx context.Context, arg QueryAttributeParams) (QueryAttributeRow, error) { + row := q.db.QueryRowContext(ctx, queryAttribute, arg.Parent, arg.Name) + var i QueryAttributeRow + err := row.Scan(&i.Type, &i.Value, &i.Context) + return i, err +} + +const queryAttributes = `-- name: QueryAttributes :many +select name from Attributes where parent = ? +` + +func (q *Queries) QueryAttributes(ctx context.Context, parent int64) ([]sql.NullString, error) { + rows, err := q.db.QueryContext(ctx, queryAttributes, parent) + if err != nil { + return nil, err + } + defer rows.Close() + var items []sql.NullString + for rows.Next() { + var name sql.NullString + if err := rows.Scan(&name); err != nil { + return nil, err + } + items = append(items, name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/sqlite/eval_cache_v5/schema.sql b/pkg/sqlite/eval_cache_v5/schema.sql new file mode 100644 index 0000000..b712121 --- /dev/null +++ b/pkg/sqlite/eval_cache_v5/schema.sql @@ -0,0 +1,8 @@ +create table if not exists Attributes ( + parent integer not null, + name text, + type integer not null, + value text, + context text, + primary key (parent, name) +); \ No newline at end of file diff --git a/pkg/sqlite/fetcher_cache_v2/db.go b/pkg/sqlite/fetcher_cache_v2/db.go new file mode 100644 index 0000000..7a14671 --- /dev/null +++ b/pkg/sqlite/fetcher_cache_v2/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package fetcher_cache_v2 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/pkg/sqlite/fetcher_cache_v2/models.go b/pkg/sqlite/fetcher_cache_v2/models.go new file mode 100644 index 0000000..0e61695 --- /dev/null +++ b/pkg/sqlite/fetcher_cache_v2/models.go @@ -0,0 +1,12 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package fetcher_cache_v2 + +type Cache struct { + Domain string + Key string + Value string + Timestamp int64 +} diff --git a/pkg/sqlite/fetcher_cache_v2/query.sql b/pkg/sqlite/fetcher_cache_v2/query.sql new file mode 100644 index 0000000..2c6f237 --- /dev/null +++ b/pkg/sqlite/fetcher_cache_v2/query.sql @@ -0,0 +1,5 @@ +-- name: UpsertCache :exec +insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?); + +-- name: QueryCache :many +select value, timestamp from Cache where domain = ? and key = ?; \ No newline at end of file diff --git a/pkg/sqlite/fetcher_cache_v2/query.sql.go b/pkg/sqlite/fetcher_cache_v2/query.sql.go new file mode 100644 index 0000000..2826adb --- /dev/null +++ b/pkg/sqlite/fetcher_cache_v2/query.sql.go @@ -0,0 +1,68 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: query.sql + +package fetcher_cache_v2 + +import ( + "context" +) + +const queryCache = `-- name: QueryCache :many +select value, timestamp from Cache where domain = ? and key = ? +` + +type QueryCacheParams struct { + Domain string + Key string +} + +type QueryCacheRow struct { + Value string + Timestamp int64 +} + +func (q *Queries) QueryCache(ctx context.Context, arg QueryCacheParams) ([]QueryCacheRow, error) { + rows, err := q.db.QueryContext(ctx, queryCache, arg.Domain, arg.Key) + if err != nil { + return nil, err + } + defer rows.Close() + var items []QueryCacheRow + for rows.Next() { + var i QueryCacheRow + if err := rows.Scan(&i.Value, &i.Timestamp); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const upsertCache = `-- name: UpsertCache :exec +insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?) +` + +type UpsertCacheParams struct { + Domain string + Key string + Value string + Timestamp int64 +} + +func (q *Queries) UpsertCache(ctx context.Context, arg UpsertCacheParams) error { + _, err := q.db.ExecContext(ctx, upsertCache, + arg.Domain, + arg.Key, + arg.Value, + arg.Timestamp, + ) + return err +} diff --git a/pkg/sqlite/fetcher_cache_v2/schema.sql b/pkg/sqlite/fetcher_cache_v2/schema.sql new file mode 100644 index 0000000..063ef6b --- /dev/null +++ b/pkg/sqlite/fetcher_cache_v2/schema.sql @@ -0,0 +1,7 @@ +create table if not exists Cache ( + domain text not null, + key text not null, + value text not null, + timestamp integer not null, + primary key (domain, key) +); \ No newline at end of file diff --git a/pkg/sqlite/nix_v10/db.go b/pkg/sqlite/nix_v10/db.go new file mode 100644 index 0000000..abf0bf6 --- /dev/null +++ b/pkg/sqlite/nix_v10/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package nix_v10 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/pkg/sqlite/nix_v10/models.go b/pkg/sqlite/nix_v10/models.go new file mode 100644 index 0000000..4211fb2 --- /dev/null +++ b/pkg/sqlite/nix_v10/models.go @@ -0,0 +1,37 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 + +package nix_v10 + +import ( + "database/sql" +) + +type DerivationOutput struct { + Drv int64 + ID string + Path string +} + +type Ref struct { + Referrer int64 + Reference int64 +} + +type SqliteSequence struct { + Name interface{} + Seq interface{} +} + +type ValidPath struct { + ID int64 + Path string + Hash string + Registrationtime int64 + Deriver sql.NullString + Narsize sql.NullInt64 + Ultimate sql.NullInt64 + Sigs sql.NullString + Ca sql.NullString +} diff --git a/pkg/sqlite/nix_v10/query.sql b/pkg/sqlite/nix_v10/query.sql new file mode 100644 index 0000000..0969e17 --- /dev/null +++ b/pkg/sqlite/nix_v10/query.sql @@ -0,0 +1,33 @@ +-- name: RegisterValidPath :exec +insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) +values (?, ?, ?, ?, ?, ?, ?, ?); + +-- name: UpdatePathInfo :exec +update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?; + +-- name: AddReference :exec +insert or replace into Refs (referrer, reference) values (?, ?); + +-- name: QueryPathInfo :one +select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?; + +-- name: QueryReferences :many +select path from Refs join ValidPaths on reference = id where referrer = ?; + +-- name: QueryReferrers :many +select path from Refs join ValidPaths on referrer = id where reference = (select vp.id from ValidPaths as vp where vp.path = ?); + +-- name: InvalidatePath :exec +delete from ValidPaths where path = ?; + +-- name: AddDerivationOutput :exec +insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?); + +-- name: QueryValidDerivers :many +select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?; + +-- name: QueryDerivationOutputs :many +select id, path from DerivationOutputs where drv = ?; + +-- name: QueryPathFromHashPart :one +select path from ValidPaths where path >= ? limit 1; \ No newline at end of file diff --git a/pkg/sqlite/nix_v10/query.sql.go b/pkg/sqlite/nix_v10/query.sql.go new file mode 100644 index 0000000..d273168 --- /dev/null +++ b/pkg/sqlite/nix_v10/query.sql.go @@ -0,0 +1,263 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: query.sql + +package nix_v10 + +import ( + "context" + "database/sql" +) + +const addDerivationOutput = `-- name: AddDerivationOutput :exec +insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?) +` + +type AddDerivationOutputParams struct { + Drv int64 + ID string + Path string +} + +func (q *Queries) AddDerivationOutput(ctx context.Context, arg AddDerivationOutputParams) error { + _, err := q.db.ExecContext(ctx, addDerivationOutput, arg.Drv, arg.ID, arg.Path) + return err +} + +const addReference = `-- name: AddReference :exec +insert or replace into Refs (referrer, reference) values (?, ?) +` + +type AddReferenceParams struct { + Referrer int64 + Reference int64 +} + +func (q *Queries) AddReference(ctx context.Context, arg AddReferenceParams) error { + _, err := q.db.ExecContext(ctx, addReference, arg.Referrer, arg.Reference) + return err +} + +const invalidatePath = `-- name: InvalidatePath :exec +delete from ValidPaths where path = ? +` + +func (q *Queries) InvalidatePath(ctx context.Context, path string) error { + _, err := q.db.ExecContext(ctx, invalidatePath, path) + return err +} + +const queryDerivationOutputs = `-- name: QueryDerivationOutputs :many +select id, path from DerivationOutputs where drv = ? +` + +type QueryDerivationOutputsRow struct { + ID string + Path string +} + +func (q *Queries) QueryDerivationOutputs(ctx context.Context, drv int64) ([]QueryDerivationOutputsRow, error) { + rows, err := q.db.QueryContext(ctx, queryDerivationOutputs, drv) + if err != nil { + return nil, err + } + defer rows.Close() + var items []QueryDerivationOutputsRow + for rows.Next() { + var i QueryDerivationOutputsRow + if err := rows.Scan(&i.ID, &i.Path); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queryPathFromHashPart = `-- name: QueryPathFromHashPart :one +select path from ValidPaths where path >= ? limit 1 +` + +func (q *Queries) QueryPathFromHashPart(ctx context.Context, path string) (string, error) { + row := q.db.QueryRowContext(ctx, queryPathFromHashPart, path) + err := row.Scan(&path) + return path, err +} + +const queryPathInfo = `-- name: QueryPathInfo :one +select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ? +` + +type QueryPathInfoRow struct { + ID int64 + Hash string + Registrationtime int64 + Deriver sql.NullString + Narsize sql.NullInt64 + Ultimate sql.NullInt64 + Sigs sql.NullString + Ca sql.NullString +} + +func (q *Queries) QueryPathInfo(ctx context.Context, path string) (QueryPathInfoRow, error) { + row := q.db.QueryRowContext(ctx, queryPathInfo, path) + var i QueryPathInfoRow + err := row.Scan( + &i.ID, + &i.Hash, + &i.Registrationtime, + &i.Deriver, + &i.Narsize, + &i.Ultimate, + &i.Sigs, + &i.Ca, + ) + return i, err +} + +const queryReferences = `-- name: QueryReferences :many +select path from Refs join ValidPaths on reference = id where referrer = ? +` + +func (q *Queries) QueryReferences(ctx context.Context, referrer int64) ([]string, error) { + rows, err := q.db.QueryContext(ctx, queryReferences, referrer) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + items = append(items, path) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queryReferrers = `-- name: QueryReferrers :many +select path from Refs join ValidPaths on referrer = id where reference = (select vp.id from ValidPaths as vp where vp.path = ?) +` + +func (q *Queries) QueryReferrers(ctx context.Context, path string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, queryReferrers, path) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + items = append(items, path) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queryValidDerivers = `-- name: QueryValidDerivers :many +select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ? +` + +type QueryValidDeriversRow struct { + ID int64 + Path string +} + +func (q *Queries) QueryValidDerivers(ctx context.Context, path string) ([]QueryValidDeriversRow, error) { + rows, err := q.db.QueryContext(ctx, queryValidDerivers, path) + if err != nil { + return nil, err + } + defer rows.Close() + var items []QueryValidDeriversRow + for rows.Next() { + var i QueryValidDeriversRow + if err := rows.Scan(&i.ID, &i.Path); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const registerValidPath = `-- name: RegisterValidPath :exec +insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) +values (?, ?, ?, ?, ?, ?, ?, ?) +` + +type RegisterValidPathParams struct { + Path string + Hash string + Registrationtime int64 + Deriver sql.NullString + Narsize sql.NullInt64 + Ultimate sql.NullInt64 + Sigs sql.NullString + Ca sql.NullString +} + +func (q *Queries) RegisterValidPath(ctx context.Context, arg RegisterValidPathParams) error { + _, err := q.db.ExecContext(ctx, registerValidPath, + arg.Path, + arg.Hash, + arg.Registrationtime, + arg.Deriver, + arg.Narsize, + arg.Ultimate, + arg.Sigs, + arg.Ca, + ) + return err +} + +const updatePathInfo = `-- name: UpdatePathInfo :exec +update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ? +` + +type UpdatePathInfoParams struct { + Narsize sql.NullInt64 + Hash string + Ultimate sql.NullInt64 + Sigs sql.NullString + Ca sql.NullString + Path string +} + +func (q *Queries) UpdatePathInfo(ctx context.Context, arg UpdatePathInfoParams) error { + _, err := q.db.ExecContext(ctx, updatePathInfo, + arg.Narsize, + arg.Hash, + arg.Ultimate, + arg.Sigs, + arg.Ca, + arg.Path, + ) + return err +} diff --git a/pkg/sqlite/nix_v10/schema.sql b/pkg/sqlite/nix_v10/schema.sql new file mode 100644 index 0000000..36bb962 --- /dev/null +++ b/pkg/sqlite/nix_v10/schema.sql @@ -0,0 +1,33 @@ +CREATE TABLE ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, -- base16 representation + registrationTime integer not null, + deriver text, + narSize integer, + ultimate integer, -- null implies "false" + sigs text, -- space-separated + ca text -- if not null, an assertion that the path is content-addressed; see ValidPathInfo +); +CREATE TABLE sqlite_sequence(name,seq); +CREATE TABLE Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); +CREATE INDEX IndexReferrer on Refs(referrer); +CREATE INDEX IndexReference on Refs(reference); +CREATE TRIGGER DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; +CREATE TABLE DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); +CREATE INDEX IndexDerivationOutputs on DerivationOutputs(path); diff --git a/pkg/sqlite/sqlite.go b/pkg/sqlite/sqlite.go new file mode 100644 index 0000000..819a333 --- /dev/null +++ b/pkg/sqlite/sqlite.go @@ -0,0 +1,49 @@ +package sqlite + +import ( + "database/sql" + "fmt" + + // enable the sqlite3 driver. + _ "github.com/mattn/go-sqlite3" + "github.com/nix-community/go-nix/pkg/sqlite/binary_cache_v6" + "github.com/nix-community/go-nix/pkg/sqlite/eval_cache_v5" + "github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2" + "github.com/nix-community/go-nix/pkg/sqlite/nix_v10" +) + +func BinaryCacheV6(dsn string) (*sql.DB, *binary_cache_v6.Queries, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, fmt.Errorf("failed to open database: %w", err) + } + + return db, binary_cache_v6.New(db), nil +} + +func EvalCacheV5(dsn string) (*sql.DB, *eval_cache_v5.Queries, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, fmt.Errorf("failed to open database: %w", err) + } + + return db, eval_cache_v5.New(db), nil +} + +func FetcherCacheV2(dsn string) (*sql.DB, *fetcher_cache_v2.Queries, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, fmt.Errorf("failed to open database: %w", err) + } + + return db, fetcher_cache_v2.New(db), nil +} + +func NixV10(dsn string) (*sql.DB, *nix_v10.Queries, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, fmt.Errorf("failed to open database: %w", err) + } + + return db, nix_v10.New(db), nil +} diff --git a/pkg/sqlite/sqlite_test.go b/pkg/sqlite/sqlite_test.go new file mode 100644 index 0000000..f0d954e --- /dev/null +++ b/pkg/sqlite/sqlite_test.go @@ -0,0 +1,71 @@ +//go:build integration + +package sqlite + +import ( + "context" + "fmt" + "os/exec" + "testing" + + "github.com/adrg/xdg" + "github.com/nix-community/go-nix/pkg/sqlite/fetcher_cache_v2" + + "github.com/stretchr/testify/require" +) + +func TestBinaryCacheV6(t *testing.T) { + as := require.New(t) + + // open our user-specific binary cache db + path, err := xdg.CacheFile("nix/binary-cache-v6.sqlite") + as.NoError(err, "failed to resolve binary cache file") + as.FileExists(path) + + // open the sqlite db + db, queries, err := BinaryCacheV6(fmt.Sprintf("file:%s?mode=ro", path)) + as.NoError(err) + defer db.Close() + + // perform a basic query, we aren't interested in the result + _, err = queries.QueryLastPurge(context.Background()) + as.NoError(err) +} + +func TestFetcherCacheV2(t *testing.T) { + as := require.New(t) + + // open our user-specific binary cache db + path, err := xdg.CacheFile("nix/fetcher-cache-v2.sqlite") + as.NoError(err, "failed to resolve fetcher cache file") + as.FileExists(path) + + // open the sqlite db + db, queries, err := FetcherCacheV2(fmt.Sprintf("file:%s?mode=ro", path)) + as.NoError(err) + defer db.Close() + + // perform a basic query, we aren't interested in the result + _, err = queries.QueryCache(context.Background(), fetcher_cache_v2.QueryCacheParams{}) + as.NoError(err) +} + +func TestNixV10(t *testing.T) { + as := require.New(t) + + // pull down a known path + path := "/nix/store/kz5clxh7s1n0fnx6d37c1wc2cs9qm53q-hello-2.12.1" + as.NoError(exec.Command("nix", "build", "--no-link", "--refresh", path).Run(), "failed to pull hello path") + + // open the sqlite db + db, queries, err := NixV10("file:/nix/var/nix/db/db.sqlite?mode=ro") + as.NoError(err) + defer db.Close() + + // query the path we just pulled down + info, err := queries.QueryPathInfo(context.Background(), path) + as.NoError(err) + as.Equal("sha256:f8340af15f7996faded748bea9e2d0b82a6f7c96417b03f7fa8e1a6a873748e8", info.Hash) + as.Equal("/nix/store/qnavcbp5ydyd12asgz7rpr7is7hlswaz-hello-2.12.1.drv", info.Deriver.String) + as.Equal(int64(226560), info.Narsize.Int64) +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..1e80405 --- /dev/null +++ b/shell.nix @@ -0,0 +1,26 @@ +# This file provides backward compatibility to nix < 2.4 clients +{ + system ? builtins.currentSystem, +}: +let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + + root = lock.nodes.${lock.root}; + inherit (lock.nodes.${root.inputs.flake-compat}.locked) + owner + repo + rev + narHash + ; + + flake-compat = fetchTarball { + url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; + sha256 = narHash; + }; + + flake = import flake-compat { + inherit system; + src = ./.; + }; +in +flake.shellNix diff --git a/sqlc.yml b/sqlc.yml new file mode 100644 index 0000000..9b03a12 --- /dev/null +++ b/sqlc.yml @@ -0,0 +1,30 @@ +version: "2" +sql: + - engine: "sqlite" + queries: "pkg/sqlite/binary_cache_v6/query.sql" + schema: "pkg/sqlite/binary_cache_v6/schema.sql" + gen: + go: + package: "binary_cache_v6" + out: "pkg/sqlite/binary_cache_v6" + - engine: "sqlite" + queries: "pkg/sqlite/eval_cache_v5/query.sql" + schema: "pkg/sqlite/eval_cache_v5/schema.sql" + gen: + go: + package: "eval_cache_v5" + out: "pkg/sqlite/eval_cache_v5" + - engine: "sqlite" + queries: "pkg/sqlite/fetcher_cache_v2/query.sql" + schema: "pkg/sqlite/fetcher_cache_v2/schema.sql" + gen: + go: + package: "fetcher_cache_v2" + out: "pkg/sqlite/fetcher_cache_v2" + - engine: "sqlite" + queries: "pkg/sqlite/nix_v10/query.sql" + schema: "pkg/sqlite/nix_v10/schema.sql" + gen: + go: + package: "nix_v10" + out: "pkg/sqlite/nix_v10"