From bcc9af53a469b173bfeac58059e244fc8d86906e Mon Sep 17 00:00:00 2001 From: cam Date: Sun, 29 Sep 2024 12:47:27 +1300 Subject: [PATCH 1/4] Update salt grains in background --- api_test.go | 6 +++--- config.go | 37 ++++++++++++++++++------------------- go.mod | 29 +++++++++++++++++++++++++++-- go.sum | 13 ++++++------- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/api_test.go b/api_test.go index dadfdb4..34351ba 100644 --- a/api_test.go +++ b/api_test.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "math/rand" "mime/multipart" "net/http" @@ -35,6 +34,7 @@ import ( goconfig "github.com/TheCacophonyProject/go-config" "github.com/TheCacophonyProject/go-config/configtest" + "github.com/TheCacophonyProject/go-utils/saltutil" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -78,8 +78,8 @@ func TestMain(m *testing.M) { os.Exit(1) } // Mock the setSaltGrains function to do nothing - setSaltGrains = func(_ string) ([]byte, error) { - return nil, nil + setSaltGrains = func(_ saltutil.Grains) error { + return nil } os.Exit(m.Run()) } diff --git a/config.go b/config.go index cb8dcc4..d09e997 100644 --- a/config.go +++ b/config.go @@ -16,17 +16,18 @@ package api import ( - "encoding/json" "fmt" - "log" - "os/exec" "regexp" "strings" goconfig "github.com/TheCacophonyProject/go-config" + "github.com/TheCacophonyProject/go-utils/logging" + "github.com/TheCacophonyProject/go-utils/saltutil" "github.com/spf13/afero" ) +var log = logging.NewLogger("info") + const ( hostnameFile = "/etc/hostname" hostsFile = "/etc/hosts" @@ -162,28 +163,26 @@ func updateHostnameAndSaltGrains(device *CacophonyDevice) error { } // Write the new salt grains. - newGrains := map[string]string{ - "device_name": device.name, - "group": device.group, + grains := saltutil.Grains{ + DeviceName: device.name, + Group: device.group, } - // Convert the map to JSON - grainsJSON, err := json.Marshal(newGrains) - if err != nil { - return err - } + // Run the salt-call grains.setvals command in the + // background as it can take about 20 seconds to run. + go func() { + err := setSaltGrains(grains) + if err != nil { + log.Errorf("Failed to set grains: %v", err) + } + }() - out, err := setSaltGrains(string(grainsJSON)) - if err != nil { - log.Println(string(out)) - return err - } return nil } -// setSaltGrains is a wrapper around the salt-call grains.setvals command. This is done for testing purposes -var setSaltGrains = func(grains string) ([]byte, error) { - return exec.Command("salt-call", "grains.setvals", grains).CombinedOutput() +// setSaltGrains is a wrapper around the saltutil.SetGrains command. This is done for testing purposes. +var setSaltGrains = func(grains saltutil.Grains) error { + return saltutil.SetGrains(grains, log) } var Fs = afero.NewOsFs() diff --git a/go.mod b/go.mod index 042f82d..d36ab5f 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,36 @@ module github.com/TheCacophonyProject/go-api -go 1.13 +go 1.22.3 require ( github.com/TheCacophonyProject/go-config v1.9.0 + github.com/TheCacophonyProject/go-utils v0.1.3 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.7.0 - golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect +) + +require ( + github.com/TheCacophonyProject/event-reporter/v3 v3.3.0 // indirect + github.com/boltdb/bolt v1.3.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.9.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.64.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index f607097..a53788d 100644 --- a/go.sum +++ b/go.sum @@ -54,14 +54,14 @@ github.com/TheCacophonyProject/go-api v0.0.0-20190923033957-174cea2ac81c/go.mod github.com/TheCacophonyProject/go-config v0.0.0-20190922224052-7c2a21bc6b88/go.mod h1:gPUJLVu408NRz9/P3BrsxzOzLc+KJLrv+jVdDw3RI0Y= github.com/TheCacophonyProject/go-config v1.9.0 h1:uangUL8e/76ApGzF2tYGKDm+DoDjezVhuJquTX41rHQ= github.com/TheCacophonyProject/go-config v1.9.0/go.mod h1:+y80PSRZudMYuVrYTGOvzc66NxVJWKS4TuU442vmvhY= +github.com/TheCacophonyProject/go-utils v0.1.3 h1:DSuDeJz7ZM00yQRLsoukWH0fnC+8X8+ziYxOl6l3wEY= +github.com/TheCacophonyProject/go-utils v0.1.3/go.mod h1:jZPUZ4GtYVxnlTtqiYKMFWDT//kmxdbwjLW3HCyCmCE= github.com/TheCacophonyProject/modemd v0.0.0-20190605010435-ae5b0f2eb760/go.mod h1:bfwJ/WcvDY9XtHKC5tcRfVrU8RWaW8DLYAAUfsrJr/4= github.com/TheCacophonyProject/window v0.0.0-20190821235241-ab92c2ee24b6/go.mod h1:Vww417iimOb0s46Ndsm8U/vYtwc0dZUet4uW8QzBo4M= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-arg v1.1.0/go.mod h1:3Rj4baqzWaGGmZA2+bVTV8zQOZEjBQAPBnL5xLT+ftY= -github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= -github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -287,7 +287,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= @@ -308,13 +307,14 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -542,9 +542,8 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From c89abfa4c8e1bcc21759f2f7946d38a1149d432f Mon Sep 17 00:00:00 2001 From: cam Date: Sun, 29 Sep 2024 14:52:12 +1300 Subject: [PATCH 2/4] Use github actions --- .github/workflows/ci.yaml | 52 +++++++++++++++++++++++++++++++++++++++ .travis.yml | 26 -------------------- 2 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..47b7d42 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,52 @@ +name: Go CI & Release + +on: + push: + branches: + - '**' + tags: + - '*' + pull_request: + branches: + - '**' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Extract Go version from go.mod + id: go-version + run: echo "GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}')" >> $GITHUB_ENV + + - name: Setting up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Go Mod Tidy + run: go mod tidy + + - name: Install Staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@latest + + - name: Run Staticcheck + run: staticcheck --checks="all,-ST1000,-ST1022,-ST1020,-ST1003" ./... + + - name: Check for Uncommitted Changes + run: | + git diff --exit-code || ( + echo "::error::Uncommitted changes found! Please run 'go mod tidy' to clean up 'go.mod' and 'go.sum'." + exit 1 + ) + + - name: Run Go Vet + run: go vet ./... + + - name: Run Go Tests + run: go test ./... \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ad7705e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ - - -sudo: required - -services: - - docker - -language: go - -go: - - "1.17.x" - -install: - - git clone "https://github.com/TheCacophonyProject/cacophony-api" - - cd cacophony-api - - docker-compose build && docker-compose up --force-recreate -d - - cd .. - - docker cp db-test-seed.sql cacophony-api:/db-seed.sql -before_script: - - ./cacophony-api/wait-for-api || { docker ps; docker logs cacophony-api; exit 1; } - - sudo docker exec cacophony-api sh -c "sudo -i -u postgres psql cacophonytest -f/db-seed.sql" -script: - - go mod tidy - - git diff --exit-code || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files. Your go version should match the one used with travis."; false) - - go vet ./... - - go test ./... From 906f22277f031b525316aa10e497b503f6eb3dba Mon Sep 17 00:00:00 2001 From: cam Date: Sun, 29 Sep 2024 15:02:44 +1300 Subject: [PATCH 3/4] staticcheck fixes --- .github/workflows/ci.yaml | 2 +- api.go | 20 +++----------------- api_test.go | 3 +-- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 47b7d42..c281d91 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,7 +36,7 @@ jobs: run: go install honnef.co/go/tools/cmd/staticcheck@latest - name: Run Staticcheck - run: staticcheck --checks="all,-ST1000,-ST1022,-ST1020,-ST1003" ./... + run: staticcheck --checks="all,-ST1000,-ST1022,-ST1020,-ST1003,-ST1021" ./... - name: Check for Uncommitted Changes run: | diff --git a/api.go b/api.go index 54988d1..e706461 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime/multipart" "net" "net/http" @@ -73,10 +72,6 @@ func joinURL(baseURL string, paths ...string) string { return u.String() } -func (api *CacophonyAPI) getAPIURL() string { - return joinURL(api.serverURL, apiBasePath) -} - func (api *CacophonyAPI) getAuthURL() string { return joinURL(api.serverURL, authURL) } @@ -363,15 +358,6 @@ type fileUploadResponse struct { Messages []string } -// message gets the first message of the supplised tokenResponse if present -// otherwise default of "unknown" -func (r *tokenResponse) message() string { - if len(r.Messages) > 0 { - return r.Messages[0] - } - return "unknown" -} - // getFileFromJWT downloads a file from the Cacophony API using supplied JWT // and saves it to the supplied path func (api *CacophonyAPI) getFileFromJWT(jwt, filePath string) error { @@ -507,7 +493,7 @@ func (api *CacophonyAPI) ReportEvent(jsonDetails []byte, times []time.Time) erro // described in error.go func handleHTTPResponse(resp *http.Response) error { if !(isHTTPSuccess(resp.StatusCode)) { - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return temporaryError(fmt.Errorf("request failed (%d) and body read failed: %v", resp.StatusCode, err)) } @@ -547,7 +533,7 @@ func (api *CacophonyAPI) GetSchedule() ([]byte, error) { } defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) } // This allows the device to be registered even @@ -699,7 +685,7 @@ func (api *CacophonyAPI) Heartbeat(nextHeartBeat time.Time) ([]byte, error) { } defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) } // Ensure names match the API diff --git a/api_test.go b/api_test.go index 34351ba..5259be2 100644 --- a/api_test.go +++ b/api_test.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "math/rand" "mime/multipart" "net/http" @@ -364,7 +363,7 @@ func TestFileDownload(t *testing.T) { require.NoError(t, err) assert.Equal(t, fileResponse.FileSize, rawFileSize) - fileData, err := ioutil.ReadFile(filePath) + fileData, err := os.ReadFile(filePath) require.NoError(t, err) assert.Equal(t, rawFileData, string(fileData)) } From b61343d3948e957ed15a955a4b0f9f08f2f3db16 Mon Sep 17 00:00:00 2001 From: cam Date: Sun, 29 Sep 2024 20:10:20 +1300 Subject: [PATCH 4/4] Fix github actions --- .github/workflows/ci.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c281d91..a8983ad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,31 @@ jobs: exit 1 ) + - name: Install Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version # Verify installation + + - name: Clone cacophony-api repository + run: git clone "https://github.com/TheCacophonyProject/cacophony-api" + + - name: Start the cacophony-api docker server + run: | + cd cacophony-api + docker-compose build + docker-compose up --force-recreate -d + cd .. + + - name: Copy the seed SQL file + run: docker cp db-test-seed.sql cacophony-api:/db-seed.sql + + - name: Wait for API to be ready + run: ./cacophony-api/wait-for-api || { docker ps; docker logs cacophony-api; exit 1; } + + - name: Seed the database + run: sudo docker exec cacophony-api sh -c "sudo -i -u postgres psql cacophonytest -f/db-seed.sql" + - name: Run Go Vet run: go vet ./...