From 8706ad7bdc8e82c4cadde953738926962ab72941 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Tue, 29 Oct 2024 09:30:04 -0700 Subject: [PATCH 1/6] cmd/agent: handle: differentiate command and args --- cmd/agent/session.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/agent/session.go b/cmd/agent/session.go index 9ebcac8..417876a 100644 --- a/cmd/agent/session.go +++ b/cmd/agent/session.go @@ -62,44 +62,44 @@ func (s session) respond(st Status, msg string, data H) { } func (s session) handle() { - var cmds = s.Command() - if len(cmds) == 0 { + var parts = s.Command() + if len(parts) == 0 { s.respond(StatusError, "no command specified", nil) return } - var command = cmds[0] + var command, args = parts[0], parts[1:] switch command { case "version": s.respond(StatusSuccess, "", H{"version": version.Version}) case "job-status": - if len(cmds) != 2 { + if len(args) != 1 { s.respond(StatusError, "You must supply a job ID", nil) return } - s.getJobStatus(cmds[1]) + s.getJobStatus(args[0]) case "job-logs": - if len(cmds) != 2 { + if len(args) != 1 { s.respond(StatusError, "You must supply a job ID", nil) return } - s.getJobLogs(cmds[1]) + s.getJobLogs(args[0]) case "load-batch": - if len(cmds) != 2 { + if len(args) != 1 { s.respond(StatusError, fmt.Sprintf("%q requires exactly one batch name", command), nil) return } - s.loadBatch(cmds[1]) + s.loadBatch(args[0]) case "purge-batch": - if len(cmds) != 2 { + if len(args) != 1 { s.respond(StatusError, fmt.Sprintf("%q requires exactly one batch name", command), nil) return } - s.purgeBatch(cmds[1]) + s.purgeBatch(args[0]) default: s.respond(StatusError, fmt.Sprintf("%q is not a valid command name", command), nil) From a15bcf5d562bc597289dee8cbcad60bcfb8ed954 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Tue, 29 Oct 2024 13:47:30 -0700 Subject: [PATCH 2/6] Set up gitattributes for friendly word diffs --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..17689fb --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go diff=golang From 6e5a2a74351173f808a2bba9fbeb140adf66e14e Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Tue, 29 Oct 2024 13:49:00 -0700 Subject: [PATCH 3/6] cmd/agent: improve missing env error reporting --- cmd/agent/main.go | 61 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 1733c91..f43227f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -36,45 +36,58 @@ var HostKeySigner ssh.Signer var JobRunner *queue.Queue func getEnvironment() { + var errList []error + var err error + BABind = os.Getenv("BA_BIND") if BABind == "" { - slog.Error("BA_BIND must be set") - os.Exit(1) + errList = append(errList, errors.New("BA_BIND must be set")) } ONILocation = os.Getenv("ONI_LOCATION") - - var info, err = os.Stat(ONILocation) - if err == nil { - if !info.IsDir() { - err = errors.New("not a valid directory") + if ONILocation == "" { + errList = append(errList, errors.New("ONI_LOCATION must be set")) + } else { + var info, err = os.Stat(ONILocation) + if err == nil { + if !info.IsDir() { + err = errors.New("not a valid directory") + } + } + if err != nil { + errList = append(errList, fmt.Errorf("Invalid setting for ONI_LOCATION: %w", err)) } - } - if err != nil { - slog.Error("Invalid setting for ONI_LOCATION", "error", err) - os.Exit(1) } BatchSource = os.Getenv("BATCH_SOURCE") - info, err = os.Stat(BatchSource) - if err == nil { - if !info.IsDir() { - err = errors.New("not a valid directory") + if BatchSource == "" { + errList = append(errList, errors.New("BATCH_SOURCE must be set")) + } else { + var info, err = os.Stat(BatchSource) + if err == nil { + if !info.IsDir() { + err = errors.New("not a valid directory") + } + } + if err != nil { + errList = append(errList, fmt.Errorf("Invalid setting for BATCH_SOURCE: %w", err)) } - } - if err != nil { - slog.Error("Invalid setting for BATCH_SOURCE", "error", err) - os.Exit(1) } var fname = os.Getenv("HOST_KEY_FILE") if fname == "" { - slog.Error("HOST_KEY_FILE must be set") - os.Exit(1) + errList = append(errList, errors.New("HOST_KEY_FILE must be set")) + } else { + HostKeySigner, err = readKey(fname) + if err != nil { + errList = append(errList, fmt.Errorf("HOST_KEY_FILE is invalid or cannot be read: %w", err)) + } } - HostKeySigner, err = readKey(fname) - if err != nil { - slog.Error("HOST_KEY_FILE is invalid or cannot be read", "error", err) + + if len(errList) > 0 { + for _, err := range errList { + fmt.Fprintf(os.Stderr, " - %s\n", err) + } os.Exit(1) } } From bd6e90f9939ce43fdea019ed59a21606e2dbf933 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Tue, 29 Oct 2024 14:55:54 -0700 Subject: [PATCH 4/6] cmd/agent: add "ensure-awardee" command --- cmd/agent/main.go | 20 +++++++++++++++ cmd/agent/session.go | 59 ++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 4 +++ 4 files changed, 85 insertions(+) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index f43227f..eaea413 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/x509" + "database/sql" "encoding/pem" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "sync/atomic" gliderssh "github.com/gliderlabs/ssh" + _ "github.com/go-sql-driver/mysql" "github.com/open-oni/oni-agent/internal/queue" "github.com/open-oni/oni-agent/internal/version" "golang.org/x/crypto/ssh" @@ -35,6 +37,9 @@ var HostKeySigner ssh.Signer // background jobs, providing status of existing jobs, etc. var JobRunner *queue.Queue +// dbPool is our single DB connection shared app-wide +var dbPool *sql.DB + func getEnvironment() { var errList []error var err error @@ -84,12 +89,26 @@ func getEnvironment() { } } + var connect = os.Getenv("DB_CONNECTION") + if connect == "" { + errList = append(errList, errors.New(`DB_CONNECTION must be set (e.g., "user:pass@tcp(127.0.0.1:3306)/dbname")`)) + } else { + dbPool, err = sql.Open("mysql", connect) + if err != nil { + errList = append(errList, fmt.Errorf(`DB_CONNECTION is invalid: %w`, err)) + } + } + if len(errList) > 0 { for _, err := range errList { fmt.Fprintf(os.Stderr, " - %s\n", err) } os.Exit(1) } + + dbPool.SetConnMaxLifetime(0) + dbPool.SetMaxIdleConns(3) + dbPool.SetMaxOpenConns(3) } func readKey(keyfile string) (ssh.Signer, error) { @@ -168,6 +187,7 @@ func main() { trapIntTerm(func() { cancel() srv.Close() + dbPool.Close() }) go JobRunner.Wait(ctx) diff --git a/cmd/agent/session.go b/cmd/agent/session.go index 417876a..d31adcf 100644 --- a/cmd/agent/session.go +++ b/cmd/agent/session.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "encoding/json" "fmt" "log/slog" @@ -101,6 +102,13 @@ func (s session) handle() { } s.purgeBatch(args[0]) + case "ensure-awardee": + if len(args) != 2 { + s.respond(StatusError, fmt.Sprintf("%q requires two args: MARC org code and awardee name", command), nil) + return + } + s.ensureAwardee(args[0], args[1]) + default: s.respond(StatusError, fmt.Sprintf("%q is not a valid command name", command), nil) return @@ -185,6 +193,57 @@ func (s session) queueJob(command string, args ...string) { s.respond(StatusSuccess, "Job added to queue", H{"job": H{"id": id}}) } +func (s session) ensureAwardee(code string, name string) { + var rows, err = dbPool.Query("SELECT COUNT(*) FROM core_awardee WHERE org_code = ?", code) + if err != nil { + s.respond(StatusError, "Unable to query database", H{"error": err.Error()}) + return + } + defer rows.Close() + + // What does it mean if there's no error reported, but no count returned? + if !rows.Next() { + s.respond(StatusError, "Unable to count awardees in database", H{"error": "no rows returned by SQL COUNT()"}) + return + } + + var count int + err = rows.Scan(&count) + if err != nil { + s.respond(StatusError, "Unable to count awardees in database", H{"error": err.Error()}) + return + } + + // We really only care that there's at least one row. If there are dupes, + // that's out of scope to deal with, and technically not an error in terms of + // what we need. + if count > 0 { + s.respond(StatusSuccess, "Awardee already exists", nil) + return + } + + // No rows, but no error: create the awardee + var result sql.Result + result, err = dbPool.Exec("INSERT INTO core_awardee (`org_code`, `name`, `created`) VALUES(?, ?, NOW())", code, name) + if err != nil { + s.respond(StatusError, "Unable to create awardee", H{"error": err.Error(), "org_code": code, "name": name}) + return + } + var n int64 + n, err = result.RowsAffected() + if err != nil { + s.respond(StatusError, "Unable to read result of INSERT", H{"error": err.Error(), "org_code": code, "name": name}) + return + } + if n != 1 { + s.respond(StatusError, "Unable to create awardee", H{"error": "No rows created", "org_code": code, "name": name}) + return + } + + s.respond(StatusSuccess, "Awardee created", nil) + return +} + // close terminates the session, always with a status of 0: Go ssh clients // return an error if the request is anything but successful, so the caller has // to parse the status instead. diff --git a/go.mod b/go.mod index 25abf42..d163bc7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.22.5 require github.com/gliderlabs/ssh v0.3.7 require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/google/go-cmp v0.6.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/sys v0.25.0 // indirect diff --git a/go.sum b/go.sum index edecaec..5664e41 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= From 443741b459841785da26ff2b0b653c15956ed183 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Wed, 30 Oct 2024 11:39:18 -0700 Subject: [PATCH 5/6] Document and show examples of DB_CONNECTION var --- README.md | 1 + oni-agent.service | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7859c13..31e304a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ export BA_BIND=":2222" export BATCH_SOURCE="/mnt/news/production-batches" export ONI_LOCATION="/opt/openoni/" export HOST_KEY_FILE="/etc/oni-agent" +export DB_CONNECTION="user:password@tcp(127.0.0.1:3306)/databasename" make ./bin/agent ``` diff --git a/oni-agent.service b/oni-agent.service index 2701768..5235316 100644 --- a/oni-agent.service +++ b/oni-agent.service @@ -5,9 +5,10 @@ [Service] Environment="BA_BIND=:2222" - Environment="BATCH_SOURCE=/mnt/news/production-batches" Environment="ONI_LOCATION=/opt/openoni/" + Environment="BATCH_SOURCE=/mnt/news/production-batches" Environment="HOST_KEY_FILE=/etc/oni-agent" + Environment="DB_CONNECTION=user:password@tcp(127.0.0.1:3306)/databasename" Type=simple ExecStart=/usr/local/oni-agent/agent SyslogIdentifier=oni-agent From b1ad0581f4688d6ea1a8299bb69d58d4d30aa1e3 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Fri, 1 Nov 2024 14:45:37 -0700 Subject: [PATCH 6/6] cmd/agent: allow awardee check when name is empty ensure-awardee essentially becomes a read-only check with no name, but there are occasionally awardees without names in NCA from very old legacy data. In these cases, ONI will almost certainly already have the awardee data, and if not, we'll know to edit NCA. --- cmd/agent/session.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/agent/session.go b/cmd/agent/session.go index d31adcf..f49aff9 100644 --- a/cmd/agent/session.go +++ b/cmd/agent/session.go @@ -103,10 +103,14 @@ func (s session) handle() { s.purgeBatch(args[0]) case "ensure-awardee": - if len(args) != 2 { - s.respond(StatusError, fmt.Sprintf("%q requires two args: MARC org code and awardee name", command), nil) + if len(args) < 1 || len(args) > 2 { + s.respond(StatusError, fmt.Sprintf("%q requires one or two args: MARC org code and awardee name. Name is required if the awardee is to be auto-created.", command), nil) return } + + if len(args) == 1 { + args = []string{args[0], ""} + } s.ensureAwardee(args[0], args[1]) default: @@ -222,7 +226,12 @@ func (s session) ensureAwardee(code string, name string) { return } - // No rows, but no error: create the awardee + // No rows, no error: if a name was given, create the awardee, otherwise abort + if name == "" { + s.respond(StatusError, "Unable to create awardee", H{"error": "awardee name must be given to auto-create awardees", "org_code": code, "name": name}) + return + } + var result sql.Result result, err = dbPool.Exec("INSERT INTO core_awardee (`org_code`, `name`, `created`) VALUES(?, ?, NOW())", code, name) if err != nil {