From 4f66ef504e3f5f29564c2e8fac4d24ecfc218284 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Tue, 17 Sep 2024 03:12:38 +0530 Subject: [PATCH 1/6] feat: working directory for tasks, and dotenv loading is now relative to the runfile --- Runfile | 3 ++- examples/Runfile | 2 +- pkg/functions/helpers.go | 18 ++++++++++++++++++ pkg/runfile/parser.go | 10 +++++++++- pkg/runfile/run.go | 36 +++++++++++++++++++++++++++++------- pkg/runfile/type.go | 13 ++++++++++--- 6 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 pkg/functions/helpers.go diff --git a/Runfile b/Runfile index 65116bd..20c5fbc 100644 --- a/Runfile +++ b/Runfile @@ -11,6 +11,7 @@ tasks: echo "DONE" example: + dir: ./examples cmd: - |+ - run -f ./examples/Runfile cook + run cook diff --git a/examples/Runfile b/examples/Runfile index 3130c74..5a1e8d4 100644 --- a/examples/Runfile +++ b/examples/Runfile @@ -9,7 +9,7 @@ tasks: k3: sh: echo -n "hello" dotenv: - - .secrets/env + - ../.secrets/env cmd: - echo "hi hello" - echo "value of k1 is '$k1'" diff --git a/pkg/functions/helpers.go b/pkg/functions/helpers.go new file mode 100644 index 0000000..ad75806 --- /dev/null +++ b/pkg/functions/helpers.go @@ -0,0 +1,18 @@ +package functions + +func DefaultIfNil[T any](v *T, dv T) T { + if v == nil { + return dv + } + return *v +} + +// Must panics if err is not nil +// It is intended to be used very sparingly, and only in cases where the caller is +// certain that the error will never be nil in ideal scenarios +func Must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} diff --git a/pkg/runfile/parser.go b/pkg/runfile/parser.go index 33ae39b..111f0dd 100644 --- a/pkg/runfile/parser.go +++ b/pkg/runfile/parser.go @@ -4,9 +4,11 @@ import ( "bufio" "fmt" "os" + "path/filepath" "strconv" "strings" + fn "github.com/nxtcoder17/runfile/pkg/functions" "sigs.k8s.io/yaml" ) @@ -18,8 +20,10 @@ func ParseRunFile(file string) (*RunFile, error) { } err = yaml.Unmarshal(f, &runfile) if err != nil { - return &runfile, err + return nil, err } + + runfile.attrs.RunfilePath = fn.Must(filepath.Abs(file)) return &runfile, nil } @@ -28,6 +32,10 @@ func parseDotEnv(files ...string) ([]string, error) { results := make([]string, 0, 5) for i := range files { + if !filepath.IsAbs(files[i]) { + return nil, fmt.Errorf("dotenv file path %s, must be absolute", files[i]) + } + f, err := os.Open(files[i]) if err != nil { return nil, err diff --git a/pkg/runfile/run.go b/pkg/runfile/run.go index 21b70ef..87b55b0 100644 --- a/pkg/runfile/run.go +++ b/pkg/runfile/run.go @@ -7,12 +7,17 @@ import ( "io" "os" "os/exec" + "path/filepath" + + fn "github.com/nxtcoder17/runfile/pkg/functions" ) type runArgs struct { - shell []string - env []string // [key=value, key=value, ...] - cmd string + shell []string + env []string // [key=value, key=value, ...] + workingDir string + + cmd string stdout io.Writer stderr io.Writer @@ -42,6 +47,7 @@ func runInShell(ctx context.Context, args runArgs) error { // cargs := append(args.shell[1:], f.Name()) cargs := append(args.shell[1:], args.cmd) c := exec.CommandContext(ctx, shell, cargs...) + c.Dir = args.workingDir c.Env = args.env c.Stdout = args.stdout c.Stderr = args.stderr @@ -86,8 +92,23 @@ func (r *RunFile) Run(ctx context.Context, taskName string) error { } } + dotenvPaths := make([]string, len(task.DotEnv)) + for i, v := range task.DotEnv { + dotenvPath := filepath.Join(filepath.Dir(r.attrs.RunfilePath), v) + fi, err := os.Stat(dotenvPath) + if err != nil { + return err + } + + if fi.IsDir() { + return fmt.Errorf("dotenv file must be a file, but %s is a directory", v) + } + + dotenvPaths[i] = dotenvPath + } + // parsing dotenv - s, err := parseDotEnv(task.DotEnv...) + s, err := parseDotEnv(dotenvPaths...) if err != nil { return err } @@ -97,9 +118,10 @@ func (r *RunFile) Run(ctx context.Context, taskName string) error { for _, cmd := range task.Commands { runInShell(ctx, runArgs{ - shell: task.Shell, - env: append(os.Environ(), env...), - cmd: cmd, + shell: task.Shell, + env: append(os.Environ(), env...), + cmd: cmd, + workingDir: fn.DefaultIfNil(task.Dir, fn.Must(os.Getwd())), }) } return nil diff --git a/pkg/runfile/type.go b/pkg/runfile/type.go index f62304d..2cf2ba7 100644 --- a/pkg/runfile/type.go +++ b/pkg/runfile/type.go @@ -1,14 +1,21 @@ package runfile +type runfileAttrs struct { + RunfilePath string +} + type RunFile struct { - Version string + attrs runfileAttrs - Tasks map[string]TaskSpec `json:"tasks"` + Version string + Tasks map[string]TaskSpec `json:"tasks"` } type TaskSpec struct { // load env vars from [.env](https://www.google.com/search?q=sample+dotenv+files&udm=2) files - DotEnv []string `json:"dotenv"` + DotEnv []string `json:"dotenv"` + // working directory for the task + Dir *string `json:"dir,omitempty"` Env map[string]any `json:"env"` Commands []string `json:"cmd"` Shell []string `json:"shell"` From 61aafba11ba68a25c0134377790b4447d9d68a93 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Wed, 25 Sep 2024 23:07:48 +0530 Subject: [PATCH 2/6] feat: parallel tasks, and loads of refactoring --- Runfile | 2 +- cmd/run/main.go | 71 ++++++++++++++++--- examples/Runfile | 7 +- go.mod | 19 ++++- go.sum | 36 +++++++++- main.go | 7 -- pkg/runfile/parser.go | 48 ++++--------- pkg/runfile/parser_test.go | 103 ++++++++++++++++++++++++++++ pkg/runfile/run.go | 137 +++++++++++++++++++++++-------------- pkg/runfile/run_test.go | 38 ++++++++++ pkg/runfile/runfile.go | 35 ++++++++++ pkg/runfile/task.go | 36 ++++++++++ pkg/runfile/type.go | 71 +++++++++++++++---- 13 files changed, 487 insertions(+), 123 deletions(-) delete mode 100644 main.go create mode 100644 pkg/runfile/parser_test.go create mode 100644 pkg/runfile/run_test.go create mode 100644 pkg/runfile/runfile.go create mode 100644 pkg/runfile/task.go diff --git a/Runfile b/Runfile index 20c5fbc..4249cdf 100644 --- a/Runfile +++ b/Runfile @@ -14,4 +14,4 @@ tasks: dir: ./examples cmd: - |+ - run cook + run cook clean diff --git a/cmd/run/main.go b/cmd/run/main.go index bfe2a49..c8a6c14 100644 --- a/cmd/run/main.go +++ b/cmd/run/main.go @@ -9,6 +9,7 @@ import ( "path/filepath" "syscall" + "github.com/nxtcoder17/fwatcher/pkg/logging" "github.com/nxtcoder17/runfile/pkg/runfile" "github.com/urfave/cli/v3" ) @@ -26,6 +27,23 @@ func main() { Aliases: []string{"f"}, Value: "", }, + + &cli.BoolFlag{ + Name: "parallel", + Aliases: []string{"p"}, + Value: false, + }, + + &cli.BoolFlag{ + Name: "watch", + Aliases: []string{"w"}, + Value: false, + }, + + &cli.BoolFlag{ + Name: "debug", + Value: false, + }, }, EnableShellCompletion: true, ShellComplete: func(ctx context.Context, c *cli.Command) { @@ -38,7 +56,7 @@ func main() { panic(err) } - runfile, err := runfile.ParseRunFile(runfilePath) + runfile, err := runfile.Parse(runfilePath) if err != nil { panic(err) } @@ -48,11 +66,18 @@ func main() { } }, Action: func(ctx context.Context, c *cli.Command) error { - if c.Args().Len() > 1 { - return fmt.Errorf("too many arguments") - } - if c.Args().Len() != 1 { - return fmt.Errorf("missing argument") + parallel := c.Bool("parallel") + watch := c.Bool("watch") + debug := c.Bool("debug") + + logging.NewSlogLogger(logging.SlogOptions{ + ShowCaller: debug, + ShowDebugLogs: debug, + SetAsDefaultLogger: true, + }) + + if c.Args().Len() < 1 { + return fmt.Errorf("missing argument, at least one argument is required") } runfilePath, err := locateRunfile(c) @@ -60,13 +85,41 @@ func main() { return err } - runfile, err := runfile.ParseRunFile(runfilePath) + rf, err := runfile.Parse(runfilePath) if err != nil { panic(err) } - s := c.Args().First() - return runfile.Run(ctx, s) + args := make([]string, 0, len(c.Args().Slice())) + for _, arg := range c.Args().Slice() { + if arg == "-p" || arg == "--parallel" { + parallel = true + continue + } + + if arg == "-w" || arg == "--watch" { + watch = true + continue + } + + if arg == "--debug" { + debug = true + continue + } + + args = append(args, arg) + } + + if parallel && watch { + return fmt.Errorf("parallel and watch can't be set together") + } + + return rf.Run(ctx, runfile.RunArgs{ + Tasks: args, + ExecuteInParallel: parallel, + Watch: watch, + Debug: debug, + }) }, } diff --git a/examples/Runfile b/examples/Runfile index 5a1e8d4..e9c548b 100644 --- a/examples/Runfile +++ b/examples/Runfile @@ -11,6 +11,7 @@ tasks: dotenv: - ../.secrets/env cmd: + # - sleep 5 - echo "hi hello" - echo "value of k1 is '$k1'" - echo "value of k2 is '$k2'" @@ -21,13 +22,15 @@ tasks: name: clean shell: ["python", "-c"] dotenv: - - .secrets/env + - ../.secrets/env cmd: - |+ import secrets import os + import time print(os.environ['key_id']) - # print(secrets.token_hex(32)) + time.sleep(6) + print(secrets.token_hex(32)) laundry: name: laundry shell: ["node", "-e"] diff --git a/go.mod b/go.mod index e4160a9..b1527b8 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,25 @@ module github.com/nxtcoder17/runfile go 1.22.7 require ( + github.com/joho/godotenv v1.5.1 + github.com/nxtcoder17/fwatcher v1.0.1 github.com/urfave/cli/v3 v3.0.0-alpha9 + golang.org/x/sync v0.8.0 sigs.k8s.io/yaml v1.4.0 ) -require github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect +require ( + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/lipgloss v0.13.0 // indirect + github.com/charmbracelet/log v0.4.0 // indirect + github.com/charmbracelet/x/ansi v0.1.4 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/sys v0.19.0 // indirect +) diff --git a/go.sum b/go.sum index 8c4c5ae..dd09597 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,47 @@ +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= +github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM= +github.com/charmbracelet/log v0.4.0/go.mod h1:63bXt/djrizTec0l11H20t8FDSvA4CRZJ1KH22MdptM= +github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= +github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/nxtcoder17/fwatcher v1.0.1 h1:Rqy+7etcGv9L1KIoK8YGGpAhdXW/pkfkXQwdlJzL1a8= +github.com/nxtcoder17/fwatcher v1.0.1/go.mod h1:MNmSwXYOrqp7U1pUxh0GWB5skpjFTWTQXhAA0+sPJcU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/urfave/cli/v3 v3.0.0-alpha9 h1:P0RMy5fQm1AslQS+XCmy9UknDXctOmG/q/FZkUFnJSo= github.com/urfave/cli/v3 v3.0.0-alpha9/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go deleted file mode 100644 index 57040c6..0000000 --- a/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "fmt" - -func main() { - fmt.Println("RUN FILE") -} diff --git a/pkg/runfile/parser.go b/pkg/runfile/parser.go index 111f0dd..91c2235 100644 --- a/pkg/runfile/parser.go +++ b/pkg/runfile/parser.go @@ -1,35 +1,21 @@ package runfile import ( - "bufio" "fmt" + "io" "os" "path/filepath" - "strconv" - "strings" - fn "github.com/nxtcoder17/runfile/pkg/functions" - "sigs.k8s.io/yaml" + "github.com/joho/godotenv" ) -func ParseRunFile(file string) (*RunFile, error) { - var runfile RunFile - f, err := os.ReadFile(file) - if err != nil { - return &runfile, err - } - err = yaml.Unmarshal(f, &runfile) - if err != nil { - return nil, err - } - - runfile.attrs.RunfilePath = fn.Must(filepath.Abs(file)) - return &runfile, nil +func parseDotEnv(reader io.Reader) (map[string]string, error) { + return godotenv.Parse(reader) } // parseDotEnv parses the .env file and returns a slice of strings as in os.Environ() -func parseDotEnv(files ...string) ([]string, error) { - results := make([]string, 0, 5) +func parseDotEnvFiles(files ...string) (map[string]string, error) { + results := make(map[string]string) for i := range files { if !filepath.IsAbs(files[i]) { @@ -40,24 +26,16 @@ func parseDotEnv(files ...string) ([]string, error) { if err != nil { return nil, err } + m, err := parseDotEnv(f) + if err != nil { + return nil, err + } + f.Close() - s := bufio.NewScanner(f) - for s.Scan() { - s2 := strings.SplitN(s.Text(), "=", 2) - if len(s2) != 2 { - continue - } - s, _ := strconv.Unquote(string(s2[1])) - - // os.Setenv(s2[0], s2[1]) - os.Setenv(s2[0], s) - results = append(results, s2[0]) + for k, v := range m { + results[k] = v } - } - for i := range results { - v := os.Getenv(results[i]) - results[i] = fmt.Sprintf("%s=%v", results[i], v) } return results, nil diff --git a/pkg/runfile/parser_test.go b/pkg/runfile/parser_test.go new file mode 100644 index 0000000..a33331f --- /dev/null +++ b/pkg/runfile/parser_test.go @@ -0,0 +1,103 @@ +package runfile + +import ( + "bytes" + "io" + "reflect" + "testing" +) + +func Test_parseDotEnvFile(t *testing.T) { + type args struct { + reader io.Reader + } + tests := []struct { + name string + args args + want map[string]string + wantErr bool + }{ + { + name: "key=", + args: args{ + reader: bytes.NewBuffer([]byte(`key=`)), + }, + want: map[string]string{ + "key": "", + }, + wantErr: false, + }, + { + name: "key=1", + args: args{ + reader: bytes.NewBuffer([]byte(`key=1`)), + }, + want: map[string]string{ + "key": "1", + }, + wantErr: false, + }, + { + name: "key=one", + args: args{ + reader: bytes.NewBuffer([]byte(`key=one`)), + }, + want: map[string]string{ + "key": "one", + }, + wantErr: false, + }, + { + name: "key='one'", + args: args{ + reader: bytes.NewBuffer([]byte(`key='one'`)), + }, + want: map[string]string{ + "key": "one", + }, + wantErr: false, + }, + { + name: `key='o"ne'`, + args: args{ + reader: bytes.NewBuffer([]byte(`key='o"ne'`)), + }, + want: map[string]string{ + "key": `o"ne`, + }, + wantErr: false, + }, + { + name: `key="one"`, + args: args{ + reader: bytes.NewBuffer([]byte(`key="one"`)), + }, + want: map[string]string{ + "key": `one`, + }, + wantErr: false, + }, + { + name: `key=sample==`, + args: args{ + reader: bytes.NewBuffer([]byte(`key=sample==`)), + }, + want: map[string]string{ + "key": `sample==`, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseDotEnv(tt.args.reader) + if (err != nil) != tt.wantErr { + t.Errorf("parseDotEnvFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseDotEnvFile()\n\t got: %#v,\n\twant: %#v", got, tt.want) + } + }) + } +} diff --git a/pkg/runfile/run.go b/pkg/runfile/run.go index 87b55b0..676a928 100644 --- a/pkg/runfile/run.go +++ b/pkg/runfile/run.go @@ -1,18 +1,20 @@ package runfile import ( - "bytes" "context" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" + "strings" fn "github.com/nxtcoder17/runfile/pkg/functions" + "golang.org/x/sync/errgroup" ) -type runArgs struct { +type cmdArgs struct { shell []string env []string // [key=value, key=value, ...] workingDir string @@ -23,7 +25,7 @@ type runArgs struct { stderr io.Writer } -func runInShell(ctx context.Context, args runArgs) error { +func createCommand(ctx context.Context, args cmdArgs) *exec.Cmd { if args.shell == nil { args.shell = []string{"sh", "-c"} } @@ -51,50 +53,18 @@ func runInShell(ctx context.Context, args runArgs) error { c.Env = args.env c.Stdout = args.stdout c.Stderr = args.stderr - return c.Run() + return c } -func (r *RunFile) Run(ctx context.Context, taskName string) error { - task, ok := r.Tasks[taskName] - if !ok { - return fmt.Errorf("task %s not found", taskName) - } - - env := make([]string, len(task.Env)) - for k, v := range task.Env { - switch v := v.(type) { - case string: - env = append(env, fmt.Sprintf("%s=%s", k, v)) - case map[string]any: - shcmd, ok := v["sh"] - if !ok { - return fmt.Errorf("env %s is not a string", k) - } - - s, ok := shcmd.(string) - if !ok { - return fmt.Errorf("shell cmd is not a string") - } - - value := new(bytes.Buffer) - - if err := runInShell(ctx, runArgs{ - shell: task.Shell, - env: os.Environ(), - cmd: s, - stdout: value, - }); err != nil { - return err - } - env = append(env, fmt.Sprintf("%s=%v", k, value.String())) - default: - panic(fmt.Sprintf("env %s is not a string (%T)", k, v)) - } +func (rf *Runfile) runTask(ctx context.Context, task Task) error { + shell := task.Shell + if shell == nil { + shell = []string{"sh", "-c"} } dotenvPaths := make([]string, len(task.DotEnv)) for i, v := range task.DotEnv { - dotenvPath := filepath.Join(filepath.Dir(r.attrs.RunfilePath), v) + dotenvPath := filepath.Join(filepath.Dir(rf.attrs.RunfilePath), v) fi, err := os.Stat(dotenvPath) if err != nil { return err @@ -108,21 +78,86 @@ func (r *RunFile) Run(ctx context.Context, taskName string) error { } // parsing dotenv - s, err := parseDotEnv(dotenvPaths...) + dotEnvVars, err := parseDotEnvFiles(dotenvPaths...) if err != nil { return err } + env := make([]string, 0, len(os.Environ())+len(dotEnvVars)) + env = append(env, os.Environ()...) + for k, v := range dotEnvVars { + env = append(env, fmt.Sprintf("%s=%v", k, v)) + } + // INFO: keys from task.Env will override those coming from dotenv files, when duplicated - env = append(s, env...) - - for _, cmd := range task.Commands { - runInShell(ctx, runArgs{ - shell: task.Shell, - env: append(os.Environ(), env...), - cmd: cmd, - workingDir: fn.DefaultIfNil(task.Dir, fn.Must(os.Getwd())), - }) + envVars, err := parseEnvVars(ctx, task.Env, EvaluationArgs{ + Shell: task.Shell, + Environ: env, + }) + if err != nil { + return err + } + + for k, v := range envVars { + env = append(env, fmt.Sprintf("%s=%v", k, v)) + } + + script := make([]string, 0, len(task.Commands)) + + for _, command := range task.Commands { + script = append(script, command) + } + + cmd := createCommand(ctx, cmdArgs{ + shell: task.Shell, + env: env, + cmd: strings.Join(script, "\n"), + workingDir: fn.DefaultIfNil(task.Dir, fn.Must(os.Getwd())), + }) + + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +type RunArgs struct { + Tasks []string + ExecuteInParallel bool + Watch bool + Debug bool +} + +func (rf *Runfile) Run(ctx context.Context, args RunArgs) error { + for _, v := range args.Tasks { + if _, ok := rf.Tasks[v]; !ok { + return ErrTaskNotFound{TaskName: v, RunfilePath: rf.attrs.RunfilePath} + } + } + + if args.ExecuteInParallel { + slog.Default().Debug("running in parallel mode", "tasks", args.Tasks) + g := new(errgroup.Group) + + for _, tn := range args.Tasks { + g.Go(func() error { + return rf.runTask(ctx, rf.Tasks[tn]) + }) + } + + // Wait for all tasks to finish + if err := g.Wait(); err != nil { + return err + } + + return nil } + + for _, tn := range args.Tasks { + if err := rf.runTask(ctx, rf.Tasks[tn]); err != nil { + return err + } + } + return nil } diff --git a/pkg/runfile/run_test.go b/pkg/runfile/run_test.go new file mode 100644 index 0000000..2f84915 --- /dev/null +++ b/pkg/runfile/run_test.go @@ -0,0 +1,38 @@ +package runfile + +import ( + "context" + "testing" +) + +func TestRunFile_Run(t *testing.T) { + type fields struct { + attrs attrs + Version string + Tasks map[string]TaskSpec + } + type args struct { + ctx context.Context + tasks []string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Runfile{ + attrs: tt.fields.attrs, + Version: tt.fields.Version, + Tasks: tt.fields.Tasks, + } + if err := r.Run(tt.args.ctx, tt.args.tasks); (err != nil) != tt.wantErr { + t.Errorf("RunFile.Run() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/runfile/runfile.go b/pkg/runfile/runfile.go new file mode 100644 index 0000000..bc483da --- /dev/null +++ b/pkg/runfile/runfile.go @@ -0,0 +1,35 @@ +package runfile + +import ( + "os" + "path/filepath" + + fn "github.com/nxtcoder17/runfile/pkg/functions" + "sigs.k8s.io/yaml" +) + +type attrs struct { + RunfilePath string +} + +type Runfile struct { + attrs attrs + + Version string + Tasks map[string]Task `json:"tasks"` +} + +func Parse(file string) (*Runfile, error) { + var runfile Runfile + f, err := os.ReadFile(file) + if err != nil { + return &runfile, err + } + err = yaml.Unmarshal(f, &runfile) + if err != nil { + return nil, err + } + + runfile.attrs.RunfilePath = fn.Must(filepath.Abs(file)) + return &runfile, nil +} diff --git a/pkg/runfile/task.go b/pkg/runfile/task.go new file mode 100644 index 0000000..044485c --- /dev/null +++ b/pkg/runfile/task.go @@ -0,0 +1,36 @@ +package runfile + +import ( + "fmt" +) + +type Task struct { + // load env vars from [.env](https://www.google.com/search?q=sample+dotenv+files&udm=2) files + DotEnv []string `json:"dotenv"` + + // working directory for the task + Dir *string `json:"dir,omitempty"` + + Env map[string]any `json:"env"` + + // List of commands to be executed in given shell (default: sh) + Commands []string `json:"cmd"` + + // Shell in which above commands will be executed + // Default: ["sh", "-c"] + /* Common Usecases could be: + - ["bash", "-c"] + - ["python", "-c"] + - ["node", "-e"] + */ + Shell []string `json:"shell"` +} + +type ErrTaskNotFound struct { + TaskName string + RunfilePath string +} + +func (e ErrTaskNotFound) Error() string { + return fmt.Sprintf("[task] %s, not found in [Runfile] at %s", e.TaskName, e.RunfilePath) +} diff --git a/pkg/runfile/type.go b/pkg/runfile/type.go index 2cf2ba7..0cb4f6c 100644 --- a/pkg/runfile/type.go +++ b/pkg/runfile/type.go @@ -1,22 +1,63 @@ package runfile -type runfileAttrs struct { - RunfilePath string -} +import ( + "bytes" + "context" + "fmt" +) + +/* +EnvVar Values could take multiple forms: + +- my_key: "value" + +or + + - my_key: + "sh": "echo hello hi" -type RunFile struct { - attrs runfileAttrs +Object values with `sh` key, such that the output of this command will be the value of the top-level key +*/ +type EnvVar map[string]any - Version string - Tasks map[string]TaskSpec `json:"tasks"` +type EvaluationArgs struct { + Shell []string + Environ []string } -type TaskSpec struct { - // load env vars from [.env](https://www.google.com/search?q=sample+dotenv+files&udm=2) files - DotEnv []string `json:"dotenv"` - // working directory for the task - Dir *string `json:"dir,omitempty"` - Env map[string]any `json:"env"` - Commands []string `json:"cmd"` - Shell []string `json:"shell"` +func parseEnvVars(ctx context.Context, ev EnvVar, args EvaluationArgs) (map[string]string, error) { + env := make(map[string]string, len(ev)) + for k, v := range ev { + switch v := v.(type) { + case string: + env[k] = v + case map[string]any: + shcmd, ok := v["sh"] + if !ok { + return nil, fmt.Errorf("sh key is missing") + } + + s, ok := shcmd.(string) + if !ok { + return nil, fmt.Errorf("shell cmd is not a string") + } + + value := new(bytes.Buffer) + + cmd := createCommand(ctx, cmdArgs{ + shell: args.Shell, + env: args.Environ, + cmd: s, + stdout: value, + }) + if err := cmd.Run(); err != nil { + return nil, err + } + env[k] = value.String() + default: + panic(fmt.Sprintf("env %s is not a string (%T)", k, v)) + } + } + + return env, nil } From 8fbbe8bf436bc712e07f28eb91d277cd09c72c4e Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Fri, 27 Sep 2024 16:11:02 +0530 Subject: [PATCH 3/6] feat: supports calling other tasks from one task - adds tests for parsing pkg/runfile --- Runfile | 4 + examples/Runfile | 23 +- flake.nix | 2 + pkg/functions/helpers.go | 4 + pkg/runfile/run.go | 90 ++---- pkg/runfile/run_test.go | 37 --- pkg/runfile/task-parser.go | 175 ++++++++++++ pkg/runfile/task-parser_test.go | 484 ++++++++++++++++++++++++++++++++ pkg/runfile/task.go | 13 +- pkg/runfile/type.go | 5 +- 10 files changed, 721 insertions(+), 116 deletions(-) create mode 100644 pkg/runfile/task-parser.go create mode 100644 pkg/runfile/task-parser_test.go diff --git a/Runfile b/Runfile index 4249cdf..38097d4 100644 --- a/Runfile +++ b/Runfile @@ -15,3 +15,7 @@ tasks: cmd: - |+ run cook clean + + test: + cmd: + - go test -json ./pkg/runfile | gotestfmt diff --git a/examples/Runfile b/examples/Runfile index e9c548b..3e04672 100644 --- a/examples/Runfile +++ b/examples/Runfile @@ -12,11 +12,12 @@ tasks: - ../.secrets/env cmd: # - sleep 5 - - echo "hi hello" - - echo "value of k1 is '$k1'" - - echo "value of k2 is '$k2'" - - echo "value of k3 is '$k3'" - - echo "value of key_id (from .dotenv) is '$key_id', ${#key_id}" + # - echo "hi hello" + # - echo "value of k1 is '$k1'" + # - echo "value of k2 is '$k2'" + # - echo "value of k3 is '$k3'" + # - echo "value of key_id (from .dotenv) is '$key_id', ${#key_id}" + - echo "hello from cook" clean: name: clean @@ -24,18 +25,24 @@ tasks: dotenv: - ../.secrets/env cmd: + - run: laundry + # vars: + # k1: v1 - |+ import secrets import os import time - print(os.environ['key_id']) - time.sleep(6) + # print("key_id from env: ", os.environ['key_id']) + time.sleep(2) + print("hello from clean") print(secrets.token_hex(32)) + laundry: name: laundry shell: ["node", "-e"] cmd: - - console.log("laundry") + - run: cook + - console.log("hello from laundry") eat: name: eat cmd: diff --git a/flake.nix b/flake.nix index dad0011..633c673 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,8 @@ go_1_22 upx + + gotestfmt ]; shellHook = '' diff --git a/pkg/functions/helpers.go b/pkg/functions/helpers.go index ad75806..51b6ccb 100644 --- a/pkg/functions/helpers.go +++ b/pkg/functions/helpers.go @@ -16,3 +16,7 @@ func Must[T any](v T, err error) T { } return v } + +func New[T any](v T) *T { + return &v +} diff --git a/pkg/runfile/run.go b/pkg/runfile/run.go index 676a928..9e40f5a 100644 --- a/pkg/runfile/run.go +++ b/pkg/runfile/run.go @@ -2,15 +2,11 @@ package runfile import ( "context" - "fmt" "io" "log/slog" "os" "os/exec" - "path/filepath" - "strings" - fn "github.com/nxtcoder17/runfile/pkg/functions" "golang.org/x/sync/errgroup" ) @@ -39,14 +35,7 @@ func createCommand(ctx context.Context, args cmdArgs) *exec.Cmd { } shell := args.shell[0] - // f, err := os.CreateTemp(os.TempDir(), "runfile-") - // if err != nil { - // return err - // } - // f.WriteString(args.cmd) - // f.Close() - - // cargs := append(args.shell[1:], f.Name()) + cargs := append(args.shell[1:], args.cmd) c := exec.CommandContext(ctx, shell, cargs...) c.Dir = args.workingDir @@ -56,68 +45,33 @@ func createCommand(ctx context.Context, args cmdArgs) *exec.Cmd { return c } -func (rf *Runfile) runTask(ctx context.Context, task Task) error { - shell := task.Shell - if shell == nil { - shell = []string{"sh", "-c"} - } - - dotenvPaths := make([]string, len(task.DotEnv)) - for i, v := range task.DotEnv { - dotenvPath := filepath.Join(filepath.Dir(rf.attrs.RunfilePath), v) - fi, err := os.Stat(dotenvPath) - if err != nil { - return err - } - - if fi.IsDir() { - return fmt.Errorf("dotenv file must be a file, but %s is a directory", v) - } - - dotenvPaths[i] = dotenvPath - } - - // parsing dotenv - dotEnvVars, err := parseDotEnvFiles(dotenvPaths...) +func (rf *Runfile) runTask(ctx context.Context, taskName string) error { + pt, err := ParseTask(ctx, rf, taskName) if err != nil { return err } - env := make([]string, 0, len(os.Environ())+len(dotEnvVars)) - env = append(env, os.Environ()...) - for k, v := range dotEnvVars { - env = append(env, fmt.Sprintf("%s=%v", k, v)) - } + // slog.Default().Info("parsing", "task", pt) - // INFO: keys from task.Env will override those coming from dotenv files, when duplicated - envVars, err := parseEnvVars(ctx, task.Env, EvaluationArgs{ - Shell: task.Shell, - Environ: env, - }) - if err != nil { - return err - } - - for k, v := range envVars { - env = append(env, fmt.Sprintf("%s=%v", k, v)) - } - - script := make([]string, 0, len(task.Commands)) + for _, command := range pt.Commands { + if command.Run != "" { + if err := rf.runTask(ctx, command.Run); err != nil { + return err + } + continue + } - for _, command := range task.Commands { - script = append(script, command) + cmd := createCommand(ctx, cmdArgs{ + shell: pt.Shell, + env: pt.Environ, + cmd: command.Command, + workingDir: pt.WorkingDir, + }) + if err := cmd.Run(); err != nil { + return err + } } - cmd := createCommand(ctx, cmdArgs{ - shell: task.Shell, - env: env, - cmd: strings.Join(script, "\n"), - workingDir: fn.DefaultIfNil(task.Dir, fn.Must(os.Getwd())), - }) - - if err := cmd.Run(); err != nil { - return err - } return nil } @@ -141,7 +95,7 @@ func (rf *Runfile) Run(ctx context.Context, args RunArgs) error { for _, tn := range args.Tasks { g.Go(func() error { - return rf.runTask(ctx, rf.Tasks[tn]) + return rf.runTask(ctx, tn) }) } @@ -154,7 +108,7 @@ func (rf *Runfile) Run(ctx context.Context, args RunArgs) error { } for _, tn := range args.Tasks { - if err := rf.runTask(ctx, rf.Tasks[tn]); err != nil { + if err := rf.runTask(ctx, tn); err != nil { return err } } diff --git a/pkg/runfile/run_test.go b/pkg/runfile/run_test.go index 2f84915..a5515d8 100644 --- a/pkg/runfile/run_test.go +++ b/pkg/runfile/run_test.go @@ -1,38 +1 @@ package runfile - -import ( - "context" - "testing" -) - -func TestRunFile_Run(t *testing.T) { - type fields struct { - attrs attrs - Version string - Tasks map[string]TaskSpec - } - type args struct { - ctx context.Context - tasks []string - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := &Runfile{ - attrs: tt.fields.attrs, - Version: tt.fields.Version, - Tasks: tt.fields.Tasks, - } - if err := r.Run(tt.args.ctx, tt.args.tasks); (err != nil) != tt.wantErr { - t.Errorf("RunFile.Run() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/pkg/runfile/task-parser.go b/pkg/runfile/task-parser.go new file mode 100644 index 0000000..e7772bd --- /dev/null +++ b/pkg/runfile/task-parser.go @@ -0,0 +1,175 @@ +package runfile + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + fn "github.com/nxtcoder17/runfile/pkg/functions" +) + +type ParsedTask struct { + Shell []string `json:"shell"` + WorkingDir string `json:"workingDir"` + Environ []string `json:"environ"` + Commands []CommandJson `json:"commands"` +} + +type ErrTask struct { + ErrTaskNotFound + Message string + Err error +} + +func (e ErrTask) Error() string { + return fmt.Sprintf("[task] %s (Runfile: %s), says %s, got %v", e.TaskName, e.RunfilePath, e.Message, e.Err) +} + +func NewErrTask(runfilePath, taskName, message string, err error) ErrTask { + return ErrTask{ + ErrTaskNotFound: ErrTaskNotFound{ + RunfilePath: runfilePath, + TaskName: taskName, + }, + Message: message, + Err: err, + } +} + +func ParseTask(ctx context.Context, rf *Runfile, taskName string) (*ParsedTask, error) { + newErr := func(message string, err error) ErrTask { + return NewErrTask(rf.attrs.RunfilePath, taskName, message, err) + } + + task, ok := rf.Tasks[taskName] + if !ok { + return nil, ErrTaskNotFound{TaskName: taskName, RunfilePath: rf.attrs.RunfilePath} + } + + if task.Shell == nil { + task.Shell = []string{"sh", "-c"} + } + + if task.Dir == nil { + task.Dir = fn.New(fn.Must(os.Getwd())) + } + + fi, err := os.Stat(*task.Dir) + if err != nil { + return nil, newErr("invalid directory", err) + } + + if !fi.IsDir() { + return nil, newErr("invalid working directory", fmt.Errorf("path (%s), is not a directory", *task.Dir)) + } + + dotenvPaths, err := resolveDotEnvFiles(filepath.Dir(rf.attrs.RunfilePath), task.DotEnv...) + if err != nil { + return nil, newErr("while resolving dotenv paths", err) + } + + dotenvVars, err := parseDotEnvFiles(dotenvPaths...) + if err != nil { + return nil, newErr("while parsing dotenv files", err) + } + + env := make([]string, 0, len(os.Environ())+len(dotenvVars)) + if !task.ignoreSystemEnv { + env = append(env, os.Environ()...) + } + for k, v := range dotenvVars { + env = append(env, fmt.Sprintf("%s=%v", k, v)) + } + + // INFO: keys from task.Env will override those coming from dotenv files, when duplicated + envVars, err := parseEnvVars(ctx, task.Env, EvaluationArgs{ + Shell: task.Shell, + Environ: env, + }) + if err != nil { + return nil, newErr("while evaluating task env vars", err) + } + + for k, v := range envVars { + env = append(env, fmt.Sprintf("%s=%v", k, v)) + } + + commands := make([]CommandJson, 0, len(task.Commands)) + for i := range task.Commands { + c2, err := parseCommand(rf, task.Commands[i]) + if err != nil { + return nil, newErr("while parsing command", err) + } + commands = append(commands, *c2) + } + + return &ParsedTask{ + Shell: task.Shell, + WorkingDir: *task.Dir, + Environ: env, + Commands: commands, + }, nil +} + +// returns absolute paths to dotenv files +func resolveDotEnvFiles(pwd string, dotEnvFiles ...string) ([]string, error) { + paths := make([]string, 0, len(dotEnvFiles)) + + for _, v := range dotEnvFiles { + dotenvPath := v + if !filepath.IsAbs(v) { + dotenvPath = filepath.Join(pwd, v) + } + fi, err := os.Stat(dotenvPath) + if err != nil { + return nil, err + } + + if fi.IsDir() { + return nil, fmt.Errorf("dotenv file must be a file, but %s is a directory", v) + } + + paths = append(paths, dotenvPath) + } + + return paths, nil +} + +func parseCommand(rf *Runfile, command any) (*CommandJson, error) { + switch c := command.(type) { + case string: + { + return &CommandJson{ + Command: c, + }, nil + } + case map[string]any: + { + var cj CommandJson + b, err := json.Marshal(c) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(b, &cj); err != nil { + return nil, err + } + + if cj.Run == "" { + return nil, fmt.Errorf("key: 'run', must be specified when setting command in json format") + } + + if _, ok := rf.Tasks[cj.Run]; !ok { + return nil, fmt.Errorf("[run target]: %s, not found in Runfile (%s)", cj.Run, rf.attrs.RunfilePath) + } + + return &cj, nil + } + default: + { + return nil, fmt.Errorf("invalid command") + } + } +} diff --git a/pkg/runfile/task-parser_test.go b/pkg/runfile/task-parser_test.go new file mode 100644 index 0000000..e136236 --- /dev/null +++ b/pkg/runfile/task-parser_test.go @@ -0,0 +1,484 @@ +package runfile + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + fn "github.com/nxtcoder17/runfile/pkg/functions" +) + +func TestParseTask(t *testing.T) { + type args struct { + ctx context.Context + rf *Runfile + taskName string + } + + areEqual := func(t *testing.T, got, want *ParsedTask) bool { + if strings.Join(got.Shell, ",") != strings.Join(want.Shell, ",") { + t.Logf("shell not equal") + return false + } + + slices.Sort(got.Environ) + slices.Sort(want.Environ) + + if strings.Join(got.Environ, ",") != strings.Join(want.Environ, ",") { + t.Logf("environ not equal") + return false + } + + if got.WorkingDir != want.WorkingDir { + t.Logf("working dir not equal") + return false + } + + if fmt.Sprintf("%#v", got.Commands) != fmt.Sprintf("%#v", want.Commands) { + t.Logf("commands not equal:\n got:\t%#v\nwant:\t%#v", got.Commands, want.Commands) + return false + } + + return true + } + + // for dotenv test + dotenvTestFile, err := os.CreateTemp(os.TempDir(), ".env") + if err != nil { + t.Error(err) + return + } + fmt.Fprintf(dotenvTestFile, "hello=world\n") + dotenvTestFile.Close() + + tests := []struct { + name string + args args + want *ParsedTask + wantErr bool + }{ + { + name: "[shell] if not specified, defaults to [sh, -c]", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: nil, + ignoreSystemEnv: true, + Dir: fn.New("."), + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: ".", + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[shell] if specified, must be acknowledged", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: []string{"python", "-c"}, + ignoreSystemEnv: true, + Dir: fn.New("."), + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"python", "-c"}, + WorkingDir: ".", + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[env] key: value", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: []string{"sh", "-c"}, + ignoreSystemEnv: true, + Env: map[string]any{ + "hello": "hi", + "k1": 1, + }, + Dir: fn.New("."), + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + Environ: []string{ + "hello=hi", + "k1=1", + }, + WorkingDir: ".", + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[env] key: JSON object format", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Env: map[string]any{ + "hello": map[string]any{ + "sh": "echo hi", + }, + }, + Dir: fn.New("."), + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + Environ: []string{ + "hello=hi", + }, + WorkingDir: ".", + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[unhappy/env] JSON object format [must throw err, when] sh key does not exist in value", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Env: map[string]any{ + "k1": map[string]any{ + "asdfasf": "asdfad", + }, + }, + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[unhappy/env] JSON object format [must throw err, when] sh (key)'s value is not a string", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Env: map[string]any{ + "k1": map[string]any{ + "sh": []string{"asdfsadf"}, + }, + }, + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[dotenv] loads environment from given file", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + DotEnv: []string{ + dotenvTestFile.Name(), + }, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Environ: []string{ + "hello=world", // from dotenv + }, + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[unhappy/dotenv] throws err, when file does not exist", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + DotEnv: []string{ + "/tmp/env-aasfksadjfkl", + }, + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[unhappy/dotenv] throws err, when filepath exists [but] is not a file (might be a directory or something else)", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + DotEnv: []string{ + "/tmp", + }, + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[working_dir] if not specified, should be current working directory", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[working_dir] when specified, must be acknowledged", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Dir: fn.New("/tmp"), + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: "/tmp", + Commands: []CommandJson{}, + }, + wantErr: false, + }, + { + name: "[unhappy/working_dir] must throw err, when directory does not exist", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Dir: fn.New("/tmp/xsdfjasdfkjdskfjasl"), + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[unhappy/working_dir] must throw err, when directory specified is not a directory (might be something else, or a file)", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Dir: fn.New(filepath.Join(fn.Must(os.Getwd()), "task.go")), + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[commands] string commands: single line", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Commands: []any{ + "echo hello", + }, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{ + {Command: "echo hello"}, + }, + }, + wantErr: false, + }, + + { + name: "[commands] string commands: multiline", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Commands: []any{ + ` +echo "hello" +echo "hi" +`, + }, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{ + { + Command: ` +echo "hello" +echo "hi" +`, + }, + }, + }, + wantErr: false, + }, + + { + name: "[commands] JSON commands", + args: args{ + ctx: nil, + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Commands: []any{ + "echo i will call hello, now", + map[string]any{ + "run": "hello", + }, + }, + }, + "hello": { + ignoreSystemEnv: true, + Commands: []any{ + "echo hello everyone", + }, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{ + {Command: "echo i will call hello, now"}, + {Run: "hello"}, + }, + }, + wantErr: false, + }, + { + name: "[unhappy/commands] JSON commands [must throw err, when] run target does not exist", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Commands: []any{ + "echo i will call hello, now", + map[string]any{ + "run": "hello", + }, + }, + }, + }, + }, + taskName: "test", + }, + wantErr: true, + }, + { + name: "[unhappy/runfile] target task does not exist", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{}, + }, + taskName: "test", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseTask(context.TODO(), tt.args.rf, tt.args.taskName) + if (err != nil) != tt.wantErr { + t.Errorf("ParseTask() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // if !reflect.DeepEqual(got, tt.want) { + if !tt.wantErr { + if !areEqual(t, got, tt.want) { + t.Errorf("ParseTask():> \n\tgot:\t%v,\n\twant:\t%v", got, tt.want) + } + } + }) + } +} diff --git a/pkg/runfile/task.go b/pkg/runfile/task.go index 044485c..2072a85 100644 --- a/pkg/runfile/task.go +++ b/pkg/runfile/task.go @@ -13,8 +13,14 @@ type Task struct { Env map[string]any `json:"env"` + // this field is for testing purposes only + ignoreSystemEnv bool `json:"-"` + // List of commands to be executed in given shell (default: sh) - Commands []string `json:"cmd"` + // can take multiple forms + // - simple string + // - a json object with key `run`, signifying other tasks to run + Commands []any `json:"cmd"` // Shell in which above commands will be executed // Default: ["sh", "-c"] @@ -26,6 +32,11 @@ type Task struct { Shell []string `json:"shell"` } +type CommandJson struct { + Command string + Run string `json:"run"` +} + type ErrTaskNotFound struct { TaskName string RunfilePath string diff --git a/pkg/runfile/type.go b/pkg/runfile/type.go index 0cb4f6c..79158f7 100644 --- a/pkg/runfile/type.go +++ b/pkg/runfile/type.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strings" ) /* @@ -53,9 +54,9 @@ func parseEnvVars(ctx context.Context, ev EnvVar, args EvaluationArgs) (map[stri if err := cmd.Run(); err != nil { return nil, err } - env[k] = value.String() + env[k] = strings.TrimSpace(value.String()) default: - panic(fmt.Sprintf("env %s is not a string (%T)", k, v)) + env[k] = fmt.Sprintf("%v", v) } } From 5955dd2818096287203c2053086d37b3b355b0a3 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Fri, 27 Sep 2024 21:31:24 +0530 Subject: [PATCH 4/6] ci: release workflows --- .github/workflows/release.yml | 136 ++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..369bd34 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,136 @@ +name: Release + +on: + push: + tags: + - 'v*' + + branches: + - master + + paths: + - "cmd/run/**" + - "pkg/**" + - go.* # go.mod, and go.sum + - flake.* + +permissions: + contents: write + packages: write + +jobs: + build-binary: + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, macos-14, macos-13] + arch: [amd64, arm64] + include: + - os: ubuntu-latest + platform: linux + + - os: macos-13 + platform: darwin + + - os: macos-14 + platform: darwin + exclude: + - os: macos-14 + arch: amd64 + - os: macos-13 + arch: arm64 + + name: Building run-${{ matrix.platform }}-${{ matrix.arch }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: nxtcoder17/actions/setup-cache-go@v1 + with: + cache_key: "run" + working_directory: . + + # it will set 2 env variables + # IMAGE_TAG - image tag + # OVERRIDE_PUSHED_IMAGE - if true, it will not use pushed image tag + - uses: nxtcoder17/actions/generate-image-tag@v1 + + - uses: nxtcoder17/actions/setup-nix-cachix@v1 + with: + flake_lock: "./flake.lock" + nix_develop_arguments: ".#default" + cachix_cache_name: ${{ secrets.CACHIX_CACHE_NAME }} + cachix_auth_token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Build Binary + shell: bash + env: + CGO_ENABLED: 0 + run: |+ + binary=bin/run-${{ matrix.platform }}-${{ matrix.arch }} + go build -o $binary -ldflags="-s -w" -tags urfave_cli_no_docs cmd/run/main.go + + - name: compress binary with upx (on linux) + if: matrix.platform == 'linux' + shell: bash + run: |+ + upx $binary + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: run-${{ matrix.platform }}-${{ matrix.arch }} + path: bin/* + + release: + needs: build-binary + runs-on: ubuntu-latest + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/binaries + pattern: "run-*" + + - name: flattening all the executable artifacts + shell: bash + run: |+ + ls -R ${{ github.workspace }}/binaries + mkdir -p ${{ github.workspace }}/upload/binaries + shopt -s globstar + file ./** | grep 'executable,' | awk -F: '{print $1}' | xargs -I {} cp {} ${{ github.workspace }}/upload/binaries + shopt -u globstar + + - uses: nxtcoder17/actions/generate-image-tag@v1 + + - name: running for master branch + if: startsWith(github.ref, 'refs/heads/master') + run: |+ + echo "IMAGE_TAG=nightly" | tee -a $GITHUB_ENV | tee -a $GITHUB_OUTPUT + + - name: ensure github release exists + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: |+ + set -e + gh release list -R ${{ github.repository }} | grep -i $IMAGE_TAG + exit_code=$1 + if [ $exit_code -ne 0 ]; then + gh release create $IMAGE_TAG -R ${{ github.respository }} --generate-notes --prerelease + fi + + - name: upload to github release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: |+ + gh release upload $IMAGE_TAG -R ${{github.repository}} ${{github.workspace}}/upload/binaries/* + + - name: mark release as latest + if: startsWith(github.ref, 'refs/tags/') + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: |+ + gh release edit $IMAGE_TAG -R ${{ github.repository }} --latest From d4fad9a1e28a298cae5bebfeab3927760fb6a37d Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Fri, 27 Sep 2024 21:43:11 +0530 Subject: [PATCH 5/6] ci: acitons workflow_dispatch, and tracking paths --- .github/workflows/release.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 369bd34..6444642 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,8 @@ name: Release on: + workflow_dispatch: + push: tags: - 'v*' @@ -10,6 +12,7 @@ on: paths: - "cmd/run/**" + - ".github/**" - "pkg/**" - go.* # go.mod, and go.sum - flake.* @@ -47,12 +50,9 @@ jobs: - uses: nxtcoder17/actions/setup-cache-go@v1 with: - cache_key: "run" + cache_key: "run-${{ matrix.platform }}-${{ matrix.arch }}" working_directory: . - # it will set 2 env variables - # IMAGE_TAG - image tag - # OVERRIDE_PUSHED_IMAGE - if true, it will not use pushed image tag - uses: nxtcoder17/actions/generate-image-tag@v1 - uses: nxtcoder17/actions/setup-nix-cachix@v1 @@ -70,11 +70,9 @@ jobs: binary=bin/run-${{ matrix.platform }}-${{ matrix.arch }} go build -o $binary -ldflags="-s -w" -tags urfave_cli_no_docs cmd/run/main.go - - name: compress binary with upx (on linux) - if: matrix.platform == 'linux' - shell: bash - run: |+ - upx $binary + if [ "${{ matrix.platform }}" = "linux" ]; then + upx $binary + fi - name: Upload Artifact uses: actions/upload-artifact@v4 @@ -113,11 +111,11 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: |+ - set -e + set +e gh release list -R ${{ github.repository }} | grep -i $IMAGE_TAG - exit_code=$1 + exit_code=$? if [ $exit_code -ne 0 ]; then - gh release create $IMAGE_TAG -R ${{ github.respository }} --generate-notes --prerelease + gh release create $IMAGE_TAG -R ${{ github.repository }} --generate-notes --prerelease fi - name: upload to github release From 5bc061779284e45046e987ca935c4659f261a7ee Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Sat, 28 Sep 2024 19:42:25 +0530 Subject: [PATCH 6/6] feat: adds requires conditions for task json environments are full featured now with `default`, `required`, and templated values --- docs/requirements-for-a-run-target.md | 27 ++++ examples/Runfile | 2 + go.mod | 1 + go.sum | 2 + pkg/runfile/errors/errors.go | 126 ++++++++++++++++++ pkg/runfile/run.go | 6 +- pkg/runfile/task-parser.go | 88 ++++++++----- pkg/runfile/task-parser_test.go | 176 +++++++++++++++++++++++++- pkg/runfile/task.go | 50 ++++---- pkg/runfile/type.go | 145 +++++++++++++++++---- 10 files changed, 537 insertions(+), 86 deletions(-) create mode 100644 docs/requirements-for-a-run-target.md create mode 100644 pkg/runfile/errors/errors.go diff --git a/docs/requirements-for-a-run-target.md b/docs/requirements-for-a-run-target.md new file mode 100644 index 0000000..d738800 --- /dev/null +++ b/docs/requirements-for-a-run-target.md @@ -0,0 +1,27 @@ +--- +Github Issue: https://github.com/nxtcoder17/Runfile/issues/8 +--- + +### Expectations from this implementation ? + +I would like to be able to do stuffs like: +- whether these environment variables have been defined +- whether this filepath exists or not +- whether `this command` or script runs sucessfully + +And, when answers to these questsions are `true`, then only run the target, otherwise throw the errors + +### Problems with Taskfile.dev implementation + +They have 2 ways to tackle this, with + +- `requires`: but, it works with vars only, ~no environment variables~ + +- `preconditions`: test conditions must be a valid linux `test` command, which assumes everyone knows how to read bash's `test` or `if` statements + + +### My Approach + +1. Support for `test` commands, must be there, for advanced users + +2. But, for simpler use cases, there should be alternate ways to do it, something that whole team just understands. diff --git a/examples/Runfile b/examples/Runfile index 3e04672..f2cdcc4 100644 --- a/examples/Runfile +++ b/examples/Runfile @@ -8,6 +8,8 @@ tasks: k2: 'f"\( asfsadfssdfas asfd $Asdfasdfa' k3: sh: echo -n "hello" + k4: + required: true dotenv: - ../.secrets/env cmd: diff --git a/go.mod b/go.mod index b1527b8..2deb5de 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/nxtcoder17/runfile go 1.22.7 require ( + github.com/go-task/slim-sprig/v3 v3.0.0 github.com/joho/godotenv v1.5.1 github.com/nxtcoder17/fwatcher v1.0.1 github.com/urfave/cli/v3 v3.0.0-alpha9 diff --git a/go.sum b/go.sum index dd09597..7fb069a 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= diff --git a/pkg/runfile/errors/errors.go b/pkg/runfile/errors/errors.go new file mode 100644 index 0000000..d7da7c4 --- /dev/null +++ b/pkg/runfile/errors/errors.go @@ -0,0 +1,126 @@ +package errors + +import ( + "encoding/json" + "fmt" +) + +type Context struct { + Task string + Runfile string + + message *string + err error +} + +func (c Context) WithMessage(msg string) Context { + c.message = &msg + return c +} + +func (c Context) WithErr(err error) Context { + c.err = err + return c +} + +func (c Context) ToString() string { + m := map[string]string{ + "task": c.Task, + "runfile": c.Runfile, + } + if c.message != nil { + m["message"] = *c.message + } + + if c.err != nil { + m["err"] = c.err.Error() + } + + b, err := json.Marshal(m) + if err != nil { + panic(err) + } + + return string(b) +} + +func (e Context) Error() string { + return e.ToString() +} + +type ( + ErrTaskInvalid struct{ Context } +) + +type ErrTaskFailedRequirements struct { + Context + Requirement string +} + +func (e ErrTaskFailedRequirements) Error() string { + if e.message == nil { + e.Context = e.Context.WithMessage(fmt.Sprintf("failed (requirement: %q)", e.Requirement)) + } + return e.Context.Error() +} + +type TaskNotFound struct { + Context +} + +func (e TaskNotFound) Error() string { + // return fmt.Sprintf("[task] %s, not found in [Runfile] at %s", e.TaskName, e.RunfilePath) + if e.message == nil { + e.Context = e.Context.WithMessage("Not Found") + } + return e.Context.Error() +} + +type ErrTaskGeneric struct { + Context +} + +type InvalidWorkingDirectory struct { + Context +} + +func (e InvalidWorkingDirectory) Error() string { + // return fmt.Sprintf("[task] %s, not found in [Runfile] at %s", e.TaskName, e.RunfilePath) + if e.message == nil { + e.Context = e.Context.WithMessage("Invalid Working Directory") + } + return e.Context.Error() +} + +type InvalidDotEnv struct { + Context +} + +func (e InvalidDotEnv) Error() string { + if e.message == nil { + e.Context = e.Context.WithMessage("invalid dotenv") + } + return e.Context.Error() +} + +type InvalidEnvVar struct { + Context +} + +func (e InvalidEnvVar) Error() string { + if e.message == nil { + e.Context = e.Context.WithMessage("invalid dotenv") + } + return e.Context.Error() +} + +type IncorrectCommand struct { + Context +} + +func (e IncorrectCommand) Error() string { + if e.message == nil { + e.Context = e.Context.WithMessage("incorrect command") + } + return e.Context.Error() +} diff --git a/pkg/runfile/run.go b/pkg/runfile/run.go index 9e40f5a..ae96ded 100644 --- a/pkg/runfile/run.go +++ b/pkg/runfile/run.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" + "github.com/nxtcoder17/runfile/pkg/runfile/errors" "golang.org/x/sync/errgroup" ) @@ -85,7 +86,10 @@ type RunArgs struct { func (rf *Runfile) Run(ctx context.Context, args RunArgs) error { for _, v := range args.Tasks { if _, ok := rf.Tasks[v]; !ok { - return ErrTaskNotFound{TaskName: v, RunfilePath: rf.attrs.RunfilePath} + return errors.TaskNotFound{Context: errors.Context{ + Task: v, + Runfile: rf.attrs.RunfilePath, + }} } } diff --git a/pkg/runfile/task-parser.go b/pkg/runfile/task-parser.go index e7772bd..e77901f 100644 --- a/pkg/runfile/task-parser.go +++ b/pkg/runfile/task-parser.go @@ -1,13 +1,17 @@ package runfile import ( + "bytes" "context" "encoding/json" "fmt" "os" "path/filepath" + "text/template" + sprig "github.com/go-task/slim-sprig/v3" fn "github.com/nxtcoder17/runfile/pkg/functions" + "github.com/nxtcoder17/runfile/pkg/runfile/errors" ) type ParsedTask struct { @@ -17,35 +21,53 @@ type ParsedTask struct { Commands []CommandJson `json:"commands"` } -type ErrTask struct { - ErrTaskNotFound - Message string - Err error -} - -func (e ErrTask) Error() string { - return fmt.Sprintf("[task] %s (Runfile: %s), says %s, got %v", e.TaskName, e.RunfilePath, e.Message, e.Err) -} - -func NewErrTask(runfilePath, taskName, message string, err error) ErrTask { - return ErrTask{ - ErrTaskNotFound: ErrTaskNotFound{ - RunfilePath: runfilePath, - TaskName: taskName, - }, - Message: message, - Err: err, - } -} - func ParseTask(ctx context.Context, rf *Runfile, taskName string) (*ParsedTask, error) { - newErr := func(message string, err error) ErrTask { - return NewErrTask(rf.attrs.RunfilePath, taskName, message, err) - } + errctx := errors.Context{Task: taskName, Runfile: rf.attrs.RunfilePath} task, ok := rf.Tasks[taskName] if !ok { - return nil, ErrTaskNotFound{TaskName: taskName, RunfilePath: rf.attrs.RunfilePath} + return nil, errors.TaskNotFound{Context: errctx} + } + + for _, requirement := range task.Requires { + if requirement == nil { + continue + } + + if requirement.Sh != nil { + cmd := createCommand(ctx, cmdArgs{ + shell: []string{"sh", "-c"}, + env: os.Environ(), + workingDir: filepath.Dir(rf.attrs.RunfilePath), + cmd: *requirement.Sh, + stdout: fn.Must(os.OpenFile(os.DevNull, os.O_WRONLY, 0o755)), + stderr: fn.Must(os.OpenFile(os.DevNull, os.O_WRONLY, 0o755)), + }) + if err := cmd.Run(); err != nil { + return nil, errors.ErrTaskFailedRequirements{Context: errctx.WithErr(err), Requirement: *requirement.Sh} + } + continue + } + + if requirement.GoTmpl != nil { + t := template.New("requirement") + t = t.Funcs(sprig.FuncMap()) + templateExpr := fmt.Sprintf(`{{ %s }}`, *requirement.GoTmpl) + t, err := t.Parse(templateExpr) + if err != nil { + return nil, err + } + b := new(bytes.Buffer) + if err := t.ExecuteTemplate(b, "requirement", map[string]string{}); err != nil { + return nil, err + } + + if b.String() != "true" { + return nil, errors.ErrTaskFailedRequirements{Context: errctx.WithErr(fmt.Errorf("`%s` evaluated to `%s` (wanted: `true`)", templateExpr, b.String())), Requirement: *requirement.GoTmpl} + } + + continue + } } if task.Shell == nil { @@ -58,21 +80,21 @@ func ParseTask(ctx context.Context, rf *Runfile, taskName string) (*ParsedTask, fi, err := os.Stat(*task.Dir) if err != nil { - return nil, newErr("invalid directory", err) + return nil, errors.InvalidWorkingDirectory{Context: errctx.WithErr(err)} } if !fi.IsDir() { - return nil, newErr("invalid working directory", fmt.Errorf("path (%s), is not a directory", *task.Dir)) + return nil, errors.InvalidWorkingDirectory{Context: errctx.WithErr(fmt.Errorf("path (%s), is not a directory", *task.Dir))} } dotenvPaths, err := resolveDotEnvFiles(filepath.Dir(rf.attrs.RunfilePath), task.DotEnv...) if err != nil { - return nil, newErr("while resolving dotenv paths", err) + return nil, errors.InvalidDotEnv{Context: errctx.WithErr(err).WithMessage("failed to resolve dotenv paths")} } dotenvVars, err := parseDotEnvFiles(dotenvPaths...) if err != nil { - return nil, newErr("while parsing dotenv files", err) + return nil, errors.InvalidDotEnv{Context: errctx.WithErr(err).WithMessage("failed to parse dotenv files")} } env := make([]string, 0, len(os.Environ())+len(dotenvVars)) @@ -85,11 +107,11 @@ func ParseTask(ctx context.Context, rf *Runfile, taskName string) (*ParsedTask, // INFO: keys from task.Env will override those coming from dotenv files, when duplicated envVars, err := parseEnvVars(ctx, task.Env, EvaluationArgs{ - Shell: task.Shell, - Environ: env, + Shell: task.Shell, + Env: dotenvVars, }) if err != nil { - return nil, newErr("while evaluating task env vars", err) + return nil, errors.InvalidEnvVar{Context: errctx.WithErr(err).WithMessage("failed to parse/evaluate env vars")} } for k, v := range envVars { @@ -100,7 +122,7 @@ func ParseTask(ctx context.Context, rf *Runfile, taskName string) (*ParsedTask, for i := range task.Commands { c2, err := parseCommand(rf, task.Commands[i]) if err != nil { - return nil, newErr("while parsing command", err) + return nil, errors.IncorrectCommand{Context: errctx.WithErr(err).WithMessage(fmt.Sprintf("failed to parse command: %+v", task.Commands[i]))} } commands = append(commands, *c2) } diff --git a/pkg/runfile/task-parser_test.go b/pkg/runfile/task-parser_test.go index e136236..60e497f 100644 --- a/pkg/runfile/task-parser_test.go +++ b/pkg/runfile/task-parser_test.go @@ -20,6 +20,10 @@ func TestParseTask(t *testing.T) { } areEqual := func(t *testing.T, got, want *ParsedTask) bool { + if want == nil { + return false + } + if strings.Join(got.Shell, ",") != strings.Join(want.Shell, ",") { t.Logf("shell not equal") return false @@ -55,12 +59,177 @@ func TestParseTask(t *testing.T) { fmt.Fprintf(dotenvTestFile, "hello=world\n") dotenvTestFile.Close() - tests := []struct { + type test struct { name string args args want *ParsedTask wantErr bool - }{ + } + + testRequires := []test{ + { + name: "[requires] condition specified, but it neither has 'sh' or 'gotmpl' key", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: nil, + ignoreSystemEnv: true, + Requires: []*Requires{ + {}, + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{}, + }, + wantErr: false, + }, + + { + name: "[requires] condition specified, with gotmpl key", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: nil, + ignoreSystemEnv: true, + Requires: []*Requires{ + { + GoTmpl: fn.New(`eq 5 5`), + }, + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{}, + }, + wantErr: false, + }, + + { + name: "[requires] condition specified, with sh key", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: nil, + ignoreSystemEnv: true, + Requires: []*Requires{ + { + Sh: fn.New(`echo hello`), + }, + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{}, + }, + wantErr: false, + }, + + { + name: "[unhappy/requires] condition specified, with sh key", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + Shell: nil, + ignoreSystemEnv: true, + Requires: []*Requires{ + { + Sh: fn.New(`echo hello && exit 1`), + }, + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Commands: []CommandJson{}, + }, + wantErr: true, + }, + } + + testEnviroments := []test{ + { + name: "[unhappy/env] required env, not provided", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Env: EnvVar{ + "hello": map[string]any{ + "required": true, + }, + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: nil, + wantErr: true, + }, + { + name: "[env] required env, provided", + args: args{ + rf: &Runfile{ + Tasks: map[string]Task{ + "test": { + ignoreSystemEnv: true, + Env: EnvVar{ + "hello": map[string]any{ + "required": true, + }, + }, + DotEnv: []string{ + dotenvTestFile.Name(), + }, + Commands: nil, + }, + }, + }, + taskName: "test", + }, + want: &ParsedTask{ + Shell: []string{"sh", "-c"}, + WorkingDir: fn.Must(os.Getwd()), + Environ: []string{ + "hello=world", + }, + Commands: []CommandJson{}, + }, + wantErr: false, + }, + } + + tests := []test{ { name: "[shell] if not specified, defaults to [sh, -c]", args: args{ @@ -465,6 +634,9 @@ echo "hi" }, } + tests = append(tests, testRequires...) + tests = append(tests, testEnviroments...) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := ParseTask(context.TODO(), tt.args.rf, tt.args.taskName) diff --git a/pkg/runfile/task.go b/pkg/runfile/task.go index 2072a85..62b2e91 100644 --- a/pkg/runfile/task.go +++ b/pkg/runfile/task.go @@ -1,47 +1,53 @@ package runfile -import ( - "fmt" -) +// Only one of the fields must be set +type Requires struct { + Sh *string `json:"sh,omitempty"` + GoTmpl *string `json:"gotmpl,omitempty"` +} + +/* +EnvVar Values could take multiple forms: +- my_key: "value" +or + - my_key: + sh: "echo hello hi" + +Object values with `sh` key, such that the output of this command will be the value of the top-level key +*/ +type EnvVar map[string]any type Task struct { + // Shell in which above commands will be executed + // Default: ["sh", "-c"] + /* Common Usecases could be: + - ["bash", "-c"] + - ["python", "-c"] + - ["node", "-e"] + */ + Shell []string `json:"shell"` + // load env vars from [.env](https://www.google.com/search?q=sample+dotenv+files&udm=2) files DotEnv []string `json:"dotenv"` // working directory for the task Dir *string `json:"dir,omitempty"` - Env map[string]any `json:"env"` + Env EnvVar `json:"env,omitempty"` // this field is for testing purposes only ignoreSystemEnv bool `json:"-"` + Requires []*Requires `json:"requires,omitempty"` + // List of commands to be executed in given shell (default: sh) // can take multiple forms // - simple string // - a json object with key `run`, signifying other tasks to run Commands []any `json:"cmd"` - - // Shell in which above commands will be executed - // Default: ["sh", "-c"] - /* Common Usecases could be: - - ["bash", "-c"] - - ["python", "-c"] - - ["node", "-e"] - */ - Shell []string `json:"shell"` } type CommandJson struct { Command string Run string `json:"run"` } - -type ErrTaskNotFound struct { - TaskName string - RunfilePath string -} - -func (e ErrTaskNotFound) Error() string { - return fmt.Sprintf("[task] %s, not found in [Runfile] at %s", e.TaskName, e.RunfilePath) -} diff --git a/pkg/runfile/type.go b/pkg/runfile/type.go index 79158f7..4333a64 100644 --- a/pkg/runfile/type.go +++ b/pkg/runfile/type.go @@ -3,27 +3,79 @@ package runfile import ( "bytes" "context" + "encoding/json" "fmt" + "os" "strings" + "text/template" + + sprig "github.com/go-task/slim-sprig/v3" + fn "github.com/nxtcoder17/runfile/pkg/functions" ) -/* -EnvVar Values could take multiple forms: +type EvaluationArgs struct { + Shell []string + Env map[string]string +} -- my_key: "value" +func ToEnviron(m map[string]string) []string { + results := os.Environ() + for k, v := range m { + results = append(results, fmt.Sprintf("%s=%v", k, v)) + } + return results +} -or +type EnvKV struct { + Key string - - my_key: - "sh": "echo hello hi" + Value *string `json:"value"` + Sh *string `json:"sh"` + GoTmpl *string `json:"gotmpl"` +} -Object values with `sh` key, such that the output of this command will be the value of the top-level key -*/ -type EnvVar map[string]any +func (ejv EnvKV) Parse(ctx context.Context, args EvaluationArgs) (*string, error) { + switch { + case ejv.Value != nil: + { + return ejv.Value, nil + } + case ejv.Sh != nil: + { + value := new(bytes.Buffer) -type EvaluationArgs struct { - Shell []string - Environ []string + cmd := createCommand(ctx, cmdArgs{ + shell: args.Shell, + env: ToEnviron(args.Env), + cmd: *ejv.Sh, + stdout: value, + }) + if err := cmd.Run(); err != nil { + return nil, err + } + + return fn.New(strings.TrimSpace(value.String())), nil + } + case ejv.GoTmpl != nil: + { + t := template.New(ejv.Key).Funcs(sprig.FuncMap()) + t, err := t.Parse(fmt.Sprintf(`{{ %s }}`, *ejv.GoTmpl)) + if err != nil { + return nil, err + } + + value := new(bytes.Buffer) + if err := t.ExecuteTemplate(value, ejv.Key, map[string]string{}); err != nil { + return nil, err + } + + return fn.New(strings.TrimSpace(value.String())), nil + } + default: + { + return nil, fmt.Errorf("failed to parse, unknown format, one of [value, sh, gotmpl] must be set") + } + } } func parseEnvVars(ctx context.Context, ev EnvVar, args EvaluationArgs) (map[string]string, error) { @@ -33,28 +85,65 @@ func parseEnvVars(ctx context.Context, ev EnvVar, args EvaluationArgs) (map[stri case string: env[k] = v case map[string]any: - shcmd, ok := v["sh"] - if !ok { - return nil, fmt.Errorf("sh key is missing") + b, err := json.Marshal(v) + if err != nil { + return nil, err } - s, ok := shcmd.(string) - if !ok { - return nil, fmt.Errorf("shell cmd is not a string") + var envAsJson struct { + *EnvKV + Required bool + Default *EnvKV } - value := new(bytes.Buffer) - - cmd := createCommand(ctx, cmdArgs{ - shell: args.Shell, - env: args.Environ, - cmd: s, - stdout: value, - }) - if err := cmd.Run(); err != nil { + if err := json.Unmarshal(b, &envAsJson); err != nil { return nil, err } - env[k] = strings.TrimSpace(value.String()) + + switch { + case envAsJson.Required: + { + isDefined := false + if _, ok := os.LookupEnv(k); ok { + isDefined = true + } + + if !isDefined { + if _, ok := args.Env[k]; ok { + isDefined = true + } + } + + if !isDefined { + return nil, fmt.Errorf("env: %q, not defined", k) + } + } + + case envAsJson.EnvKV != nil: + { + envAsJson.Key = k + s, err := envAsJson.EnvKV.Parse(ctx, args) + if err != nil { + return nil, err + } + env[k] = *s + } + + case envAsJson.Default != nil: + { + envAsJson.Default.Key = k + s, err := envAsJson.Default.Parse(ctx, args) + if err != nil { + return nil, err + } + env[k] = *s + } + default: + { + return nil, fmt.Errorf("either required, value, sh, gotmpl or default, must be defined") + } + } + default: env[k] = fmt.Sprintf("%v", v) }