Skip to content

Commit

Permalink
APL support
Browse files Browse the repository at this point in the history
  • Loading branch information
lukasmalkmus committed Aug 26, 2021
1 parent eb0ccd5 commit 0e029c8
Show file tree
Hide file tree
Showing 10 changed files with 332 additions and 76 deletions.
26 changes: 16 additions & 10 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ jobs:
needs: build
if: github.event.pull_request.head.repo.full_name == github.repository
strategy:
max-parallel: 2
fail-fast: false
matrix:
os:
- macos-latest
Expand All @@ -118,11 +120,13 @@ jobs:
goos: linux
- os: windows-latest
goos: windows
experimental: true
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
env:
AXIOM_TOKEN: ${{ secrets.TESTING_AZURE_1_STAGING_ACCESS_TOKEN }}
AXIOM_URL: ${{ secrets.TESTING_AZURE_1_STAGING_DEPLOYMENT_URL }}
DATASET_SUFFIX: ${{ github.run_id }}-${{ matrix.goos }}
AXIOM_DATASET: cli-test-${{ github.run_id }}-${{ matrix.goos }}
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
Expand All @@ -135,22 +139,24 @@ jobs:
chmod +x dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom
mv dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom /usr/local/bin/axiom
axiom version -I
axiom dataset create -n=cli-test-${{ env.DATASET_SUFFIX }} -d="CLI Integration test ${{ env.DATASET_SUFFIX }}"
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.json -f=testdata/logs.ndjson -f=testdata/logs.csv
axiom dataset info cli-test-${{ env.DATASET_SUFFIX }}
axiom dataset create -n=${{ env.AXIOM_DATASET }} -d="CLI Integration test"
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.json -f=testdata/logs.ndjson -f=testdata/logs.csv
axiom dataset info ${{ env.AXIOM_DATASET }}
axiom dataset list
axiom query "['${{ env.AXIOM_DATASET }}']"
- name: Test (Windows)
if: matrix.goos == 'windows'
run: |
chmod +x dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom.exe
mv dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom.exe C:/Windows/System32/axiom.exe
axiom version -I
axiom dataset create -n=cli-test-${{ env.DATASET_SUFFIX }} -d="CLI Integration test ${{ env.DATASET_SUFFIX }}"
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.json
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.ndjson
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.csv
axiom dataset info cli-test-${{ env.DATASET_SUFFIX }}
axiom dataset create -n=${{ env.AXIOM_DATASET }} -d="CLI Integration test"
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.json
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.ndjson
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.csv
axiom dataset info ${{ env.AXIOM_DATASET }}
axiom dataset list
axiom query "['${{ env.AXIOM_DATASET }}']"
- name: Cleanup
if: always()
run: axiom dataset delete -f cli-test-${{ env.DATASET_SUFFIX }}
run: axiom dataset delete -f ${{ env.AXIOM_DATASET }}
32 changes: 19 additions & 13 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ jobs:
name: Binary integration
needs: build
strategy:
max-parallel: 2
fail-fast: false
matrix:
os:
- macos-latest
Expand All @@ -121,11 +123,13 @@ jobs:
goos: linux
- os: windows-latest
goos: windows
experimental: true
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
env:
AXIOM_TOKEN: ${{ secrets.TESTING_AZURE_1_STAGING_ACCESS_TOKEN }}
AXIOM_URL: ${{ secrets.TESTING_AZURE_1_STAGING_DEPLOYMENT_URL }}
DATASET_SUFFIX: ${{ github.run_id }}-${{ matrix.goos }}
AXIOM_DATASET: cli-test-${{ github.run_id }}-${{ matrix.goos }}
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
Expand All @@ -136,24 +140,26 @@ jobs:
if: matrix.goos == 'darwin' || matrix.goos == 'linux'
run: |
chmod +x dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom
echo "dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom" >> $GITHUB_PATH
mv dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom /usr/local/bin/axiom
axiom version -I
axiom dataset create -n=cli-test-${{ env.DATASET_SUFFIX }} -d="CLI Integration test ${{ env.DATASET_SUFFIX }}"
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.json -f=testdata/logs.ndjson -f=testdata/logs.csv
axiom dataset info cli-test-${{ env.DATASET_SUFFIX }}
axiom dataset create -n=${{ env.AXIOM_DATASET }} -d="CLI Integration test"
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.json -f=testdata/logs.ndjson -f=testdata/logs.csv
axiom dataset info ${{ env.AXIOM_DATASET }}
axiom dataset list
axiom query "['${{ env.AXIOM_DATASET }}']"
- name: Test (Windows)
if: matrix.goos == 'windows'
run: |
chmod +x dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom
echo "dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom.exe" >> $GITHUB_PATH
chmod +x dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom.exe
mv dist/${{ matrix.goos }}_${{ matrix.goos }}_amd64/axiom.exe C:/Windows/System32/axiom.exe
axiom version -I
axiom dataset create -n=cli-test-${{ env.DATASET_SUFFIX }} -d="CLI Integration test ${{ env.DATASET_SUFFIX }}"
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.json
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.ndjson
axiom ingest cli-test-${{ env.DATASET_SUFFIX }} -f=testdata/logs.csv
axiom dataset info cli-test-${{ env.DATASET_SUFFIX }}
axiom dataset create -n=${{ env.AXIOM_DATASET }} -d="CLI Integration test"
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.json
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.ndjson
axiom ingest ${{ env.AXIOM_DATASET }} -f=testdata/logs.csv
axiom dataset info ${{ env.AXIOM_DATASET }}
axiom dataset list
axiom query "['${{ env.AXIOM_DATASET }}']"
- name: Cleanup
if: always()
run: axiom dataset delete -f cli-test-${{ env.DATASET_SUFFIX }}
run: axiom dataset delete -f ${{ env.AXIOM_DATASET }}
10 changes: 6 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,11 @@ For full command reference, see the list below, or visit

**Core Commands**

| Commands | Description |
| ------------ | ---------------- |
| axiom ingest | Ingest data |
| axiom stream | Live stream data |
| Commands | Description |
| ------------ | -------------------- |
| axiom ingest | Ingest data |
| axiom query | Query data using APL |
| axiom stream | Live stream data |

**Management Commands**

Expand All @@ -123,6 +124,7 @@ For full command reference, see the list below, or visit
| axiom auth logout | Logout of an Axiom deployment |
| axiom auth select | Select an Axiom deployment |
| axiom auth status | View authentication status |
| axiom auth switch-org | Switch the organization |
| axiom auth update-token | Update the token of a deloyment |
| axiom config edit | Edit the configuration file |
| axiom config get | Get a configuration value |
Expand Down
9 changes: 7 additions & 2 deletions cmd/axiom/completion/completion_bash.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
)

func newBashCmd(f *cmdutil.Factory) *cobra.Command {
var completionNoDesc bool

cmd := &cobra.Command{
Use: "bash",
Short: "Generate shell completion script for bash",
Expand All @@ -22,14 +24,17 @@ func newBashCmd(f *cmdutil.Factory) *cobra.Command {
# To load completions for every new session, execute once:
# Linux:
$ axiom completion bash > /etc/bash_completion.d/axiom
# MacOS:
# macOS:
$ axiom completion bash > /usr/local/etc/bash_completion.d/axiom
`),

RunE: func(cmd *cobra.Command, _ []string) error {
return cmd.Root().GenBashCompletion(f.IO.Out())
return cmd.Root().GenBashCompletionV2(f.IO.Out(), !completionNoDesc)
},
}

cmd.Flags().BoolVar(&completionNoDesc, "no-descriptions", false, "Disable completion descriptions")

return cmd
}
4 changes: 2 additions & 2 deletions cmd/axiom/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ func main() {
cmdutil.InheritRootPersistenPreRun(rootCmd)

// Finally execute the root command.
if err := rootCmd.ExecuteContext(ctx); err != nil {
printError(f.IO.ErrOut(), err, rootCmd)
if cmd, err := rootCmd.ExecuteContextC(ctx); err != nil {
printError(f.IO.ErrOut(), err, cmd)
os.Exit(1)
} else if root.HasFailed() {
os.Exit(1)
Expand Down
209 changes: 209 additions & 0 deletions cmd/axiom/query/query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
package query

import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"

"github.com/AlecAivazis/survey/v2"
"github.com/MakeNowJust/heredoc"
"github.com/araddon/dateparse"
"github.com/axiomhq/axiom-go/axiom/apl"
"github.com/nwidger/jsoncolor"
"github.com/spf13/cobra"

"github.com/axiomhq/cli/internal/cmdutil"
"github.com/axiomhq/cli/pkg/iofmt"
)

type options struct {
*cmdutil.Factory

// Query to run. If not supplied as an argument, which is optional, the user
// will be asked for it.
Query string
// StartTime of the query.
StartTime string
// EndTime of the query.
EndTime string
// TimestampFormat the timestamp is formatted in.
TimestampFormat string
// Format to output data in. Defaults to tabular output.
Format string
// NoCache disables cache usage for the query.
NoCache bool
// Save the query on the server.
Save bool

startTime time.Time
endTime time.Time
}

// NewQueryCmd creates and returns the query command.
func NewQueryCmd(f *cmdutil.Factory) *cobra.Command {
opts := &options{
Factory: f,
}

cmd := &cobra.Command{
Use: "query [<apl-query>] [(-f|--format=)json|table] [--start-time <start-time>] [--end-time <end-time>] [--timestamp-format <timestamp-format>] [-c|--no-cache] [-s|--save]",
Short: "Query data using APL",
Long: heredoc.Doc(`
Query data from an Axiom dataset using APL, the Axiom Processing
Language.
The query range can be specified by specifying start and end time.
The timestamp format can be configured by specifying a pattern with
the reference date:
Mon Jan 2 15:04:05 -0700 MST 2006
Omitted elements in the pattern are treated as zero or one as
applicable. See the Go reference documentation for examples:
https://pkg.go.dev/time#pkg-constants
`),

DisableFlagsInUseLine: true,

Args: cmdutil.PopulateFromArgs(f, &opts.Query),

Example: heredoc.Doc(`
# Query the "nginx-logs" dataset for logs with a 304 status code:
$ axiom query "['nginx-logs'] | where response == 304"
# Query all logs of the "http" dataset and save the query in the
# history. The histories entry ID is returned with the result:
$ axiom query -s "['http']"
`),

Annotations: map[string]string{
"IsCore": "true",
},

PreRunE: cmdutil.ChainRunFuncs(
cmdutil.NeedsActiveDeployment(f),
cmdutil.NeedsPersonalAccessToken(f),
cmdutil.NeedsDatasets(f),
),

RunE: func(cmd *cobra.Command, args []string) error {
if err := complete(opts); err != nil {
return err
}
return run(cmd.Context(), opts)
},
}

cmd.Flags().StringVarP(&opts.Format, "format", "f", iofmt.Table.String(), "Format to output data in")
cmd.Flags().StringVar(&opts.StartTime, "start-time", "", "Start time of the query")
cmd.Flags().StringVar(&opts.EndTime, "end-time", "", "End time of the query")
cmd.Flags().StringVar(&opts.TimestampFormat, "timestamp-format", "", "Format used in the the timestamp field. Default uses a heuristic parser. Must be expressed using the reference time 'Mon Jan 2 15:04:05 -0700 MST 2006'")
cmd.Flags().BoolVarP(&opts.NoCache, "no-cache", "c", false, "Disable cache usage")
cmd.Flags().BoolVarP(&opts.Save, "save", "s", false, "Save query on the server side")

_ = cmd.RegisterFlagCompletionFunc("format", formatCompletion)
_ = cmd.RegisterFlagCompletionFunc("start-time", cmdutil.NoCompletion)
_ = cmd.RegisterFlagCompletionFunc("end-time", cmdutil.NoCompletion)
_ = cmd.RegisterFlagCompletionFunc("timestamp-format", cmdutil.NoCompletion)
_ = cmd.RegisterFlagCompletionFunc("no-cache", cmdutil.NoCompletion)
_ = cmd.RegisterFlagCompletionFunc("save", cmdutil.NoCompletion)

return cmd
}

func complete(opts *options) (err error) {
if ts := opts.StartTime; ts != "" {
if tsf := opts.TimestampFormat; tsf != "" {
opts.startTime, err = time.Parse(tsf, ts)
} else {
opts.startTime, err = dateparse.ParseAny(ts)
}
if err != nil {
return err
}
}

if ts := opts.EndTime; ts != "" {
if tsf := opts.TimestampFormat; tsf != "" {
opts.endTime, err = time.Parse(tsf, ts)
} else {
opts.endTime, err = dateparse.ParseAny(ts)
}
if err != nil {
return err
}
}

if opts.Query != "" {
return nil
}

return survey.AskOne(&survey.Input{
Message: "Which query to run?",
}, &opts.Query, opts.IO.SurveyIO())
}

func run(ctx context.Context, opts *options) error {
client, err := opts.Client()
if err != nil {
return err
}

cs := opts.IO.ColorScheme()

var enc interface {
Encode(interface{}) error
}
if opts.IO.ColorEnabled() {
enc = jsoncolor.NewEncoder(opts.IO.Out())
} else {
enc = json.NewEncoder(opts.IO.Out())
}

res, err := client.Datasets.APLQuery(ctx, opts.Query, apl.Options{
StartTime: opts.startTime,
EndTime: opts.endTime,
NoCache: opts.NoCache,
Save: opts.Save,
})
if err != nil {
return err
} else if res == nil || len(res.Matches) == 0 {
return errors.New("query returned no results")
}

if opts.IO.IsStdoutTTY() {
s := cs.Bold(opts.Query)
if res.SavedQueryID != "" {
s += fmt.Sprintf(" (saved as %s)", cs.Bold(res.SavedQueryID))
}
fmt.Fprintf(opts.IO.Out(), "Result of query %s:\n\n", s)
}

for _, entry := range res.Matches {
switch opts.Format {
case iofmt.JSON.String():
_ = enc.Encode(entry)
default:
fmt.Fprintf(opts.IO.Out(), "%s\t",
cs.Gray(entry.Time.Format(time.RFC1123)))
_ = enc.Encode(entry.Data)
}
fmt.Fprintln(opts.IO.Out())
}

return nil
}

func formatCompletion(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) {
res := make([]string, 0, len(iofmt.Formats()))
for _, validFormat := range iofmt.Formats() {
if strings.HasPrefix(validFormat.String(), toComplete) {
res = append(res, validFormat.String())
}
}
return res, cobra.ShellCompDirectiveNoFileComp
}
Loading

0 comments on commit 0e029c8

Please sign in to comment.