From 3ed82db00474d5318f579e4edfae1bab120516b6 Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Fri, 29 Nov 2024 23:49:42 +0530 Subject: [PATCH 1/6] feat: Add new stdlib functions --- docs/sources/reference/stdlib/encoding.md | 23 +++++++++++++++++++++++ syntax/internal/stdlib/stdlib.go | 18 +++++++++++++++--- syntax/vm/vm_stdlib_test.go | 2 ++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/sources/reference/stdlib/encoding.md b/docs/sources/reference/stdlib/encoding.md index 9bc8fcaed5..de7e78e21d 100644 --- a/docs/sources/reference/stdlib/encoding.md +++ b/docs/sources/reference/stdlib/encoding.md @@ -26,6 +26,29 @@ The `encoding.from_base64` function decodes a RFC4648-compliant Base64-encoded s tangerine ``` +## encoding.to_base64 + +The `encoding.to_base64` function encodes a string to RFC4648-compliant base64 string +into the original string. + +### Examples + +``` +> encoding.to_base64("foobar123!?$*&()'-=@~") +Zm9vYmFyMTIzIT8kKiYoKSctPUB+ +``` + +## encoding.to_URLbase64 + +The `encoding.to_URLbase64` function encodes a string to RFC4648-compliant url safe base64 string + +### Examples + +``` +> encoding.to_URLbase64("foobar123!?$*&()'-=@~") +Zm9vYmFyMTIzIT8kKiYoKSctPUB- +``` + ## encoding.from_json The `encoding.from_json` function decodes a string representing JSON into an {{< param "PRODUCT_NAME" >}} value. diff --git a/syntax/internal/stdlib/stdlib.go b/syntax/internal/stdlib/stdlib.go index 85dadd9c67..f5d4d9d56c 100644 --- a/syntax/internal/stdlib/stdlib.go +++ b/syntax/internal/stdlib/stdlib.go @@ -75,9 +75,11 @@ var file = map[string]interface{}{ } var encoding = map[string]interface{}{ - "from_json": jsonDecode, - "from_yaml": yamlDecode, - "from_base64": base64Decode, + "from_json": jsonDecode, + "from_yaml": yamlDecode, + "from_base64": base64Decode, + "to_base64": base64Encode, + "to_URLbase64": base64URLEncode, } var str = map[string]interface{}{ @@ -313,6 +315,16 @@ func base64Decode(in string) (interface{}, error) { return decoded, nil } +func base64URLEncode(in string) (interface{}, error) { + encoded := base64.URLEncoding.EncodeToString([]byte(in)) + return encoded, nil +} + +func base64Encode(in string) (interface{}, error) { + encoded := base64.StdEncoding.EncodeToString([]byte(in)) + return encoded, nil +} + func jsonPath(jsonString string, path string) (interface{}, error) { jsonPathExpr, err := jp.ParseString(path) if err != nil { diff --git a/syntax/vm/vm_stdlib_test.go b/syntax/vm/vm_stdlib_test.go index 36a74c29ab..a584d17eac 100644 --- a/syntax/vm/vm_stdlib_test.go +++ b/syntax/vm/vm_stdlib_test.go @@ -39,6 +39,8 @@ func TestVM_Stdlib(t *testing.T) { {"encoding.from_yaml nil field", "encoding.from_yaml(`foo: null`)", map[string]interface{}{"foo": nil}}, {"encoding.from_yaml nil array element", `encoding.from_yaml("[0, null]")`, []interface{}{0, nil}}, {"encoding.from_base64", `encoding.from_base64("Zm9vYmFyMTIzIT8kKiYoKSctPUB+")`, string(`foobar123!?$*&()'-=@~`)}, + {"encoding.to_base64", `encoding.to_base64("foobar123!?$*&()'-=@~")`, string(`Zm9vYmFyMTIzIT8kKiYoKSctPUB+`)}, + {"encoding.to_URLbase64", `encoding.to_URLbase64("foobar123!?$*&()'-=@~")`, string(`Zm9vYmFyMTIzIT8kKiYoKSctPUB-`)}, // Map tests { From 0c033b12d6a216093d262c2bc7c65be43165200e Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 3 Dec 2024 19:37:07 +0530 Subject: [PATCH 2/6] Added from_URLbase64 and review comments --- CHANGELOG.md | 1 + docs/sources/reference/stdlib/encoding.md | 32 ++++++++++++++++------- syntax/internal/stdlib/stdlib.go | 19 ++++++++++---- syntax/vm/vm_stdlib_test.go | 5 ++-- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a912ad5e57..3b36370362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ Main (unreleased) - Add perf_schema quantile columns to collector - For sharding targets during clustering, `loki.source.podlogs` now only takes into account some labels. (@ptodev) +- Add two new stdlib functions to_base64 and to_URLbase64 (@ravishankar15) ### Bugfixes - Fixed an issue in the `pyroscope.write` component to prevent TLS connection churn to Pyroscope when the `pyroscope.receive_http` clients don't request keepalive (@madaraszg-tulip) diff --git a/docs/sources/reference/stdlib/encoding.md b/docs/sources/reference/stdlib/encoding.md index de7e78e21d..7b2ef3710f 100644 --- a/docs/sources/reference/stdlib/encoding.md +++ b/docs/sources/reference/stdlib/encoding.md @@ -19,34 +19,46 @@ The `encoding.from_base64` function decodes a RFC4648-compliant Base64-encoded s `encoding.from_base64` fails if the provided string argument contains invalid Base64 data. -### Examples +### Example ```text > encoding.from_base64("dGFuZ2VyaW5l") tangerine ``` +## encoding.from_URLbase64 + +The `encoding.from_URLbase64` function decodes a RFC4648-compliant Base64 url safe encoded string into the original string. + +`encoding.from_URLbase64` fails if the provided string argument contains invalid Base64 data. + +### Example + +``` +> encoding.from_URLbase64("c3RyaW5nMTIzIT8kKiYoKSctPUB-") +string123!?$*&()'-=@~ +``` + ## encoding.to_base64 -The `encoding.to_base64` function encodes a string to RFC4648-compliant base64 string -into the original string. +The `encoding.to_base64` function encodes the original string into RFC4648-compliant Base64 encoded string -### Examples +### Example ``` -> encoding.to_base64("foobar123!?$*&()'-=@~") -Zm9vYmFyMTIzIT8kKiYoKSctPUB+ +> encoding.to_base64("string123!?$*&()'-=@~") +c3RyaW5nMTIzIT8kKiYoKSctPUB+ ``` ## encoding.to_URLbase64 -The `encoding.to_URLbase64` function encodes a string to RFC4648-compliant url safe base64 string +The `encoding.to_base64` function encodes the original string into RFC4648-compliant URL safe Base64 encoded string -### Examples +### Example ``` -> encoding.to_URLbase64("foobar123!?$*&()'-=@~") -Zm9vYmFyMTIzIT8kKiYoKSctPUB- +> encoding.to_URLbase64("string123!?$*&()'-=@~") +c3RyaW5nMTIzIT8kKiYoKSctPUB- ``` ## encoding.from_json diff --git a/syntax/internal/stdlib/stdlib.go b/syntax/internal/stdlib/stdlib.go index f5d4d9d56c..e81856ae30 100644 --- a/syntax/internal/stdlib/stdlib.go +++ b/syntax/internal/stdlib/stdlib.go @@ -75,11 +75,12 @@ var file = map[string]interface{}{ } var encoding = map[string]interface{}{ - "from_json": jsonDecode, - "from_yaml": yamlDecode, - "from_base64": base64Decode, - "to_base64": base64Encode, - "to_URLbase64": base64URLEncode, + "from_json": jsonDecode, + "from_yaml": yamlDecode, + "from_base64": base64Decode, + "from_URLbase64": base64URLDecode, + "to_base64": base64Encode, + "to_URLbase64": base64URLEncode, } var str = map[string]interface{}{ @@ -315,6 +316,14 @@ func base64Decode(in string) (interface{}, error) { return decoded, nil } +func base64URLDecode(in string) (interface{}, error) { + decoded, err := base64.URLEncoding.DecodeString(in) + if err != nil { + return nil, err + } + return decoded, nil +} + func base64URLEncode(in string) (interface{}, error) { encoded := base64.URLEncoding.EncodeToString([]byte(in)) return encoded, nil diff --git a/syntax/vm/vm_stdlib_test.go b/syntax/vm/vm_stdlib_test.go index a584d17eac..70dc2711c6 100644 --- a/syntax/vm/vm_stdlib_test.go +++ b/syntax/vm/vm_stdlib_test.go @@ -39,8 +39,9 @@ func TestVM_Stdlib(t *testing.T) { {"encoding.from_yaml nil field", "encoding.from_yaml(`foo: null`)", map[string]interface{}{"foo": nil}}, {"encoding.from_yaml nil array element", `encoding.from_yaml("[0, null]")`, []interface{}{0, nil}}, {"encoding.from_base64", `encoding.from_base64("Zm9vYmFyMTIzIT8kKiYoKSctPUB+")`, string(`foobar123!?$*&()'-=@~`)}, - {"encoding.to_base64", `encoding.to_base64("foobar123!?$*&()'-=@~")`, string(`Zm9vYmFyMTIzIT8kKiYoKSctPUB+`)}, - {"encoding.to_URLbase64", `encoding.to_URLbase64("foobar123!?$*&()'-=@~")`, string(`Zm9vYmFyMTIzIT8kKiYoKSctPUB-`)}, + {"encoding.from_URLbase64", `encoding.from_URLbase64("c3RyaW5nMTIzIT8kKiYoKSctPUB-")`, string(`string123!?$*&()'-=@~`)}, + {"encoding.to_base64", `encoding.to_base64("string123!?$*&()'-=@~")`, string(`c3RyaW5nMTIzIT8kKiYoKSctPUB+`)}, + {"encoding.to_URLbase64", `encoding.to_URLbase64("string123!?$*&()'-=@~")`, string(`c3RyaW5nMTIzIT8kKiYoKSctPUB-`)}, // Map tests { From 69f0817d66324c3a9e85144c7bc87f66f8ef04bc Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 3 Dec 2024 21:57:18 +0530 Subject: [PATCH 3/6] Update docs/sources/reference/stdlib/encoding.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/reference/stdlib/encoding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/stdlib/encoding.md b/docs/sources/reference/stdlib/encoding.md index 7b2ef3710f..e79a40b433 100644 --- a/docs/sources/reference/stdlib/encoding.md +++ b/docs/sources/reference/stdlib/encoding.md @@ -28,7 +28,7 @@ tangerine ## encoding.from_URLbase64 -The `encoding.from_URLbase64` function decodes a RFC4648-compliant Base64 url safe encoded string into the original string. +The `encoding.from_URLbase64` function decodes a RFC4648-compliant Base64 URL safe encoded string into the original string. `encoding.from_URLbase64` fails if the provided string argument contains invalid Base64 data. From 80667ad985203fbddd992261ba6cd0c2008e5c01 Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 3 Dec 2024 21:57:27 +0530 Subject: [PATCH 4/6] Update docs/sources/reference/stdlib/encoding.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/reference/stdlib/encoding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/stdlib/encoding.md b/docs/sources/reference/stdlib/encoding.md index e79a40b433..b5cb550631 100644 --- a/docs/sources/reference/stdlib/encoding.md +++ b/docs/sources/reference/stdlib/encoding.md @@ -41,7 +41,7 @@ string123!?$*&()'-=@~ ## encoding.to_base64 -The `encoding.to_base64` function encodes the original string into RFC4648-compliant Base64 encoded string +The `encoding.to_base64` function encodes the original string into RFC4648-compliant Base64 encoded string. ### Example From 3a042d73573d052e3623a154682518964b20a0eb Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 3 Dec 2024 21:57:35 +0530 Subject: [PATCH 5/6] Update docs/sources/reference/stdlib/encoding.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/reference/stdlib/encoding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/stdlib/encoding.md b/docs/sources/reference/stdlib/encoding.md index b5cb550631..7e3aa84d16 100644 --- a/docs/sources/reference/stdlib/encoding.md +++ b/docs/sources/reference/stdlib/encoding.md @@ -52,7 +52,7 @@ c3RyaW5nMTIzIT8kKiYoKSctPUB+ ## encoding.to_URLbase64 -The `encoding.to_base64` function encodes the original string into RFC4648-compliant URL safe Base64 encoded string +The `encoding.to_base64` function encodes the original string into RFC4648-compliant URL safe Base64 encoded string. ### Example From 07a0140fff6f82f20c5eb349cce8baa241a5e07b Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Tue, 3 Dec 2024 22:00:11 +0530 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc2693a86b..7375be47c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,7 @@ Main (unreleased) - Change processlist query to support ONLY_FULL_GROUP_BY sql_mode - Add perf_schema quantile columns to collector -- Add two new stdlib functions to_base64 and to_URLbase64 (@ravishankar15) +- Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15) ### Bugfixes