From 1b60805dd817e2959a383c22b5c5c3162555ffc4 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Wed, 16 Aug 2023 13:07:01 -0500 Subject: [PATCH] Fix double http in the Spark Driver UI Link (#389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix double http in the Spark Driver UI Link Signed-off-by: mucahit-kantepe * fix test Signed-off-by: mucahit-kantepe * Apply suggestions from code review Co-authored-by: Ketan Umare <16888709+kumare3@users.noreply.github.com> Signed-off-by: Mücahit Kantepe * Update spark.go Signed-off-by: Mücahit Kantepe * fmt Signed-off-by: mucahit-kantepe * updated to append https if neither http or https prefixes spark uri Signed-off-by: Daniel Rammer --------- Signed-off-by: mucahit-kantepe Signed-off-by: Mücahit Kantepe Signed-off-by: Daniel Rammer Co-authored-by: mucahit-kantepe Co-authored-by: Mücahit Kantepe Co-authored-by: Ketan Umare <16888709+kumare3@users.noreply.github.com> --- go/tasks/plugins/k8s/spark/spark.go | 9 +++++++-- go/tasks/plugins/k8s/spark/spark_test.go | 5 ++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/go/tasks/plugins/k8s/spark/spark.go b/go/tasks/plugins/k8s/spark/spark.go index f0d328b2c..18d11a507 100755 --- a/go/tasks/plugins/k8s/spark/spark.go +++ b/go/tasks/plugins/k8s/spark/spark.go @@ -413,8 +413,13 @@ func getEventInfoForSpark(pluginContext k8s.PluginContext, sj *sparkOp.SparkAppl }) } } else if sj.Status.AppState.State == sparkOp.RunningState && sj.Status.DriverInfo.WebUIIngressAddress != "" { - // Append https as the operator doesn't currently. - customInfoMap[sparkDriverUI] = fmt.Sprintf("https://%s", sj.Status.DriverInfo.WebUIIngressAddress) + // Older versions of spark-operator does not append http:// but newer versions do. + uri := sj.Status.DriverInfo.WebUIIngressAddress + if !strings.HasPrefix(uri, "https://") && !strings.HasPrefix(uri, "http://") { + uri = fmt.Sprintf("https://%s", uri) + } + customInfoMap[sparkDriverUI] = uri + // Custom doesn't work unless the UI has a custom plugin to parse this, hence add to Logs as well. taskLogs = append(taskLogs, &core.TaskLog{ Uri: customInfoMap[sparkDriverUI], diff --git a/go/tasks/plugins/k8s/spark/spark_test.go b/go/tasks/plugins/k8s/spark/spark_test.go index 5750643a2..efa52274b 100755 --- a/go/tasks/plugins/k8s/spark/spark_test.go +++ b/go/tasks/plugins/k8s/spark/spark_test.go @@ -2,7 +2,6 @@ package spark import ( "context" - "fmt" "os" "strconv" "testing" @@ -34,7 +33,7 @@ import ( const sparkMainClass = "MainClass" const sparkApplicationFile = "local:///spark_app.py" const testImage = "image://" -const sparkUIAddress = "spark-ui.flyte" +const sparkUIAddress = "https://spark-ui.flyte" var ( dummySparkConf = map[string]string{ @@ -92,7 +91,7 @@ func TestGetEventInfo(t *testing.T) { info, err := getEventInfoForSpark(taskCtx, dummySparkApplication(sj.RunningState)) assert.NoError(t, err) assert.Len(t, info.Logs, 6) - assert.Equal(t, fmt.Sprintf("https://%s", sparkUIAddress), info.CustomInfo.Fields[sparkDriverUI].GetStringValue()) + assert.Equal(t, "https://spark-ui.flyte", info.CustomInfo.Fields[sparkDriverUI].GetStringValue()) generatedLinks := make([]string, 0, len(info.Logs)) for _, l := range info.Logs { generatedLinks = append(generatedLinks, l.Uri)