From 37818bc65f6a8fbecddd02198e9c6f14b9ff7516 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 11 Sep 2024 05:51:53 -0700 Subject: [PATCH] add it for sql query on views Signed-off-by: Surya Sashank Nistala --- .../opensearch/flint/spark/DisallowCatalogViews.scala | 8 +++----- .../covering/ApplyFlintSparkCoveringIndexSuite.scala | 9 +++++++++ .../flint/spark/ppl/FlintSparkPPLGrokITSuite.scala | 1 - .../opensearch/flint/spark/FlintPPLSparkExtensions.scala | 2 -- project/build.properties | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/DisallowCatalogViews.scala b/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/DisallowCatalogViews.scala index 61c7a4df1..8de43e0c5 100644 --- a/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/DisallowCatalogViews.scala +++ b/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/DisallowCatalogViews.scala @@ -9,16 +9,14 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View} import org.apache.spark.sql.catalyst.rules.Rule /** - * This analysis rule validates that the submitted query is not referencing a Glue Catalog view. The rule simply traverses - * the plan and validates that none of the nodes resolved to a [[View]]. + * This analysis rule validates that the submitted query is not referencing a Glue Catalog view. + * The rule simply traverses the plan and validates that none of the nodes resolved to a [[View]]. */ class DisallowCatalogViews extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { plan.foreachUp { case _: View => - throw new IllegalArgumentException( - s"Catalog View is not allowed to be queried" - ) + throw new IllegalArgumentException(s"Catalog View is not allowed to be queried") case other => other } diff --git a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/covering/ApplyFlintSparkCoveringIndexSuite.scala b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/covering/ApplyFlintSparkCoveringIndexSuite.scala index 2c5518778..5aed6514f 100644 --- a/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/covering/ApplyFlintSparkCoveringIndexSuite.scala +++ b/flint-spark-integration/src/test/scala/org/opensearch/flint/spark/covering/ApplyFlintSparkCoveringIndexSuite.scala @@ -55,6 +55,9 @@ class ApplyFlintSparkCoveringIndexSuite extends FlintSuite with Matchers { | ('F', 35), ('G', 40), ('H', 45), ('I', 50), ('J', 55) | """.stripMargin) + // Create a view + sql(s"CREATE VIEW myView AS SELECT name, age FROM $testTable WHERE age > 20") + // Mock static create method in FlintClientBuilder used by Flint data source clientBuilder .when(() => FlintClientBuilder.build(any(classOf[FlintOptions]))) @@ -141,6 +144,12 @@ class ApplyFlintSparkCoveringIndexSuite extends FlintSuite with Matchers { filterCondition = Some("name = 'A' OR age > 30"))) .assertIndexNotUsed(testTable) } + test("test querying catalog views throws validation exception") { + the[IllegalArgumentException] thrownBy { + sql(s"SELECT * FROM myview WHERE name = 'A' AND age > 30") + } should have message "Catalog View is not allowed to be queried" + + } test("should not apply if covering index is logically deleted") { assertFlintQueryRewriter diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLGrokITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLGrokITSuite.scala index 6df00e2cd..6da891c32 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLGrokITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLGrokITSuite.scala @@ -39,7 +39,6 @@ class FlintSparkPPLGrokITSuite } } - test("test querying catalog views throws validation exception") { the[IllegalArgumentException] thrownBy { sql(""" diff --git a/ppl-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintPPLSparkExtensions.scala b/ppl-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintPPLSparkExtensions.scala index de0491f86..26ad4b69b 100644 --- a/ppl-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintPPLSparkExtensions.scala +++ b/ppl-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintPPLSparkExtensions.scala @@ -18,7 +18,5 @@ class FlintPPLSparkExtensions extends (SparkSessionExtensions => Unit) { extensions.injectParser { (spark, parser) => new FlintSparkPPLParser(parser) } - - } } diff --git a/project/build.properties b/project/build.properties index 136f452e0..f344c1483 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.10.1 +sbt.version = 1.8.2