From 9b660fa993274bde7f93ea1187039761214f1fdd Mon Sep 17 00:00:00 2001 From: Stewart Bryson Date: Fri, 2 Sep 2022 15:46:08 -0400 Subject: [PATCH 1/7] internal-stage example. --- .gitignore | 1 + examples/internal-stage/build.gradle | 36 +++++++++++++++++++ examples/internal-stage/settings.gradle | 12 +++++++ .../src/main/java/AddNumbers.java | 21 +++++++++++ 4 files changed, 70 insertions(+) create mode 100644 examples/internal-stage/build.gradle create mode 100644 examples/internal-stage/settings.gradle create mode 100644 examples/internal-stage/src/main/java/AddNumbers.java diff --git a/.gitignore b/.gitignore index 8e5b76f..f0acc27 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ bin/ .settings .classpath .factorypath +.vscode/ \ No newline at end of file diff --git a/examples/internal-stage/build.gradle b/examples/internal-stage/build.gradle new file mode 100644 index 0000000..4604d45 --- /dev/null +++ b/examples/internal-stage/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'io.noumenal.gradle.snowflake' version '0.1.9' + id 'java' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.apache.logging.log4j:log4j-api:2.18.0' + implementation 'org.apache.logging.log4j:log4j-core:2.18.0' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + +snowflake { + // groupId = 'io.noumenal' + // artifactId = 'test-gradle-snowflake' + role = 'devops' + database = 'devops' + schema = 'gradle' + applications { + add_numbers { + inputs = ["a integer", "b integer"] + returns = "string" + handler = "AddNumbers.addNum" + } + } +} + +version='0.1.0' \ No newline at end of file diff --git a/examples/internal-stage/settings.gradle b/examples/internal-stage/settings.gradle new file mode 100644 index 0000000..80d7636 --- /dev/null +++ b/examples/internal-stage/settings.gradle @@ -0,0 +1,12 @@ +pluginManagement { + repositories { + maven { + url "s3://nio-maven" + authentication { + awsIm(AwsImAuthentication) + } + } + gradlePluginPortal() + } +} +rootProject.name = 'internal-stage' diff --git a/examples/internal-stage/src/main/java/AddNumbers.java b/examples/internal-stage/src/main/java/AddNumbers.java new file mode 100644 index 0000000..3991293 --- /dev/null +++ b/examples/internal-stage/src/main/java/AddNumbers.java @@ -0,0 +1,21 @@ +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class AddNumbers { + + private static final Logger logger = LogManager.getLogger(); + + public String addNum(int num1, int num2) { + try { + int sum = num1 + num2; + return ("Sum is: " + sum); + } catch (Exception e) { + logger.warn("Error: " + e.toString()); + return null; + } + } + + public static void main(String[] args) { + System.out.println("Hello World"); + } +} From eef110e701511ee1e48c73b91f5d64efa8c1a5fe Mon Sep 17 00:00:00 2001 From: Stewart Bryson Date: Tue, 6 Sep 2022 15:36:47 -0400 Subject: [PATCH 2/7] Adding examples. Changing FunctionalTest. --- examples/internal-stage/build.gradle | 16 --------- examples/simple-jar/build.gradle | 36 +++++++++++++++++++ examples/simple-jar/settings.gradle | 12 +++++++ .../simple-jar/src/main/java/AddNumbers.java | 21 +++++++++++ plugin/build.gradle | 18 ++++------ .../groovy/io/noumenal/FunctionalTest.groovy | 29 +++++++++++---- .../groovy/io/noumenal/SnowflakePlugin.groovy | 3 ++ .../io/noumenal/SnowflakePublish.groovy | 5 ++- 8 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 examples/simple-jar/build.gradle create mode 100644 examples/simple-jar/settings.gradle create mode 100644 examples/simple-jar/src/main/java/AddNumbers.java diff --git a/examples/internal-stage/build.gradle b/examples/internal-stage/build.gradle index 4604d45..6ebdd18 100644 --- a/examples/internal-stage/build.gradle +++ b/examples/internal-stage/build.gradle @@ -1,5 +1,4 @@ plugins { - id 'io.noumenal.gradle.snowflake' version '0.1.9' id 'java' } @@ -18,19 +17,4 @@ java { } } -snowflake { - // groupId = 'io.noumenal' - // artifactId = 'test-gradle-snowflake' - role = 'devops' - database = 'devops' - schema = 'gradle' - applications { - add_numbers { - inputs = ["a integer", "b integer"] - returns = "string" - handler = "AddNumbers.addNum" - } - } -} - version='0.1.0' \ No newline at end of file diff --git a/examples/simple-jar/build.gradle b/examples/simple-jar/build.gradle new file mode 100644 index 0000000..4604d45 --- /dev/null +++ b/examples/simple-jar/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'io.noumenal.gradle.snowflake' version '0.1.9' + id 'java' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.apache.logging.log4j:log4j-api:2.18.0' + implementation 'org.apache.logging.log4j:log4j-core:2.18.0' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + +snowflake { + // groupId = 'io.noumenal' + // artifactId = 'test-gradle-snowflake' + role = 'devops' + database = 'devops' + schema = 'gradle' + applications { + add_numbers { + inputs = ["a integer", "b integer"] + returns = "string" + handler = "AddNumbers.addNum" + } + } +} + +version='0.1.0' \ No newline at end of file diff --git a/examples/simple-jar/settings.gradle b/examples/simple-jar/settings.gradle new file mode 100644 index 0000000..80d7636 --- /dev/null +++ b/examples/simple-jar/settings.gradle @@ -0,0 +1,12 @@ +pluginManagement { + repositories { + maven { + url "s3://nio-maven" + authentication { + awsIm(AwsImAuthentication) + } + } + gradlePluginPortal() + } +} +rootProject.name = 'internal-stage' diff --git a/examples/simple-jar/src/main/java/AddNumbers.java b/examples/simple-jar/src/main/java/AddNumbers.java new file mode 100644 index 0000000..3991293 --- /dev/null +++ b/examples/simple-jar/src/main/java/AddNumbers.java @@ -0,0 +1,21 @@ +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class AddNumbers { + + private static final Logger logger = LogManager.getLogger(); + + public String addNum(int num1, int num2) { + try { + int sum = num1 + num2; + return ("Sum is: " + sum); + } catch (Exception e) { + logger.warn("Error: " + e.toString()); + return null; + } + } + + public static void main(String[] args) { + System.out.println("Hello World"); + } +} diff --git a/plugin/build.gradle b/plugin/build.gradle index 2490fac..3e0663f 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -36,7 +36,12 @@ testing { targets { all { // This test suite should run after the built-in test suite has run its tests - testTask.configure { shouldRunAfter(test) } + testTask.configure { + shouldRunAfter(test) + systemProperties project.properties.findAll { it.key.startsWith "snowflake."} + failFast true + testLogging.showStandardStreams true + } } } } @@ -60,19 +65,8 @@ gradlePlugin { } gradlePlugin.testSourceSets(sourceSets.functionalTest) - build.dependsOn validatePlugins -tasks.withType(Test) { - failFast true - testLogging.showStandardStreams true - systemProperty "account", findProperty("snowflake.account") - systemProperty "user", findProperty("snowflake.user") - systemProperty "password", findProperty("snowflake.password") - systemProperty "schema", findProperty("snowflake.schema") - systemProperty "publishUrl", findProperty("snowflake.publishUrl") -} - publishing { repositories { maven { diff --git a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy index 1b72f97..53fa004 100644 --- a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy +++ b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy @@ -28,10 +28,13 @@ class FunctionalTest extends Specification { File buildFile, settingsFile, javaFile @Shared - String account = System.getProperty("account"), - user = System.getProperty("user"), - password = System.getProperty("password"), - publishUrl = System.getProperty("publishUrl") + String account = System.getProperty("snowflake.account"), + user = System.getProperty("snowflake.user"), + password = System.getProperty("snowflake.password"), + publishUrl = System.getProperty("snowflake.publishUrl"), + role = System.getProperty("snowflake.role"), + database = System.getProperty("snowflake.database"), + schema = System.getProperty("snowflake.schema") def setupSpec() { settingsFile = new File(projectDir, 'settings.gradle') @@ -53,9 +56,9 @@ class FunctionalTest extends Specification { |snowflake { | groupId = 'io.noumenal' | artifactId = 'test-gradle-snowflake' - | role = 'devops' - | database = 'devops' - | schema = 'gradle' + | role = '$role' + | database = '$database' + | schema = '$schema' | applications { | add_numbers { | inputs = ["a integer", "b integer"] @@ -90,6 +93,7 @@ class FunctionalTest extends Specification { // helper method def executeSingleTask(String taskName, List args, Boolean logOutput = true) { + // ultra secure handling List systemArgs = [ "-Psnowflake.account=$account".toString(), "-Psnowflake.user=$user".toString(), @@ -114,6 +118,17 @@ class FunctionalTest extends Specification { return result } + def "shadowJar"() { + given: + taskName = 'shadowJar' + + when: + result = executeSingleTask(taskName, ['-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + def "snowflakePublish with publishUrl option"() { given: taskName = 'snowflakePublish' diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy index 1132d02..ca87789 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy @@ -28,6 +28,9 @@ class SnowflakePlugin implements Plugin { project.pluginProps.setParameters(project, PLUGIN) project.afterEvaluate { + // add shadowJar to build + project.tasks.build.dependsOn project.tasks.shadowJar + // create maven publishing if (!extension.useCustomMaven && extension.publishUrl) { diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy index a01c96b..b5edb38 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy @@ -1,5 +1,6 @@ package io.noumenal +import com.snowflake.snowpark.SnowparkClientException import com.snowflake.snowpark_java.PutResult import com.snowflake.snowpark_java.Session import groovy.util.logging.Slf4j @@ -154,7 +155,9 @@ class SnowflakePublish extends DefaultTask { try { session = Session.builder().configs(props).create() } catch (NullPointerException npe) { - throw new Exception("Snowflake connection details are missing.") + throw new Exception("Snowflake connection details are missing.", npe) + } catch (SnowparkClientException sce) { + throw new Exception("Snowflake connection details are incorrect.", sce) } return session } From 005ee971c80c75fe05ff32e2c0fef0470c999ea0 Mon Sep 17 00:00:00 2001 From: stewartbryson Date: Tue, 6 Sep 2022 21:40:41 -0400 Subject: [PATCH 3/7] Intermediate commit. --- plugin/build.gradle | 138 +++--- plugin/gradle.properties | 2 + .../groovy/io/noumenal/FunctionalTest.groovy | 229 +++++---- .../groovy/io/noumenal/SnowflakeTest.groovy | 37 ++ .../main/groovy/io/noumenal/Snowflake.groovy | 40 ++ .../io/noumenal/SnowflakeExtension.groovy | 18 +- .../groovy/io/noumenal/SnowflakePlugin.groovy | 2 +- .../io/noumenal/SnowflakePublish.groovy | 458 +++++++++--------- 8 files changed, 527 insertions(+), 397 deletions(-) create mode 100644 plugin/gradle.properties create mode 100644 plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy create mode 100644 plugin/src/main/groovy/io/noumenal/Snowflake.groovy diff --git a/plugin/build.gradle b/plugin/build.gradle index 3e0663f..06fd22d 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -1,51 +1,59 @@ plugins { - id 'java-gradle-plugin' - id 'groovy' - id 'maven-publish' - id 'com.adarshr.test-logger' version '3.2.0' - id "com.redpillanalytics.gradle-properties" version "1.0.7" - id 'com.fuseanalytics.gradle.s3' version '1.2.0' + id 'java-gradle-plugin' + id 'groovy' + id 'maven-publish' + id 'com.adarshr.test-logger' version '3.2.0' + id "com.redpillanalytics.gradle-properties" version "1.0.7" + id 'com.fuseanalytics.gradle.s3' version '1.2.0' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } } dependencies { - implementation 'org.codehaus.groovy:groovy:3.0.12' - implementation 'org.slf4j:slf4j-simple:1.7.36' - implementation 'com.snowflake:snowpark:1.5.0' - implementation 'gradle.plugin.com.redpillanalytics:gradle-properties:1.0.7' - implementation 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' + implementation 'org.codehaus.groovy:groovy:3.0.12' + implementation 'org.slf4j:slf4j-simple:2.0.0' + implementation 'com.snowflake:snowpark:1.6.0' + implementation 'gradle.plugin.com.redpillanalytics:gradle-properties:1.0.7' + implementation 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' } testing { - suites { - // Configure the built-in test suite - test { - // Use Spock test framework - useSpock('2.1-groovy-3.0') - } + suites { + // Configure the built-in test suite + test { + // Use Spock test framework + useSpock('2.1-groovy-3.0') + } - // Create a new test suite - functionalTest(JvmTestSuite) { - // Use Spock test framework - useSpock('2.1-groovy-3.0') + // Create a new test suite + functionalTest(JvmTestSuite) { + // Use Spock test framework + useSpock('2.1-groovy-3.0') - dependencies { - // functionalTest test suite depends on the production code in tests - implementation project - } + dependencies { + // functionalTest test suite depends on the production code in tests + implementation project + } - targets { - all { - // This test suite should run after the built-in test suite has run its tests - testTask.configure { - shouldRunAfter(test) - systemProperties project.properties.findAll { it.key.startsWith "snowflake."} - failFast true - testLogging.showStandardStreams true - } + targets { + all { + // This test suite should run after the built-in test suite has run its tests + testTask.configure { + shouldRunAfter(test) + systemProperties project.properties.findAll { it.key.startsWith "snowflake." } + systemProperty 'internalStage', findProperty('internalStage') + systemProperty 'externalStage', findProperty('externalStage') + failFast true + testLogging.showStandardStreams true + } + } } - } - } - } + } + } } //pluginBundle { @@ -55,50 +63,50 @@ testing { //} gradlePlugin { - // Define the plugin - plugins { - snowflake { - id = 'io.noumenal.gradle.snowflake' - implementationClass = 'io.noumenal.SnowflakePlugin' - } - } + // Define the plugin + plugins { + snowflake { + id = 'io.noumenal.gradle.snowflake' + implementationClass = 'io.noumenal.SnowflakePlugin' + } + } } gradlePlugin.testSourceSets(sourceSets.functionalTest) build.dependsOn validatePlugins publishing { - repositories { - maven { - name 's3' - url 's3://nio-maven' - authentication { - awsIm(AwsImAuthentication) - } - } - } + repositories { + maven { + name 's3' + url 's3://nio-maven' + authentication { + awsIm(AwsImAuthentication) + } + } + } } s3 { - bucket = 'docs.noumenal.io' - region = 'us-east-1' + bucket = 'docs.noumenal.io' + region = 'us-east-1' } tasks.register("publishLatestDocs", com.fuseanalytics.gradle.s3.S3Upload) { - description = "Publish the latest docs to S3." - keyPrefix = "${rootProject.name}/latest" - sourceDir = tasks.groovydoc.destinationDir - dependsOn groovydoc + description = "Publish the latest docs to S3." + keyPrefix = "${rootProject.name}/latest" + sourceDir = tasks.groovydoc.destinationDir + dependsOn groovydoc } tasks.register("publishVersionDocs", com.fuseanalytics.gradle.s3.S3Upload) { - description = "Publish the version docs to S3." - keyPrefix = "${rootProject.name}/$version" - sourceDir = tasks.groovydoc.destinationDir - dependsOn groovydoc + description = "Publish the version docs to S3." + keyPrefix = "${rootProject.name}/$version" + sourceDir = tasks.groovydoc.destinationDir + dependsOn groovydoc } tasks.register("publishDocs") { - dependsOn tasks.publishLatestDocs, tasks.publishVersionDocs - mustRunAfter tasks.publish + dependsOn tasks.publishLatestDocs, tasks.publishVersionDocs + mustRunAfter tasks.publish } diff --git a/plugin/gradle.properties b/plugin/gradle.properties new file mode 100644 index 0000000..2026e58 --- /dev/null +++ b/plugin/gradle.properties @@ -0,0 +1,2 @@ +internalStage=upload +externalStage=maven_snapshot diff --git a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy index 53fa004..d445ac4 100644 --- a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy +++ b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy @@ -4,6 +4,7 @@ package io.noumenal import groovy.util.logging.Slf4j +import spock.lang.Ignore import spock.lang.Shared import spock.lang.Specification import spock.lang.TempDir @@ -14,36 +15,38 @@ import org.gradle.testkit.runner.GradleRunner */ @Slf4j class FunctionalTest extends Specification { - @Shared - def result - - @Shared - String taskName - - @TempDir - @Shared - private File projectDir - - @Shared - File buildFile, settingsFile, javaFile - - @Shared - String account = System.getProperty("snowflake.account"), - user = System.getProperty("snowflake.user"), - password = System.getProperty("snowflake.password"), - publishUrl = System.getProperty("snowflake.publishUrl"), - role = System.getProperty("snowflake.role"), - database = System.getProperty("snowflake.database"), - schema = System.getProperty("snowflake.schema") - - def setupSpec() { - settingsFile = new File(projectDir, 'settings.gradle') - settingsFile.write(""" + @Shared + def result + + @Shared + String taskName + + @TempDir + @Shared + private File projectDir + + @Shared + File buildFile, settingsFile, javaFile + + @Shared + String account = System.getProperty("snowflake.account"), + user = System.getProperty("snowflake.user"), + password = System.getProperty("snowflake.password"), + publishUrl = System.getProperty("snowflake.publishUrl"), + role = System.getProperty("snowflake.role"), + database = System.getProperty("snowflake.database"), + schema = System.getProperty("snowflake.schema"), + internalStage = System.getProperty("internalStage"), + externalStage = System.getProperty("externalStage") + + def setupSpec() { + settingsFile = new File(projectDir, 'settings.gradle') + settingsFile.write(""" |rootProject.name = 'unit-test' |""".stripMargin()) - buildFile = new File(projectDir, 'build.gradle') - buildFile.write(""" + buildFile = new File(projectDir, 'build.gradle') + buildFile.write(""" |plugins { | id 'io.noumenal.gradle.snowflake' | id 'java' @@ -70,9 +73,9 @@ class FunctionalTest extends Specification { |version='0.1.0' |""".stripMargin()) - javaFile = new File("${projectDir}/src/main/java", 'AddNumbers.java') - javaFile.parentFile.mkdirs() - javaFile.write(""" + javaFile = new File("${projectDir}/src/main/java", 'AddNumbers.java') + javaFile.parentFile.mkdirs() + javaFile.write(""" |public class AddNumbers |{ | public String addNum(int num1, int num2) { @@ -89,76 +92,98 @@ class FunctionalTest extends Specification { | } |} |""".stripMargin()) - } - - // helper method - def executeSingleTask(String taskName, List args, Boolean logOutput = true) { - // ultra secure handling - List systemArgs = [ - "-Psnowflake.account=$account".toString(), - "-Psnowflake.user=$user".toString(), - "-Psnowflake.password=$password".toString() - ] - args.add(0, taskName) - args.addAll(systemArgs) - - // Don't print the password - //log.warn "runner arguments: ${args}" - - // execute the Gradle test build - result = GradleRunner.create() - .withProjectDir(projectDir) - .withArguments(args) - .withPluginClasspath() - .forwardOutput() - .build() - - // log the results - if (logOutput) log.warn result.getOutput() - return result - } - - def "shadowJar"() { - given: - taskName = 'shadowJar' - - when: - result = executeSingleTask(taskName, ['-Si']) - - then: - !result.tasks.collect { it.outcome }.contains('FAILURE') - } - - def "snowflakePublish with publishUrl option"() { - given: - taskName = 'snowflakePublish' - - when: - result = executeSingleTask(taskName, ['-Psnowflake.publishUrl=s3://nio-maven-test', '-Si']) - - then: - !result.tasks.collect { it.outcome }.contains('FAILURE') - } - - def "snowflakePublish without publishUrl option"() { - given: - taskName = 'snowflakePublish' - - when: - result = executeSingleTask(taskName, ['-Psnowflake.stage=upload', '-Si']) - - then: - !result.tasks.collect { it.outcome }.contains('FAILURE') - } - - def "snowflakePublish with custom JAR"() { - given: - taskName = 'snowflakePublish' - - when: - result = executeSingleTask(taskName, ['--jar', 'build/libs/unit-test-0.1.0-all.jar', '-Psnowflake.stage=upload', '-Si']) - - then: - !result.tasks.collect { it.outcome }.contains('FAILURE') - } + } + + // helper method + def executeSingleTask(String taskName, List args, Boolean logOutput = true) { + // ultra secure handling + List systemArgs = [ + "-Psnowflake.account=$account".toString(), + "-Psnowflake.user=$user".toString(), + "-Psnowflake.password=$password".toString() + ] + args.add(0, taskName) + args.addAll(systemArgs) + + // Don't print the password + //log.warn "runner arguments: ${args}" + + // execute the Gradle test build + result = GradleRunner.create() + .withProjectDir(projectDir) + .withArguments(args) + .withPluginClasspath() + .forwardOutput() + .build() + + // log the results + if (logOutput) log.warn result.getOutput() + return result + } + + def "dry run without publishUrl"() { + given: + taskName = 'snowflakePublish' + + when: + result = executeSingleTask(taskName, ['-Sim']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + + def "tasks with publishUrl"() { + given: + taskName = 'tasks' + + when: + result = executeSingleTask(taskName, ["-Psnowflake.stage=$externalStage".toString(), "-Psnowflake.publishUrl=$publishUrl".toString(), '-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + + def "shadowJar"() { + given: + taskName = 'shadowJar' + + when: + result = executeSingleTask(taskName, ['-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + + def "snowflakePublish with publishUrl option"() { + given: + taskName = 'snowflakePublish' + + when: + result = executeSingleTask(taskName, ["-Psnowflake.publishUrl=${publishUrl}".toString(), "-Psnowflake.stage=$externalStage".toString(), '-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + + def "snowflakePublish without publishUrl option"() { + given: + taskName = 'snowflakePublish' + + when: + result = executeSingleTask(taskName, ["-Psnowflake.stage=$internalStage".toString(), '-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } + + def "snowflakePublish with custom JAR"() { + given: + taskName = 'snowflakePublish' + + when: + result = executeSingleTask(taskName, ['--jar', 'build/libs/unit-test-0.1.0-all.jar', '-Psnowflake.stage=upload', '-Si']) + + then: + !result.tasks.collect { it.outcome }.contains('FAILURE') + } } diff --git a/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy b/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy new file mode 100644 index 0000000..1d156ed --- /dev/null +++ b/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy @@ -0,0 +1,37 @@ +package io.noumenal + +import groovy.util.logging.Slf4j +import spock.lang.Shared +import spock.lang.Specification + +@Slf4j +class SnowflakeTest extends Specification { + @Shared + String account = System.getProperty("snowflake.account"), + user = System.getProperty("snowflake.user"), + password = System.getProperty("snowflake.password"), + publishUrl = System.getProperty("snowflake.publishUrl"), + role = System.getProperty("snowflake.role"), + database = System.getProperty("snowflake.database"), + schema = System.getProperty("snowflake.schema"), + stage = System.getProperty("snowflake.stage"), + warehouse = System.getProperty("snowflake.warehouse") + + def "snowflake connection"() { + when: + def snowflake = new Snowflake([ + url : account, + user : user, + password : password, + role : role, + warehouse: warehouse, + db : database, + schema : schema + ]) + + then: + snowflake.props + snowflake.session + snowflake.assertStage(stage, schema, publishUrl) + } +} diff --git a/plugin/src/main/groovy/io/noumenal/Snowflake.groovy b/plugin/src/main/groovy/io/noumenal/Snowflake.groovy new file mode 100644 index 0000000..e53148a --- /dev/null +++ b/plugin/src/main/groovy/io/noumenal/Snowflake.groovy @@ -0,0 +1,40 @@ +package io.noumenal + +import com.snowflake.snowpark_java.Session +import groovy.util.logging.Slf4j +import net.snowflake.client.jdbc.SnowflakeStatement + +import java.sql.ResultSet +import java.sql.Statement + +@Slf4j +class Snowflake { + Map props + Session session + + Snowflake(Map props) { + this.props = props + Map printable = props.clone() + printable.password = "*********" + log.info "Snowflake config: $printable" + try { + this.session = Session.builder().configs(this.props).create() + } catch (NullPointerException npe) { + throw new Exception("Snowflake connection details are missing.", npe) + } + } + + def assertStage(String stage, String schema, String publishUrl) { + Statement statement = session.jdbcConnection().createStatement() + String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" + ResultSet rs = statement.executeQuery(sql) + String selectStage + if (rs.next()) { + selectStage = rs.getString(1) + } + rs.close() + statement.close() + // ensure we are matching our stage with our url + selectStage == publishUrl + } +} diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakeExtension.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakeExtension.groovy index 1cac77c..58c1354 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakeExtension.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakeExtension.groovy @@ -65,18 +65,20 @@ class SnowflakeExtension { private static String toSnakeCase( String text ) { text.replaceAll( /([A-Z])/, /_$1/ ).toLowerCase().replaceAll( /^_/, '' ) } - /** - * Convert names to be Camel Case. - */ - private static String toCamelCase( String text, boolean capitalized = false ) { - text = text.replaceAll( "(_)([A-Za-z0-9])", { Object[] it -> it[2].toUpperCase() } ) - return capitalized ? capitalize(text) : text - } + +// /** +// * Convert names to be Camel Case. +// */ +// private static String toCamelCase( String text, boolean capitalized = false ) { +// text = text.replaceAll( "(_)([A-Za-z0-9])", { Object[] it -> it[2].toUpperCase() } ) +// return capitalized ? capitalize(text) : text +// } /** * Return the name of the Maven publication task associated with the external stage. */ String getPublishTask() { - toCamelCase("publish_snowflake_publication_to_${stage}Repository") + //toCamelCase("publish_snowflake_publication_to_${stage}Repository") + "publishSnowflakePublicationTo${stage.capitalize()}Repository" } } \ No newline at end of file diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy index ca87789..74c12f9 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakePlugin.groovy @@ -66,7 +66,7 @@ class SnowflakePlugin implements Plugin { project.tasks.getByName(extension.publishTask).mustRunAfter project.tasks.test } - project.tasks.snowflakePublish.dependsOn project.tasks.test + project.tasks.snowflakePublish.dependsOn project.tasks.test, project.tasks.shadowJar } } } diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy index b5edb38..1a03fd5 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy @@ -1,6 +1,5 @@ package io.noumenal -import com.snowflake.snowpark.SnowparkClientException import com.snowflake.snowpark_java.PutResult import com.snowflake.snowpark_java.Session import groovy.util.logging.Slf4j @@ -22,224 +21,241 @@ import java.sql.Statement */ @Slf4j @CacheableTask -class SnowflakePublish extends DefaultTask { - private static String PLUGIN = 'snowflake' - - /** - * A helper for getting the plugin extension. - * - * @return A reference to the plugin extension. - */ - @Internal - def getExtension() { - project.extensions."$PLUGIN" - } - - /** - * The task Constructor with 'description' and 'group'. - * - * @return A custom task class. - */ - SnowflakePublish() { - description = "Publish a Java artifact to an external stage and create Snowflake Functions and Procedures." - group = "publishing" - } - - /** - * The Snowflake account URL, for instance: https://gradle-snowflake.us-east-1.snowflakecomputing.com:443. Overrides {@link SnowflakeExtension#account}. - */ - @Optional - @Input - @Option(option = "account", - description = "Override the URL of the Snowflake account." - ) - String account = extension.account - - /** - * The Snowflake user to connect as. Overrides {@link SnowflakeExtension#user}. - */ - @Optional - @Input - @Option(option = "user", - description = "Override the Snowflake user to connect as." - ) - String user = extension.user - - /** - * The Snowflake password to connect with. Overrides {@link SnowflakeExtension#password}. - */ - @Optional - @Input - @Option(option = "password", - description = "Override the Snowflake password to connect with." - ) - String password = extension.password - - /** - * The Snowflake database to connect to. Overrides {@link SnowflakeExtension#database}. - */ - @Optional - @Input - @Option(option = "database", - description = "Override the Snowflake database to connect to." - ) - String database = extension.database - - /** - * The Snowflake schema to connect with. Overrides {@link SnowflakeExtension#schema}. - */ - @Input - @Option(option = "schema", - description = "Override the Snowflake schema to connect with." - ) - String schema = extension.schema - - /** - * The Snowflake role to connect with. Overrides {@link SnowflakeExtension#warehouse}. - */ - @Input - @Option(option = "warehouse", - description = "Override the Snowflake role to connect with." - ) - String warehouse = extension.warehouse - - /** - * The Snowflake warehouse to connect with. Overrides {@link SnowflakeExtension#role}. - */ - @Input - @Option(option = "role", - description = "The Snowflake role to use." - ) - String role = extension.role - - /** - * The Snowflake stage to upload to. Overrides {@link SnowflakeExtension#stage}. - */ - @Optional - @Input - @Option(option = "stage", - description = "The Snowflake external stage to publish to." - ) - String stage = extension.stage - - /** - * Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata. - */ - @Optional - @Input - @Option(option = "jar", description = "Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.") - String jar = project.tasks.shadowJar.archiveFile.get() - - /** - * Create a Snowflake session. - * - * @return a Snowflake session. - */ - @Internal - Session getSession() { - Map props = [ - url : account, - user : user, - password : password, - role : role, - warehouse: warehouse, - db : database, - schema : schema - ] - Map printable = props.clone() - printable.password = "*********" - log.info "Snowflake config: $printable" - - Session session - // get a Snowflake session - try { - session = Session.builder().configs(props).create() - } catch (NullPointerException npe) { - throw new Exception("Snowflake connection details are missing.", npe) - } catch (SnowparkClientException sce) { - throw new Exception("Snowflake connection details are incorrect.", sce) - } - return session - } - - /** - * A simple text output file for the Snowflake applications create statements. Mainly for making the class Cacheable. - */ - @OutputFile - File output = project.file("${project.buildDir}/${PLUGIN}/output.txt") - - /** - * Get the 'import' property for the UDF. - * - * @return the 'import' property. - */ - String getImports(Session session) { - - String basePath = "@${stage}/${extension.groupId.replace('.', '/')}/${extension.artifactId}/${project.version}" - //log.warn "basePath: $basePath" - Statement statement = session.jdbcConnection().createStatement() - String sql = "LIST $basePath pattern='(.)*(-all)\\.jar'; select * from table(result_scan(last_query_id())) order by 'last_modified' asc;" - statement.unwrap(SnowflakeStatement.class).setParameter( - "MULTI_STATEMENT_COUNT", 2) - ResultSet rs = statement.executeQuery(sql) - String fileName - String filePath - try { - while (rs.next()) { - filePath = rs.getString(1) - } - fileName = filePath.replaceAll(/(.*)($project.version)(\/)(.*)/) { all, first, version, slash, filename -> - filename - } - } catch (Exception e) { - throw new Exception("Unable to detect the correct JAR in stage '${stage}'.") - } - rs.close() - statement.close() - "'$basePath/$fileName'" - } - - /** - * The Gradle TaskAction method. Publish the Snowflake Application. - */ - @TaskAction - def publish() { - // keep the session - Session session = this.session - String jar = project.tasks.shadowJar.archiveFile.get() - - if (!extension.publishUrl) { - def options = [ - AUTO_COMPRESS: 'false', - PARALLEL : '4' - ] - PutResult[] pr = session.file().put(jar, "$stage/libs", options) - pr.each { - log.warn "File ${it.sourceFileName}: ${it.status}" - } - } else { - // ensure that the stage and the publishUrl are aligned - Statement statement = session.jdbcConnection().createStatement() - String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" - ResultSet rs = statement.executeQuery(sql) - String selectStage - if (rs.next()) { - selectStage = rs.getString(1) - } - // ensure we are matching our stage with our url - assert selectStage == extension.publishUrl - } - - // automatically create application spec objects - output.write("Snowflake Application:\n\n") - project."$PLUGIN".applications.each { ApplicationContainer app -> - File jarFile = project.file(jar) - String createText = app.getCreate(extension.publishUrl ? getImports(session) : "'@$stage/libs/${jarFile.name}'") - String message = "Deploying ==> \n$createText" - log.warn message - output.append("$message\n") - session.jdbcConnection().createStatement().execute(createText) - } - session.close() - } +abstract class SnowflakePublish extends DefaultTask { + + private static String PLUGIN = 'snowflake' + + /** + * A helper for getting the plugin extension. + * + * @return A reference to the plugin extension. + */ + @Internal + def getExtension() { + project.extensions."$PLUGIN" + } + + /** + * The task Constructor with 'description' and 'group'. + * + * @return A custom task class. + */ + SnowflakePublish() { + description = "Publish a Java artifact to an external stage and create Snowflake Functions and Procedures." + group = "publishing" + } + + /** + * The Snowflake account URL, for instance: https://gradle-snowflake.us-east-1.snowflakecomputing.com:443. Overrides {@link SnowflakeExtension#account}. + */ + @Optional + @Input + @Option(option = "account", + description = "Override the URL of the Snowflake account." + ) + String account = extension.account + + /** + * The Snowflake user to connect as. Overrides {@link SnowflakeExtension#user}. + */ + @Optional + @Input + @Option(option = "user", + description = "Override the Snowflake user to connect as." + ) + String user = extension.user + + /** + * The Snowflake password to connect with. Overrides {@link SnowflakeExtension#password}. + */ + @Optional + @Input + @Option(option = "password", + description = "Override the Snowflake password to connect with." + ) + String password = extension.password + + /** + * The Snowflake database to connect to. Overrides {@link SnowflakeExtension#database}. + */ + @Optional + @Input + @Option(option = "database", + description = "Override the Snowflake database to connect to." + ) + String database = extension.database + + /** + * The Snowflake schema to connect with. Overrides {@link SnowflakeExtension#schema}. + */ + @Input + @Option(option = "schema", + description = "Override the Snowflake schema to connect with." + ) + String schema = extension.schema + + /** + * The Snowflake role to connect with. Overrides {@link SnowflakeExtension#warehouse}. + */ + @Input + @Option(option = "warehouse", + description = "Override the Snowflake role to connect with." + ) + String warehouse = extension.warehouse + + /** + * The Snowflake warehouse to connect with. Overrides {@link SnowflakeExtension#role}. + */ + @Input + @Option(option = "role", + description = "The Snowflake role to use." + ) + String role = extension.role + + /** + * The Snowflake stage to upload to. Overrides {@link SnowflakeExtension#stage}. + */ + @Optional + @Input + @Option(option = "stage", + description = "The Snowflake external stage to publish to." + ) + String stage = extension.stage + + /** + * Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata. + */ + @Optional + @Input + @Option(option = "jar", description = "Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.") + String jar = project.tasks.shadowJar.archiveFile.get() + + /** + * Create a Snowflake object + * + * @return Snowflake object + */ + @Internal + Snowflake snowflake = new Snowflake([ + url : account, + user : user, + password : password, + role : role, + warehouse: warehouse, + db : database, + schema : schema + ]) + +// /** +// * Create a Snowflake session. +// * +// * @return a Snowflake session. +// */ +// @Internal +// Session getSession() { +// Map props = [ +// url : account, +// user : user, +// password : password, +// role : role, +// warehouse: warehouse, +// db : database, +// schema : schema +// ] +// Map printable = props.clone() +// printable.password = "*********" +// log.info "Snowflake config: $printable" +// +// Session session +// // get a Snowflake session +// try { +// session = Session.builder().configs(props).create() +// } catch (NullPointerException npe) { +// throw new Exception("Snowflake connection details are missing.", npe) +// } +// return session +// } + + /** + * A simple text output file for the Snowflake applications create statements. Mainly for making the class Cacheable. + */ + @OutputFile + File output = project.file("${project.buildDir}/${PLUGIN}/output.txt") + + /** + * Get the 'import' property for the UDF. + * + * @return the 'import' property. + */ + @Internal + String getImports() { + + String basePath = "@${stage}/${extension.groupId.replace('.', '/')}/${extension.artifactId}/${project.version}" + //log.warn "basePath: $basePath" + Statement statement = snowflake.session.jdbcConnection().createStatement() + String sql = "LIST $basePath pattern='(.)*(-all)\\.jar'; select * from table(result_scan(last_query_id())) order by 'last_modified' asc;" + statement.unwrap(SnowflakeStatement.class).setParameter( + "MULTI_STATEMENT_COUNT", 2) + ResultSet rs = statement.executeQuery(sql) + String fileName + String filePath + try { + while (rs.next()) { + filePath = rs.getString(1) + } + fileName = filePath.replaceAll(/(.*)($project.version)(\/)(.*)/) { all, first, version, slash, filename -> + filename + } + } catch (Exception e) { + throw new Exception("Unable to detect the correct JAR in stage '${stage}'.") + } + rs.close() + statement.close() + "'$basePath/$fileName'" + } + + /** + * The Gradle TaskAction method. Publish the Snowflake Application. + */ + @TaskAction + def publish() { + // keep the session + //Session session = this.session + String jar = project.tasks.shadowJar.archiveFile.get() + + if (!extension.publishUrl) { + def options = [ + AUTO_COMPRESS: 'false', + PARALLEL : '4' + ] + PutResult[] pr = snowflake.session.file().put(jar, "$stage/libs", options) + pr.each { + log.warn "File ${it.sourceFileName}: ${it.status}" + } + } else { + // ensure that the stage and the publishUrl are aligned + assert snowflake.assertStage(stage, schema, extension.publishUrl.toString()) +// Statement statement = snowflake.session.jdbcConnection().createStatement() +// String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" +// ResultSet rs = statement.executeQuery(sql) +// String selectStage +// if (rs.next()) { +// selectStage = rs.getString(1) +// } +// // ensure we are matching our stage with our url +// assert selectStage == extension.publishUrl + } + + // automatically create application spec objects + output.write("Snowflake Application:\n\n") + project."$PLUGIN".applications.each { ApplicationContainer app -> + File jarFile = project.file(jar) + String createText = app.getCreate(extension.publishUrl ? getImports() : "'@$stage/libs/${jarFile.name}'") + String message = "Deploying ==> \n$createText" + log.warn message + output.append("$message\n") + snowflake.session.jdbcConnection().createStatement().execute(createText) + } + snowflake.session.close() + } } From e167387bfed41c99210aad9d7ba56c54fa797c8d Mon Sep 17 00:00:00 2001 From: stewartbryson Date: Tue, 6 Sep 2022 21:54:12 -0400 Subject: [PATCH 4/7] Removed the Snowflake class. --- .../groovy/io/noumenal/FunctionalTest.groovy | 1 - .../groovy/io/noumenal/SnowflakeTest.groovy | 37 ------- .../main/groovy/io/noumenal/Snowflake.groovy | 40 -------- .../io/noumenal/SnowflakePublish.groovy | 96 ++++++++----------- 4 files changed, 41 insertions(+), 133 deletions(-) delete mode 100644 plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy delete mode 100644 plugin/src/main/groovy/io/noumenal/Snowflake.groovy diff --git a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy index d445ac4..3033419 100644 --- a/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy +++ b/plugin/src/functionalTest/groovy/io/noumenal/FunctionalTest.groovy @@ -4,7 +4,6 @@ package io.noumenal import groovy.util.logging.Slf4j -import spock.lang.Ignore import spock.lang.Shared import spock.lang.Specification import spock.lang.TempDir diff --git a/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy b/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy deleted file mode 100644 index 1d156ed..0000000 --- a/plugin/src/functionalTest/groovy/io/noumenal/SnowflakeTest.groovy +++ /dev/null @@ -1,37 +0,0 @@ -package io.noumenal - -import groovy.util.logging.Slf4j -import spock.lang.Shared -import spock.lang.Specification - -@Slf4j -class SnowflakeTest extends Specification { - @Shared - String account = System.getProperty("snowflake.account"), - user = System.getProperty("snowflake.user"), - password = System.getProperty("snowflake.password"), - publishUrl = System.getProperty("snowflake.publishUrl"), - role = System.getProperty("snowflake.role"), - database = System.getProperty("snowflake.database"), - schema = System.getProperty("snowflake.schema"), - stage = System.getProperty("snowflake.stage"), - warehouse = System.getProperty("snowflake.warehouse") - - def "snowflake connection"() { - when: - def snowflake = new Snowflake([ - url : account, - user : user, - password : password, - role : role, - warehouse: warehouse, - db : database, - schema : schema - ]) - - then: - snowflake.props - snowflake.session - snowflake.assertStage(stage, schema, publishUrl) - } -} diff --git a/plugin/src/main/groovy/io/noumenal/Snowflake.groovy b/plugin/src/main/groovy/io/noumenal/Snowflake.groovy deleted file mode 100644 index e53148a..0000000 --- a/plugin/src/main/groovy/io/noumenal/Snowflake.groovy +++ /dev/null @@ -1,40 +0,0 @@ -package io.noumenal - -import com.snowflake.snowpark_java.Session -import groovy.util.logging.Slf4j -import net.snowflake.client.jdbc.SnowflakeStatement - -import java.sql.ResultSet -import java.sql.Statement - -@Slf4j -class Snowflake { - Map props - Session session - - Snowflake(Map props) { - this.props = props - Map printable = props.clone() - printable.password = "*********" - log.info "Snowflake config: $printable" - try { - this.session = Session.builder().configs(this.props).create() - } catch (NullPointerException npe) { - throw new Exception("Snowflake connection details are missing.", npe) - } - } - - def assertStage(String stage, String schema, String publishUrl) { - Statement statement = session.jdbcConnection().createStatement() - String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" - ResultSet rs = statement.executeQuery(sql) - String selectStage - if (rs.next()) { - selectStage = rs.getString(1) - } - rs.close() - statement.close() - // ensure we are matching our stage with our url - selectStage == publishUrl - } -} diff --git a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy index 1a03fd5..ed5d7bd 100644 --- a/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy +++ b/plugin/src/main/groovy/io/noumenal/SnowflakePublish.groovy @@ -131,50 +131,34 @@ abstract class SnowflakePublish extends DefaultTask { String jar = project.tasks.shadowJar.archiveFile.get() /** - * Create a Snowflake object + * Create a Snowflake session. * - * @return Snowflake object + * @return a Snowflake session. */ @Internal - Snowflake snowflake = new Snowflake([ - url : account, - user : user, - password : password, - role : role, - warehouse: warehouse, - db : database, - schema : schema - ]) - -// /** -// * Create a Snowflake session. -// * -// * @return a Snowflake session. -// */ -// @Internal -// Session getSession() { -// Map props = [ -// url : account, -// user : user, -// password : password, -// role : role, -// warehouse: warehouse, -// db : database, -// schema : schema -// ] -// Map printable = props.clone() -// printable.password = "*********" -// log.info "Snowflake config: $printable" -// -// Session session -// // get a Snowflake session -// try { -// session = Session.builder().configs(props).create() -// } catch (NullPointerException npe) { -// throw new Exception("Snowflake connection details are missing.", npe) -// } -// return session -// } + Session getSession() { + Map props = [ + url : account, + user : user, + password : password, + role : role, + warehouse: warehouse, + db : database, + schema : schema + ] + Map printable = props.clone() + printable.password = "*********" + log.info "Snowflake config: $printable" + + Session session + // get a Snowflake session + try { + session = Session.builder().configs(props).create() + } catch (NullPointerException npe) { + throw new Exception("Snowflake connection details are missing.", npe) + } + return session + } /** * A simple text output file for the Snowflake applications create statements. Mainly for making the class Cacheable. @@ -192,7 +176,7 @@ abstract class SnowflakePublish extends DefaultTask { String basePath = "@${stage}/${extension.groupId.replace('.', '/')}/${extension.artifactId}/${project.version}" //log.warn "basePath: $basePath" - Statement statement = snowflake.session.jdbcConnection().createStatement() + Statement statement = session.jdbcConnection().createStatement() String sql = "LIST $basePath pattern='(.)*(-all)\\.jar'; select * from table(result_scan(last_query_id())) order by 'last_modified' asc;" statement.unwrap(SnowflakeStatement.class).setParameter( "MULTI_STATEMENT_COUNT", 2) @@ -228,22 +212,24 @@ abstract class SnowflakePublish extends DefaultTask { AUTO_COMPRESS: 'false', PARALLEL : '4' ] - PutResult[] pr = snowflake.session.file().put(jar, "$stage/libs", options) + PutResult[] pr = session.file().put(jar, "$stage/libs", options) pr.each { log.warn "File ${it.sourceFileName}: ${it.status}" } } else { // ensure that the stage and the publishUrl are aligned - assert snowflake.assertStage(stage, schema, extension.publishUrl.toString()) -// Statement statement = snowflake.session.jdbcConnection().createStatement() -// String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" -// ResultSet rs = statement.executeQuery(sql) -// String selectStage -// if (rs.next()) { -// selectStage = rs.getString(1) -// } -// // ensure we are matching our stage with our url -// assert selectStage == extension.publishUrl + //assert snowflake.assertStage(stage, schema, extension.publishUrl.toString()) + Statement statement = session.jdbcConnection().createStatement() + String sql = "select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$schema') and stage_type='External Named'" + ResultSet rs = statement.executeQuery(sql) + String selectStage + if (rs.next()) { + selectStage = rs.getString(1) + } + // ensure we are matching our stage with our url + rs.close() + statement.close() + assert selectStage == extension.publishUrl } // automatically create application spec objects @@ -254,8 +240,8 @@ abstract class SnowflakePublish extends DefaultTask { String message = "Deploying ==> \n$createText" log.warn message output.append("$message\n") - snowflake.session.jdbcConnection().createStatement().execute(createText) + session.jdbcConnection().createStatement().execute(createText) } - snowflake.session.close() + session.close() } } From 4a3dc1d8a1c6094e9d66b8fe73fd875d92a5b592 Mon Sep 17 00:00:00 2001 From: stewartbryson Date: Wed, 7 Sep 2022 12:57:06 -0400 Subject: [PATCH 5/7] Testing new AWS creds for PR only. --- .github/workflows/gradle-pr.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle-pr.yml b/.github/workflows/gradle-pr.yml index c548f74..0a73479 100644 --- a/.github/workflows/gradle-pr.yml +++ b/.github/workflows/gradle-pr.yml @@ -3,10 +3,6 @@ name: Run Gradle on PRs on: pull_request -env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} - jobs: build: runs-on: ubuntu-latest @@ -23,9 +19,12 @@ jobs: uses: gradle/gradle-build-action@v2 with: arguments: build validatePlugin functionalTest buildDashboard + env: + AWS_ACCESS_KEY_ID: ${{secrets.CIO_AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.CIO_AWS_SECRET_ACCESS_KEY}} - name: Publish tests uses: mikepenz/action-junit-report@v3 if: always() with: - report_paths: '**/build/test-results/**/TEST-*.xml' \ No newline at end of file + report_paths: '**/build/test-results/**/TEST-*.xml' From f90f610d736c842a7a0e93120a9fd294fbb0fd41 Mon Sep 17 00:00:00 2001 From: stewartbryson Date: Wed, 7 Sep 2022 13:05:42 -0400 Subject: [PATCH 6/7] Adding publish AWS creds. --- .github/workflows/gradle-publish.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle-publish.yml b/.github/workflows/gradle-publish.yml index 8d56814..01a57e6 100644 --- a/.github/workflows/gradle-publish.yml +++ b/.github/workflows/gradle-publish.yml @@ -6,10 +6,6 @@ on: - main types: [ closed ] -env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} - jobs: build: if: ${{ github.event.pull_request.merged }} @@ -30,10 +26,21 @@ jobs: with: arguments: release -Prelease.disableChecks -Prelease.localOnly + - name: Gradle build and test + uses: gradle/gradle-build-action@v2 + with: + arguments: build buildDashboard + env: + AWS_ACCESS_KEY_ID: ${{secrets.CIO_AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.CIO_AWS_SECRET_ACCESS_KEY}} + - name: Gradle publish uses: gradle/gradle-build-action@v2 with: - arguments: build publish githubRelease publishDocs buildDashboard -PgithubToken=${{secrets.GITHUB_TOKEN}} + arguments: publish githubRelease publishDocs buildDashboard -PgithubToken=${{secrets.GITHUB_TOKEN}} + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} - name: Publish tests uses: mikepenz/action-junit-report@v3 From 39ad74e4fb20d6ec6d66dd011ddef7f21e225a1b Mon Sep 17 00:00:00 2001 From: stewartbryson Date: Wed, 7 Sep 2022 13:14:27 -0400 Subject: [PATCH 7/7] Moved gradle cache. --- settings.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.gradle b/settings.gradle index 5559ec3..e93bbe8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,8 +13,8 @@ buildCache { enabled = false } remote(com.github.burrunan.s3cache.AwsS3BuildCache) { - region = 'us-east-1' - bucket = 'nio-gradle-cache' + region = 'us-west-2' + bucket = 'coalesce.gradle.cache' prefix = "${rootProject.name}/" push = isCiServer lookupDefaultAwsCredentials = true