diff --git a/.github/workflows/gradle-pr.yml b/.github/workflows/gradle-pr.yml index 69b021e..f8a8124 100644 --- a/.github/workflows/gradle-pr.yml +++ b/.github/workflows/gradle-pr.yml @@ -28,11 +28,11 @@ jobs: env: GRADLE_PROPERTIES: ${{secrets.GRADLE_PROPERTIES}} - - id: snowsql_config + - id: credentials_file run: | - mkdir -p $HOME/.snowsql - echo "$SNOW_CONFIG" > $HOME/.snowsql/config - name: 'Write .snowsql/config file' + mkdir -p $HOME/.snowflake + echo "$SNOW_CONFIG" > $HOME/.snowflake/config.toml + name: 'Write .snowflake/config.toml file' shell: bash env: SNOW_CONFIG: ${{secrets.SNOW_CONFIG}} @@ -79,11 +79,11 @@ jobs: env: GRADLE_PROPERTIES: ${{secrets.GRADLE_PROPERTIES}} - - id: snowsql_config + - id: credentials_file run: | - mkdir -p $HOME/.snowsql - echo "$SNOW_CONFIG" > $HOME/.snowsql/config - name: 'Write .snowsql/config file' + mkdir -p $HOME/.snowflake + echo "$SNOW_CONFIG" > $HOME/.snowflake/config.toml + name: 'Write .snowflake/config.toml file' shell: bash env: SNOW_CONFIG: ${{secrets.SNOW_CONFIG}} diff --git a/.github/workflows/gradle-publish.yml b/.github/workflows/gradle-publish.yml index a635258..0e7e080 100644 --- a/.github/workflows/gradle-publish.yml +++ b/.github/workflows/gradle-publish.yml @@ -29,9 +29,9 @@ jobs: GRADLE_PROPERTIES: ${{secrets.GRADLE_PROPERTIES}} - run: | - mkdir -p $HOME/.snowsql - echo "$SNOW_CONFIG" > $HOME/.snowsql/config - name: 'Write .snowsql/config file' + mkdir -p $HOME/.snowflake + echo "$SNOW_CONFIG" > $HOME/.snowflake/config.toml + name: 'Write .snowflake/config.toml file' shell: bash env: SNOW_CONFIG: ${{secrets.SNOW_CONFIG}} @@ -93,11 +93,11 @@ jobs: env: GRADLE_PROPERTIES: ${{secrets.GRADLE_PROPERTIES}} - - id: snowsql_config + - id: credentials_file run: | - mkdir -p $HOME/.snowsql - echo "$SNOW_CONFIG" > $HOME/.snowsql/config - name: 'Write .snowsql/config file' + mkdir -p $HOME/.snowflake + echo "$SNOW_CONFIG" > $HOME/.snowflake/config.toml + name: 'Write .snowflake/config.toml file' shell: bash env: SNOW_CONFIG: ${{secrets.SNOW_CONFIG}} diff --git a/examples/groovy/build.gradle b/examples/groovy/build.gradle index ad5a873..f293630 100644 --- a/examples/groovy/build.gradle +++ b/examples/groovy/build.gradle @@ -9,7 +9,7 @@ repositories { } dependencies { - implementation 'org.codehaus.groovy:groovy:3.0.17' + implementation 'org.codehaus.groovy:groovy:3.0.18' } java { diff --git a/examples/java-testing/build.gradle b/examples/java-testing/build.gradle index 267f50c..f661dab 100644 --- a/examples/java-testing/build.gradle +++ b/examples/java-testing/build.gradle @@ -47,7 +47,7 @@ testing { } testTask.configure { failFast true - // which SnowSQL connection to use + // which credentials connection to use systemProperty 'connection', snowflake.connection // if this is ephemeral, the test spec needs the name to connect to if (snowflake.useEphemeral) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37aef8d..a363877 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/plugin/build.gradle b/plugin/build.gradle index aa662d2..bd0e690 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -14,7 +14,7 @@ java { } dependencies { - implementation 'org.codehaus.groovy:groovy:3.0.17' + implementation 'org.codehaus.groovy:groovy:3.0.18' implementation 'org.slf4j:slf4j-simple:2.0.7' implementation 'com.snowflake:snowpark:1.8.0' implementation 'gradle.plugin.com.redpillanalytics:gradle-properties:1.0.7' diff --git a/plugin/src/functionalTest/groovy/io/github/stewartbryson/SnowflakeTest.groovy b/plugin/src/functionalTest/groovy/io/github/stewartbryson/SnowflakeTest.groovy index 1bd0536..a4f5b92 100644 --- a/plugin/src/functionalTest/groovy/io/github/stewartbryson/SnowflakeTest.groovy +++ b/plugin/src/functionalTest/groovy/io/github/stewartbryson/SnowflakeTest.groovy @@ -10,7 +10,7 @@ class SnowflakeTest extends Specification { @Shared Snowflake snowflake - def "parse snowsql config"() { + def "parse credentials config"() { given: "a Snowflake class" snowflake = new Snowflake('gradle_plugin') diff --git a/plugin/src/main/groovy/io/github/stewartbryson/SnowConfig.groovy b/plugin/src/main/groovy/io/github/stewartbryson/SnowConfig.groovy index 993e3a9..9566402 100644 --- a/plugin/src/main/groovy/io/github/stewartbryson/SnowConfig.groovy +++ b/plugin/src/main/groovy/io/github/stewartbryson/SnowConfig.groovy @@ -4,91 +4,96 @@ import groovy.util.logging.Slf4j import org.ini4j.Ini /** - * A class for parsing a SnowSQL config file. + * A class for parsing a credentials config file. */ @Slf4j class SnowConfig { - /** - * The snowsql config file. - */ - File config + /** + * The credentials config file. + */ + File config - /** - * The SnowSQL connection to use. - */ - String connection + /** + * The credentials connection to use. + */ + String connection - /** - * Constructor using auto-detected SnowSQL config file. - * - * @return Snowflake class. - */ - SnowConfig(String connection) { - this.connection = connection - // first look for ~/.snowsql/config - def homeConfig = new File(System.getProperty("user.home").toString() + "/.snowsql/config") - // then look for ./snowconfig - def projectConfig = new File('snow-config') - if (homeConfig.exists()) { - config = homeConfig - } else if (projectConfig.exists()) { - config = projectConfig - } else { - throw new Exception("Unable to find a SnowSQL config file.") - } - log.warn "Using snowsql config file: ${config.absolutePath}" - } + /** + * Constructor using auto-detected credentials config file. + * + * @return Snowflake class. + */ + SnowConfig(String connection) { + this.connection = connection + // first look for ~/.snowflake/config.toml + def snowcli = new File(System.getProperty("user.home").toString() + "/.snowflake/config.toml") + // second look for ./config.toml + def projectConfig = new File('config.toml') + // third look for ~/.snowsql/config + def snowsql = new File(System.getProperty("user.home").toString() + "/.snowsql/config") - /** - * Constructor using explicit SnowSQL config file as a File object. - * - * @return Snowflake class. - */ - SnowConfig(File config, String connection) { - this.connection = connection - this.config = config - } + if (snowcli.exists()) { + config = snowcli + } else if (projectConfig.exists()) { + config = projectConfig + } else if (snowsql.exists()) { + config = snowsql + } else { + throw new Exception("Unable to find a credentials config file.") + } + log.warn "Using credentials config file: ${config.absolutePath}" + } - /** - * Constructor using explicit SnowSQL config file as a String path. - * - * @return Snowflake class. - */ - SnowConfig(String config, String connection) { - this.connection = connection - this.config = new File(config) - } + /** + * Constructor using explicit credentials config file as a File object. + * + * @return Snowflake class. + */ + SnowConfig(File config, String connection) { + this.connection = connection + this.config = config + } - /** - * Build a Map of connection properties for making a Snowflake connection. - * - * @return Snowflake connection properties. - */ - Map getConnectionsProps() { - //Map props1 = [account: "account", user: "user", password: "password"] - Map props = [:] - Ini ini = new Ini(config) - // first get all the connection defaults - ini.get("connections").each { key, value -> - props."${key.replaceAll(/name$/, '')}" = value - } - // now replace defaults with the connection props - String connectionName = "connections" + '.' + connection - ini.get(connectionName).each { key, value -> - props."${key.replaceAll(/name$/, '')}" = value - } - // construct url from account - props.url = "https://" + props.account + ".snowflakecomputing.com" - props.remove("account") - // special password handling to support quoted values - props.password = props.password.toString().replaceAll(/("*)([^"$]+)("*)/) {all, q1, pwd, q2 -> - pwd - } + /** + * Constructor using explicit credentials config file as a String path. + * + * @return Snowflake class. + */ + SnowConfig(String config, String connection) { + this.connection = connection + this.config = new File(config) + } - // we need at least these three to make a connection - if (!props.url || !props.user || !props.password) { - throw new Exception("'url', 'user', or 'password' is not configured.") - } - return props - } + /** + * Build a Map of connection properties for making a Snowflake connection. + * + * @return Snowflake connection properties. + */ + Map getConnectionsProps() { + //Map props1 = [account: "account", user: "user", password: "password"] + Map props = [:] + Ini ini = new Ini(config) + // first get all the connection defaults + ini.get("connections").each { key, value -> + props."${key.replaceAll(/name$/, '')}" = value + } + // now replace defaults with the connection props + String connectionName = "connections" + '.' + connection + ini.get(connectionName).each { key, value -> + props."${key.replaceAll(/name$/, '')}" = value + } + // construct url from account + props.url = "https://" + props.account + ".snowflakecomputing.com" + props.remove("account") + // special password handling to support quoted values + props.password = props.password.toString().replaceAll(/("*)([^"$]+)("*)/) { all, q1, pwd, q2 -> + pwd + } + + // we need at least these three to make a connection + if (!props.url || !props.user || !props.password) { + throw new Exception("'url', 'user', or 'password' is not configured.") + } + return props + } } \ No newline at end of file diff --git a/plugin/src/main/groovy/io/github/stewartbryson/Snowflake.groovy b/plugin/src/main/groovy/io/github/stewartbryson/Snowflake.groovy index 00eb797..4bc9717 100644 --- a/plugin/src/main/groovy/io/github/stewartbryson/Snowflake.groovy +++ b/plugin/src/main/groovy/io/github/stewartbryson/Snowflake.groovy @@ -74,7 +74,7 @@ class Snowflake { Snowflake() {} /** - * Constructor using autodetected SnowSQL config file. + * Constructor using auto-detected credentials config file. * * @return Snowflake class. */ @@ -84,7 +84,7 @@ class Snowflake { } /** - * Constructor using explicit SnowSQL config file as a File object. + * Constructor using explicit credentials config file as a File object. * * @return Snowflake class. */ diff --git a/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeExtension.groovy b/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeExtension.groovy index 8fdccc9..d3a8a5a 100644 --- a/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeExtension.groovy +++ b/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeExtension.groovy @@ -29,7 +29,7 @@ class SnowflakeExtension { } /** - * The SnowSQL connection to use. Default: use the base connection info in SnowSQL config. + * The credentials connection to use. Default: use the base connection info in credentials config. */ String connection diff --git a/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeTask.groovy b/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeTask.groovy index e8c61d1..a407cf3 100644 --- a/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeTask.groovy +++ b/plugin/src/main/groovy/io/github/stewartbryson/SnowflakeTask.groovy @@ -25,21 +25,21 @@ abstract class SnowflakeTask extends DefaultTask { } /** - * The SnowSQL config file to use. Default: Looks first for '~/.snowsql/config' followed by './snow-config'. + * The credentials config file to use. Default: Looks first for '~/.snowflake/config.toml' followed by '~/.snowsql/config'. */ @Input @Optional - @Option(option = "snow-config", - description = "Custom SnowSQL config file." + @Option(option = "config", + description = "Custom credentials config file." ) String snowConfig /** - * Override the SnowSQL connection to use. Default: use the base connection info in SnowSQL config. + * Override the credentials connection to use. Default: use the base connection info in credentials config. */ @Input @Option(option = "connection", - description = "Override the SnowSQL connection to use. Default: use the base connection info in SnowSQL config." + description = "Override the credentials connection to use. Default: use the base connection info in credentials config." ) String connection = extension.connection diff --git a/plugin/src/test/groovy/io/github/stewartbryson/SnowConfigTest.groovy b/plugin/src/test/groovy/io/github/stewartbryson/SnowConfigTest.groovy index beec82f..3504542 100644 --- a/plugin/src/test/groovy/io/github/stewartbryson/SnowConfigTest.groovy +++ b/plugin/src/test/groovy/io/github/stewartbryson/SnowConfigTest.groovy @@ -34,7 +34,7 @@ class SnowConfigTest extends Specification { snow = new SnowConfig(config, "example") } - def "Parse snowsql default connection"() { + def "Parse credentials default connection"() { when: def props = snow.getConnectionsProps() @@ -46,7 +46,7 @@ class SnowConfigTest extends Specification { props.password == 'defaultpassword' } - def "Parse snowsql connection with quoted password"() { + def "Parse credentials connection with quoted password"() { given: config.append(""" | diff --git a/src/examples/groovy/build.gradle b/src/examples/groovy/build.gradle index 7dd8b24..e208928 100644 --- a/src/examples/groovy/build.gradle +++ b/src/examples/groovy/build.gradle @@ -9,7 +9,7 @@ repositories { } dependencies { - implementation 'org.codehaus.groovy:groovy:3.0.17' + implementation 'org.codehaus.groovy:groovy:3.0.18' } java { diff --git a/src/examples/java-testing/build.gradle b/src/examples/java-testing/build.gradle index 6f07338..673310e 100644 --- a/src/examples/java-testing/build.gradle +++ b/src/examples/java-testing/build.gradle @@ -47,7 +47,7 @@ testing { } testTask.configure { failFast true - // which SnowSQL connection to use + // which credentials connection to use systemProperty 'connection', snowflake.connection // if this is ephemeral, the test spec needs the name to connect to if (snowflake.useEphemeral) { diff --git a/src/markdown/README.md b/src/markdown/README.md index 43dbee2..9199351 100644 --- a/src/markdown/README.md +++ b/src/markdown/README.md @@ -1,26 +1,8 @@ -# Breaking Changes - -We introduced the following breaking changes in version `2.0.0`: - -### SnowSQL Config File -Instead of continuing to use plugin DSL or Gradle properties to provide Snowflake authentication, we made the -decision to switch to using the SnowSQL config moving forward. -This was inspired by the [Snowflake Developer CLI](https://github.com/Snowflake-Labs/snowcli) project, and it seems to -be a reasonable standard. - -### Legacy Plugin Application -Hopefully no one was using the Gradle legacy plugin application, but if so, the coordinates have changed. -You can always get the most recent coordinates on -the [Gradle Plugin Portal](https://plugins.gradle.org/plugin/io.github.stewartbryson.snowflake). - -### Ephemeral Cloning for Functional Testing -Initially, the ephemeral cloning functionality was only present for the `snowflakeJvm` task, with the clone being -created (and possibly dropped) as part of that task. -To support the new functional testing feature (described below), ephemeral cloning was moved out into the `createEphemeral` -and `dropEphemeral` tasks that are now added at the beginning and end of the build, respectively. -This allows for the `functionalTest` task (if applied) to be run in the ephemeral clone just after `snowflakeJvm`. -As described below, cloning tasks are automatically managed with the `useEphemeral`, `keepEphemeral` -and `ephemeralName` properties as before, but the task options on `snowflakeJvm` have been removed. +# Recent changes + +### snowcli `config.toml` support +The [Snowflake Developer CLI](https://github.com/Snowflake-Labs/snowcli) project recently introduced the `~/.snowflake/config.toml` for credentials. +We are mirroring that support by first looking for `~/.snowflake/config.toml` and then looking for `~/.snowsql/config`. # Motivation @@ -97,24 +79,24 @@ Type SnowflakeJvm (io.github.stewartbryson.SnowflakeJvm) Options - --connection Override the SnowSQL connection to use. Default: use the base connection info in SnowSQL config. + --config Custom credentials config file. - --jar Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata. + --connection Override the credentials connection to use. Default: use the base connection info in credentials config. - --snow-config Custom SnowSQL config file. + --jar Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata. --stage Override the Snowflake stage to publish to. --rerun Causes the task to be re-run even if up-to-date. Description - A Cacheable Gradle task for publishing UDFs and procedures to Snowflake. + A Cacheable Gradle task for publishing UDFs and procedures to Snowflake Group publishing -BUILD SUCCESSFUL in 1s -5 actionable tasks: 1 executed, 4 up-to-date +BUILD SUCCESSFUL in 2s +5 actionable tasks: 3 executed, 2 up-to-date ``` Several command-line options mention _overriding_ other configuration values. @@ -136,18 +118,18 @@ snowflake { } ``` -Beginning in version `2.0.0`, the plugin uses -the [SnowSQL config](https://docs.snowflake.com/en/user-guide/snowsql-config) file, a choice that was inspired by -the [Snowflake Developer CLI](https://github.com/Snowflake-Labs/snowcli) project. -The `connection` property in the plugin DSL defines which connection from the config file to use, relying on the default values if none is -provided. -It first loads all the default values, and replaces any values from the connection, similar to how SnowSQL works. -Unfortunately, it doesn't yet look for a config file in all the places that SnowSQL does. -Instead, it looks in this order: +Snowflake ceedentials are managed in a config file, with the default being `~/.snowflake/config.toml` as prescribed by the [Snowflake Developer CLI](https://github.com/Snowflake-Labs/snowcli) project. +As a secondary location, we also support the [SnowSQL config](https://docs.snowflake.com/en/user-guide/snowsql-config) file. +In searching for a credentials config file, the plugin works in the following order: + +1. A custom location of your choosing, configured with the `--config` option in applicable tasks. +2. `/.snowflake/config.toml` +3. `/.snowsql/config` +4. `./config.toml` (This is useful in CI/CD pipelines, where secrets can be written easily to this file.) -1. A custom location of your choosing, configured with the `--snow-config` option in applicable tasks. -2. `/.snowsql/config` -3. `./snow-config` (This is useful in CI/CD pipelines, where secrets can easily be written to this file.) +The `connection` property in the plugin DSL defines which connection to use from the config file, relying on the default values if none is +provided. +It first loads all the default values, and replaces any values from the connection, similar to how Snowflake CLI and SnowSQL works. The nested [`applications` DSL](https://s3.amazonaws.com/stewartbryson.docs/gradle-snowflake/latest/io/github/stewartbryson/ApplicationContainer.html) @@ -174,7 +156,7 @@ Note that if the named internal stage does not exist, the plugin will create it ❯ ./gradlew snowflakeJvm > Task :snowflakeJvm -Using snowsql config file: /Users/stewartbryson/.snowsql/config +Using credentials config file: /Users/stewartbryson/.snowflake/config.toml File java-0.1.0-all.jar: UPLOADED Deploying ==> CREATE OR REPLACE function add_numbers (a integer, b integer) @@ -184,8 +166,8 @@ CREATE OR REPLACE function add_numbers (a integer, b integer) imports = ('@upload/libs/java-0.1.0-all.jar') -BUILD SUCCESSFUL in 4s -7 actionable tasks: 7 executed +BUILD SUCCESSFUL in 5s +7 actionable tasks: 2 executed, 5 up-to-date ``` Our function now exists in Snowflake: