Skip to content

Commit

Permalink
Merge pull request #89 from stewartbryson/stewart
Browse files Browse the repository at this point in the history 
Stewart
  • Loading branch information
@stewartbryson
stewartbryson authored Apr 12, 2023
2 parents fed53bb + 4e30e72 commit 0631bcd
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 115 deletions.
3 changes: 1 addition & 2 deletions plugin/src/main/groovy/io/github/stewartbryson/SnowflakeExtension.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,9 @@ class SnowflakeExtension {
/**
* The name of the cloned Snowflake database. Default: auto-generated.
*
* Dynamically generated name for an ephemeral Snowflake clone to create. Uses CICD properties when they are available, and a simple unique name when they are not.
* The autogenerated value is always prefixed with 'EPHEMERAL_<gradle project name>_', followed by pull request, tag or branch information when running in CICD environemnts. Otherwise, the suffix is autogenerated.
*/
String ephemeralName

/**
* Return the name of the Maven publication task associated with the external stage.
*/
Expand Down
224 changes: 112 additions & 112 deletions plugin/src/main/groovy/io/github/stewartbryson/SnowflakeJvm.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,129 +22,129 @@ import java.sql.Statement
@CacheableTask
abstract class SnowflakeJvm extends SnowflakeTask {

/**
* The task Constructor with 'description' and 'group'.
*
* @return A custom task class.
*/
SnowflakeJvm() {
description = "A Cacheable Gradle task for publishing UDFs and procedures to Snowflake"
group = "publishing"
}
/**
* The task Constructor with 'description' and 'group'.
*
* @return A custom task class.
*/
SnowflakeJvm() {
description = "A Cacheable Gradle task for publishing UDFs and procedures to Snowflake"
group = "publishing"
}

/**
* The Snowflake stage to publish to. Overrides {@link SnowflakeExtension#stage}.
*/
@Optional
@Input
@Option(option = "stage",
description = "Override the Snowflake stage to publish to."
)
String stage = extension.stage
/**
* The Snowflake stage to publish to. Overrides {@link SnowflakeExtension#stage}.
*/
@Optional
@Input
@Option(option = "stage",
description = "Override the Snowflake stage to publish to."
)
String stage = extension.stage

/**
* Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.
*/
@Optional
@Input
@Option(option = "jar", description = "Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.")
String jar = project.tasks.shadowJar.archiveFile.get()
/**
* Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.
*/
@Optional
@Input
@Option(option = "jar", description = "Optional: manually pass a JAR file path to upload instead of relying on Gradle metadata.")
String jar = project.tasks.shadowJar.archiveFile.get()

/**
* A simple text output file for the Snowflake applications create statements. Makes the class Cacheable.
*/
@OutputFile
File output = project.file("${project.buildDir}/${PLUGIN}/output.txt")
/**
* A simple text output file for the Snowflake applications create statements. Makes the class Cacheable.
*/
@OutputFile
File output = project.file("${project.buildDir}/${PLUGIN}/output.txt")

/**
* Get the 'import' property for the UDF.
*
* @return the 'import' property.
*/
@Internal
String getImports() {
/**
* Get the 'import' property for the UDF.
*
* @return the 'import' property.
*/
@Internal
String getImports() {

String basePath = "@${stage}/${extension.groupId.replace('.', '/')}/${extension.artifactId}/${project.version}"
//log.warn "basePath: $basePath"
Statement statement = snowflake.session.jdbcConnection().createStatement()
String sql = "LIST $basePath pattern='(.)*(-all)\\.jar'; select * from table(result_scan(last_query_id())) order by 'last_modified' asc;"
statement.unwrap(SnowflakeStatement.class).setParameter(
"MULTI_STATEMENT_COUNT", 2)
ResultSet rs = statement.executeQuery(sql)
String fileName
String filePath
try {
while (rs.next()) {
filePath = rs.getString(1)
}
fileName = filePath.replaceAll(/(.*)($project.version)(\/)(.*)/) { all, first, version, slash, filename ->
filename
}
} catch (Exception e) {
throw new Exception("Unable to detect the correct JAR in stage '${stage}'.")
}
rs.close()
statement.close()
"'$basePath/$fileName'"
}
String basePath = "@${stage}/${extension.groupId.replace('.', '/')}/${extension.artifactId}/${project.version}"
//log.warn "basePath: $basePath"
Statement statement = snowflake.session.jdbcConnection().createStatement()
String sql = "LIST $basePath pattern='(.)*(-all)\\.jar'; select * from table(result_scan(last_query_id())) order by 'last_modified' asc;"
statement.unwrap(SnowflakeStatement.class).setParameter(
"MULTI_STATEMENT_COUNT", 2)
ResultSet rs = statement.executeQuery(sql)
String fileName
String filePath
try {
while (rs.next()) {
filePath = rs.getString(1)
}
fileName = filePath.replaceAll(/(.*)($project.version)(\/)(.*)/) { all, first, version, slash, filename ->
filename
}
} catch (Exception e) {
throw new Exception("Unable to detect the correct JAR in stage '${stage}'.")
}
rs.close()
statement.close()
"'$basePath/$fileName'"
}

/**
* The Gradle TaskAction method. Publish the Snowflake Application.
*/
@TaskAction
def publish() {
// create the session
createSession()
if (extension.useEphemeral) {
snowflake.ephemeral = extension.ephemeralName
snowflake.setEphemeralContext()
}
/**
* The Gradle TaskAction method. Publish the Snowflake Application.
*/
@TaskAction
def publish() {
// create the session
createSession()
if (extension.useEphemeral) {
snowflake.ephemeral = extension.ephemeralName
snowflake.setEphemeralContext()
}

String jar = project.tasks.shadowJar.archiveFile.get()
log.info "Jar to upload: $jar"
String jar = project.tasks.shadowJar.archiveFile.get()
log.info "Jar to upload: $jar"

if (!extension.publishUrl && !extension.useCustomMaven) {
// create the internal stage if it doesn't exist
snowflake.session.jdbcConnection().createStatement().execute("create stage if not exists ${stage}")
if (!extension.publishUrl && !extension.useCustomMaven) {
// create the internal stage if it doesn't exist
snowflake.session.jdbcConnection().createStatement().execute("create stage if not exists ${stage}")

//
def options = [
AUTO_COMPRESS: 'FALSE',
PARALLEL : '4',
OVERWRITE : 'TRUE'
]
try {
PutResult[] pr = snowflake.session.file().put(jar, "$stage/libs", options)
pr.each {
log.warn "File ${it.sourceFileName}: ${it.status}"
}
} catch (SnowflakeSQLException e) {
// this tells us there's misconfiguration
// we are using an external stage without setting publishUrl
if (e.message.contains("GET and PUT commands are not supported with external stage")) {
throw new Exception("Using an external stage requires setting the 'publishUrl' property.")
}
//
def options = [
AUTO_COMPRESS: 'FALSE',
PARALLEL : '4',
OVERWRITE : 'TRUE'
]
try {
PutResult[] pr = snowflake.session.file().put(jar, "$stage/libs", options)
pr.each {
log.warn "File ${it.sourceFileName}: ${it.status}"
}
} else if (extension.publishUrl) {
// ensure that the stage and the publishUrl are aligned
String selectStage = snowflake.getScalarValue("select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$snowflake.connectionSchema') and stage_type='External Named'")
if( selectStage != extension.publishUrl) {
throw new Exception("'publishUrl' does not match the external stage URL in Snowflake.")
} catch (SnowflakeSQLException e) {
// this tells us there's misconfiguration
// we are using an external stage without setting publishUrl
if (e.message.contains("GET and PUT commands are not supported with external stage")) {
throw new Exception("Using an external stage requires setting the 'publishUrl' property.")
}
}
}
} else if (extension.publishUrl) {
// ensure that the stage and the publishUrl are aligned
String selectStage = snowflake.getScalarValue("select stage_url from information_schema.stages where stage_name=upper('$stage') and stage_schema=upper('$snowflake.connectionSchema') and stage_type='External Named'")
if (selectStage != extension.publishUrl) {
throw new Exception("'publishUrl' does not match the external stage URL in Snowflake.")
}
}

// automatically create application spec objects
output.write("Snowflake Application:\n\n")
project."$PLUGIN".applications.each { ApplicationContainer app ->
File jarFile = project.file(jar)
String createText = app.getCreate(extension.publishUrl ? getImports() : "'@$stage/libs/${jarFile.name}'")
String message = "Deploying ==> \n$createText"
log.warn message
output.append("$message\n")
snowflake.session.jdbcConnection().createStatement().execute(createText)
}
// automatically create application spec objects
output.write("Snowflake Application:\n\n")
project."$PLUGIN".applications.each { ApplicationContainer app ->
File jarFile = project.file(jar)
String createText = app.getCreate(extension.publishUrl ? getImports() : "'@$stage/libs/${jarFile.name}'")
String message = "Deploying ==> \n$createText"
log.warn message
output.append("$message\n")
snowflake.session.jdbcConnection().createStatement().execute(createText)
}

// close the session
//snowflake.session.close()
}
// close the session
//snowflake.session.close()
}
}
9 changes: 8 additions & 1 deletion plugin/src/main/groovy/io/github/stewartbryson/SnowflakePlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ class SnowflakePlugin implements Plugin<Project> {

// add shadowJar to build
project.build.dependsOn project.shadowJar
// exclude some things in shadowJar
project.shadowJar {
dependencies {
exclude(dependency('com.snowflake:snowpark:.*'))
}
}

if (extension.useCustomMaven || extension.publishUrl) {
// assert that we have artifact and group
Expand Down Expand Up @@ -107,7 +113,8 @@ class SnowflakePlugin implements Plugin<Project> {
if (extension.useEphemeral) {
project.tasks.snowflakeJvm.configure {
// snowflakeJvm should always run when using ephemeral clones
// that's because the clone is dropped at the end
// that's because the clone may be dropped at the end of the last run
//TODO #88
outputs.upToDateWhen {false}
// clone should be created before publishing
dependsOn project.tasks.createEphemeral
Expand Down

0 comments on commit 0631bcd

Please sign in to comment.