From 57b58b82ab2e8a5f2955b059cc3b1e4725d3914f Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle <54168385+mgabelle@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:56:29 +0100 Subject: [PATCH] refactor: migrate to dynamic properties (#178) --- .../plugin/googleworkspace/AbstractTask.java | 15 ++-- .../plugin/googleworkspace/GcpInterface.java | 11 ++- .../calendar/AbstractCalendar.java | 4 +- .../calendar/AbstractInsertEvent.java | 42 ++++++----- .../googleworkspace/calendar/InsertEvent.java | 5 +- .../googleworkspace/drive/AbstractCreate.java | 24 +++---- .../googleworkspace/drive/AbstractDrive.java | 4 +- .../plugin/googleworkspace/drive/Delete.java | 6 +- .../googleworkspace/drive/Download.java | 6 +- .../plugin/googleworkspace/drive/Export.java | 11 ++- .../plugin/googleworkspace/drive/List.java | 13 ++-- .../plugin/googleworkspace/drive/Upload.java | 18 +++-- .../googleworkspace/sheets/AbstractLoad.java | 36 +++++----- .../googleworkspace/sheets/AbstractRead.java | 24 +++---- .../googleworkspace/sheets/AbstractSheet.java | 4 +- .../sheets/CreateSpreadsheet.java | 10 +-- .../googleworkspace/sheets/DataParser.java | 8 +-- .../sheets/DeleteSpreadsheet.java | 14 ++-- .../plugin/googleworkspace/sheets/Load.java | 12 ++-- .../plugin/googleworkspace/sheets/Read.java | 20 +++--- .../googleworkspace/sheets/ReadRange.java | 15 ++-- .../calendar/InsertEventTest.java | 15 ++-- .../googleworkspace/drive/ListTest.java | 5 +- .../googleworkspace/drive/SuiteTest.java | 71 ++++++++++--------- .../sheets/CreateSpreadsheetTest.java | 9 +-- .../sheets/DeleteSpreadsheetTest.java | 13 ++-- .../googleworkspace/sheets/LoadTest.java | 71 ++++++++++--------- .../googleworkspace/sheets/ReadRangeTest.java | 15 ++-- .../googleworkspace/sheets/ReadTest.java | 15 ++-- 29 files changed, 258 insertions(+), 258 deletions(-) diff --git a/src/main/java/io/kestra/plugin/googleworkspace/AbstractTask.java b/src/main/java/io/kestra/plugin/googleworkspace/AbstractTask.java index a22df0b..699ed06 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/AbstractTask.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/AbstractTask.java @@ -10,6 +10,7 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; import io.kestra.core.exceptions.IllegalVariableEvaluationException; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.Task; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -30,17 +31,17 @@ public abstract class AbstractTask extends Task implements GcpInterface { protected static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - protected String serviceAccount; + protected Property serviceAccount; @Builder.Default - protected Integer readTimeout = 120; + protected Property readTimeout = Property.of(120); protected HttpCredentialsAdapter credentials(RunContext runContext) throws IllegalVariableEvaluationException, IOException { GoogleCredentials credentials; if (serviceAccount != null) { - String serviceAccount = runContext.render(this.serviceAccount); + String serviceAccount = runContext.render(this.serviceAccount).as(String.class).orElseThrow(); ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(serviceAccount.getBytes()); credentials = ServiceAccountCredentials.fromStream(byteArrayInputStream); Logger logger = runContext.logger(); @@ -56,16 +57,18 @@ protected HttpCredentialsAdapter credentials(RunContext runContext) throws Illeg credentials = GoogleCredentials.getApplicationDefault(); } - if (this.getScopes() != null) { - credentials = credentials.createScoped(runContext.render(this.getScopes())); + var renderedScopes = runContext.render(this.getScopes()).asList(String.class); + if (!renderedScopes.isEmpty()) { + credentials = credentials.createScoped(renderedScopes); } + var renderedTiemout = runContext.render(this.readTimeout).as(Integer.class).orElseThrow(); return new HttpCredentialsAdapter(credentials) { @Override public void initialize(HttpRequest request) throws IOException { super.initialize(request); - request.setReadTimeout(readTimeout * 1000); + request.setReadTimeout(renderedTiemout * 1000); } }; } diff --git a/src/main/java/io/kestra/plugin/googleworkspace/GcpInterface.java b/src/main/java/io/kestra/plugin/googleworkspace/GcpInterface.java index 73ab129..3cc96bb 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/GcpInterface.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/GcpInterface.java @@ -1,6 +1,6 @@ package io.kestra.plugin.googleworkspace; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.swagger.v3.oas.annotations.media.Schema; import java.util.List; @@ -9,18 +9,15 @@ public interface GcpInterface { @Schema( title = "The GCP service account key" ) - @PluginProperty(dynamic = true) - String getServiceAccount(); + Property getServiceAccount(); @Schema( title = "The GCP scopes to used" ) - @PluginProperty(dynamic = true) - List getScopes(); + Property> getScopes(); @Schema( title = "The read timeout for the request (in seconds)" ) - @PluginProperty(dynamic = true) - Integer getReadTimeout(); + Property getReadTimeout(); } diff --git a/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractCalendar.java b/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractCalendar.java index de88674..3151aff 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractCalendar.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractCalendar.java @@ -5,6 +5,7 @@ import com.google.auth.http.HttpCredentialsAdapter; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.plugin.googleworkspace.AbstractTask; import lombok.*; @@ -21,8 +22,7 @@ @NoArgsConstructor public abstract class AbstractCalendar extends AbstractTask { @Builder.Default - @PluginProperty(dynamic = true) - protected List scopes = List.of(CalendarScopes.CALENDAR); + protected Property> scopes = Property.of(List.of(CalendarScopes.CALENDAR)); protected Calendar connection(RunContext runContext) throws IllegalVariableEvaluationException, IOException, GeneralSecurityException { HttpCredentialsAdapter credentials = this.credentials(runContext); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractInsertEvent.java b/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractInsertEvent.java index 6b46f26..ed10432 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractInsertEvent.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/calendar/AbstractInsertEvent.java @@ -2,6 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotNull; @@ -28,15 +29,13 @@ public abstract class AbstractInsertEvent extends AbstractCalendar { title = "Calendar ID." ) @NotNull - @PluginProperty(dynamic = true) - protected String calendarId; + protected Property calendarId; @Schema( title = "Title of the event." ) @NotNull - @PluginProperty(dynamic = true) - protected String summary; + protected Property summary; @Schema( title = "Description of the event." @@ -47,8 +46,7 @@ public abstract class AbstractInsertEvent extends AbstractCalendar { @Schema( title = "Geographic location of the event as free-form text." ) - @PluginProperty(dynamic = true) - protected String location; + protected Property location; @Schema( title = "Start time of the event." @@ -86,14 +84,12 @@ public static class CalendarTime { @Schema( title = "Time of the event in the ISO 8601 Datetime format, for example, `2024-11-28T09:00:00-07:00`." ) - @PluginProperty(dynamic = true) - protected String dateTime; - + protected Property dateTime; + @Schema( title = "Timezone associated with the dateTime, for example, `America/Los_Angeles`." ) - @PluginProperty(dynamic = true) - protected String timeZone; + protected Property timeZone; } @Builder @@ -106,45 +102,47 @@ public static class Attendee { @Schema( title = "Display name of the attendee." ) - @PluginProperty(dynamic = true) - protected String displayName; - + protected Property displayName; + @Schema( title = "Email of the attendee." ) - @PluginProperty(dynamic = true) - protected String email; + protected Property email; } protected Event event(RunContext runContext) throws IllegalVariableEvaluationException { Event eventMetadata = new Event(); - eventMetadata.setSummary(runContext.render(this.summary)); + eventMetadata.setSummary(runContext.render(this.summary).as(String.class).orElseThrow()); if (this.description != null) { eventMetadata.setDescription(runContext.render(this.description)); } if (this.location != null) { - eventMetadata.setLocation(runContext.render(this.location)); + eventMetadata.setLocation(runContext.render(this.location).as(String.class).orElseThrow()); } - EventDateTime eventStartTime = new EventDateTime().setDateTime(new DateTime(runContext.render(startTime.dateTime))).setTimeZone(runContext.render(startTime.timeZone)); + EventDateTime eventStartTime = new EventDateTime().setDateTime(new DateTime(runContext.render(startTime.dateTime).as(String.class).orElse(null))) + .setTimeZone(runContext.render(startTime.timeZone).as(String.class).orElse(null)); eventMetadata.setStart(eventStartTime); - EventDateTime eventEndTime = new EventDateTime().setDateTime(new DateTime(runContext.render(endTime.dateTime))).setTimeZone(runContext.render(endTime.timeZone)); + EventDateTime eventEndTime = new EventDateTime().setDateTime(new DateTime(runContext.render(endTime.dateTime).as(String.class).orElse(null))) + .setTimeZone(runContext.render(endTime.timeZone).as(String.class).orElse(null)); eventMetadata.setEnd(eventEndTime); if (attendees != null && attendees.size() > 0) { List eventAttendees = new ArrayList<>(); for (Attendee attendee: attendees){ - EventAttendee eventAttendee = new EventAttendee().setDisplayName(runContext.render(attendee.displayName)).setEmail(runContext.render(attendee.email)); + EventAttendee eventAttendee = new EventAttendee().setDisplayName(runContext.render(attendee.displayName).as(String.class).orElse(null)) + .setEmail(runContext.render(attendee.email).as(String.class).orElse(null)); eventAttendees.add(eventAttendee); } eventMetadata.setAttendees(eventAttendees); } if (creator != null) { - Creator eventCreator = new Event.Creator().setDisplayName(runContext.render(creator.displayName)).setEmail(runContext.render(creator.email)); + Creator eventCreator = new Creator().setDisplayName(runContext.render(creator.displayName).as(String.class).orElse(null)) + .setEmail(runContext.render(creator.email).as(String.class).orElse(null)); eventMetadata.setCreator(eventCreator); } diff --git a/src/main/java/io/kestra/plugin/googleworkspace/calendar/InsertEvent.java b/src/main/java/io/kestra/plugin/googleworkspace/calendar/InsertEvent.java index 5e435ff..3ca3f02 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/calendar/InsertEvent.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/calendar/InsertEvent.java @@ -57,13 +57,14 @@ public Output run(RunContext runContext) throws Exception { Event eventMetadata = event(runContext); + var renderedCalendarId= runContext.render(calendarId).as(String.class).orElseThrow(); Event event = service .events() - .insert(calendarId, eventMetadata) + .insert(renderedCalendarId, eventMetadata) .setFields("id") .execute(); - logger.debug("Inserted event '{}' in calendar '{}'", event.getId(), calendarId); + logger.debug("Inserted event '{}' in calendar '{}'", event.getId(), renderedCalendarId); return Output .builder() diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractCreate.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractCreate.java index e9996a9..c29ad13 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractCreate.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractCreate.java @@ -3,6 +3,7 @@ import com.google.api.services.drive.model.File; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -22,15 +23,13 @@ public abstract class AbstractCreate extends AbstractDrive { @Schema( title = "The destination path" ) - @PluginProperty(dynamic = true) - protected List parents; + protected Property> parents; @Schema( title = "The name of the file", description = "This is not necessarily unique within a folder" ) - @PluginProperty(dynamic = true) - protected String name; + protected Property name; @Schema( title = "A short description of the file." @@ -45,32 +44,31 @@ public abstract class AbstractCreate extends AbstractDrive { "with a Google Doc MIME type, the uploaded content will be imported if possible. " + "The supported import formats are published [here](https://developers.google.com/drive/api/v3/mime-types)." ) - @PluginProperty(dynamic = true) - protected String mimeType; + protected Property mimeType; @Schema( title = "ID of the Team Drive the file resides in." ) - @PluginProperty(dynamic = true) - protected String teamDriveId; + protected Property teamDriveId; protected File file(RunContext runContext) throws IllegalVariableEvaluationException { File fileMetadata = new File(); if (this.name != null) { - fileMetadata.setName(runContext.render(this.name)); + fileMetadata.setName(runContext.render(this.name).as(String.class).orElseThrow()); } - if (this.parents != null) { - fileMetadata.setParents(runContext.render(this.parents)); + var renderedParents = runContext.render(this.parents).asList(String.class); + if (!renderedParents.isEmpty()) { + fileMetadata.setParents(renderedParents); } if (mimeType != null) { - fileMetadata.setMimeType(runContext.render(mimeType)); + fileMetadata.setMimeType(runContext.render(mimeType).as(String.class).orElseThrow()); } if (teamDriveId != null) { - fileMetadata.setTeamDriveId(runContext.render(teamDriveId)); + fileMetadata.setTeamDriveId(runContext.render(teamDriveId).as(String.class).orElseThrow()); } if (description != null) { diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractDrive.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractDrive.java index e96e284..648e017 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractDrive.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/AbstractDrive.java @@ -5,6 +5,7 @@ import com.google.auth.http.HttpCredentialsAdapter; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.plugin.googleworkspace.AbstractTask; import lombok.*; @@ -21,8 +22,7 @@ @NoArgsConstructor public abstract class AbstractDrive extends AbstractTask { @Builder.Default - @PluginProperty(dynamic = true) - protected List scopes = List.of(DriveScopes.DRIVE); + protected Property> scopes = Property.of(List.of(DriveScopes.DRIVE)); Drive connection(RunContext runContext) throws IllegalVariableEvaluationException, IOException, GeneralSecurityException { HttpCredentialsAdapter credentials = this.credentials(runContext); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/Delete.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/Delete.java index cb6ceac..5cecc9a 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/Delete.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/Delete.java @@ -4,6 +4,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -41,14 +42,13 @@ public class Delete extends AbstractDrive implements RunnableTask @Schema( title = "The file id to delete" ) - @PluginProperty(dynamic = true) - private String fileId; + private Property fileId; @Override public Output run(RunContext runContext) throws Exception { Drive service = this.connection(runContext); Logger logger = runContext.logger(); - String id = runContext.render(this.fileId); + String id = runContext.render(this.fileId).as(String.class).orElse(null); Void execute = service .files() diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/Download.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/Download.java index 6f6dab1..182def6 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/Download.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/Download.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -44,15 +45,14 @@ public class Download extends AbstractDrive implements RunnableTask fileId; @Override public Output run(RunContext runContext) throws Exception { Drive service = this.connection(runContext); Logger logger = runContext.logger(); - String fileId = runContext.render(this.fileId); + String fileId = runContext.render(this.fileId).as(String.class).orElseThrow(); File tempFile = runContext.workingDir().createTempFile().toFile(); try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) { diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/Export.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/Export.java index a27020d..f90fe97 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/Export.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/Export.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -46,23 +47,21 @@ public class Export extends AbstractDrive implements RunnableTask @Schema( title = "The file id to copy" ) - @PluginProperty(dynamic = true) @NotNull - private String fileId; + private Property fileId; @Schema( title = "The content-type of the file.", description = "a valid [RFC2045](https://datatracker.ietf.org/doc/html/rfc2045) like `text/csv`, `application/msword`, ... " ) - @PluginProperty(dynamic = true) @NotNull - private String contentType; + private Property contentType; @Override public Output run(RunContext runContext) throws Exception { Drive service = this.connection(runContext); Logger logger = runContext.logger(); - String fileId = runContext.render(this.fileId); + String fileId = runContext.render(this.fileId).as(String.class).orElseThrow(); File tempFile = runContext.workingDir().createTempFile().toFile(); @@ -73,7 +72,7 @@ public Output run(RunContext runContext) throws Exception { .setFields("id, name, size, version, createdTime, parents, trashed, mimeType") .setSupportsTeamDrives(true) .execute(); - Drive.Files.Export export = service.files().export(fileId, runContext.render(contentType)); + Drive.Files.Export export = service.files().export(fileId, runContext.render(contentType).as(String.class).orElseThrow()); export.executeMediaAndDownloadTo(outputStream); outputStream.flush(); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/List.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/List.java index a73039e..a4d602a 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/List.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/List.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.googleworkspace.drive.models.File; @@ -36,7 +37,7 @@ - id: list type: io.kestra.plugin.googleworkspace.drive.List query: | - mimeType = 'application/vnd.google-apps.folder' + mimeType = 'application/vnd.google-apps.folder' and '1z2GZgLEX12BN9zbVE6TodrCHyTRMj_ka' in parents """ ) @@ -51,8 +52,7 @@ public class List extends AbstractDrive implements RunnableTask { description = "see details [here](https://developers.google.com/drive/api/v3/search-files)\n" + "if not defined, will list all files that the service account have access" ) - @PluginProperty(dynamic = true) - private String query; + private Property query; @Schema( title = "list of bodies of items (files/documents) to which the query applies.", @@ -60,15 +60,14 @@ public class List extends AbstractDrive implements RunnableTask { " be combined with 'user'; all other values must be used in isolation. Prefer 'user' or 'teamDrive' " + "to 'allTeamDrives' for efficiency." ) - @PluginProperty(dynamic = false) - private java.util.List corpora; + private Property> corpora; @Override public Output run(RunContext runContext) throws Exception { Drive service = this.connection(runContext); Logger logger = runContext.logger(); - String query = this.query != null ? runContext.render(this.query) : null; + String query = this.query != null ? runContext.render(this.query).as(String.class).orElseThrow() : null; Drive.Files.List list = service.files() .list() @@ -76,7 +75,7 @@ public Output run(RunContext runContext) throws Exception { .setQ(query); if (this.corpora != null) { - list.setCorpora(this.corpora + list.setCorpora(runContext.render(this.corpora).asList(Corpora.class) .stream() .map(Enum::name) .collect(Collectors.joining(",")) diff --git a/src/main/java/io/kestra/plugin/googleworkspace/drive/Upload.java b/src/main/java/io/kestra/plugin/googleworkspace/drive/Upload.java index c68793f..197f46c 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/drive/Upload.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/drive/Upload.java @@ -7,6 +7,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -56,32 +57,29 @@ ) public class Upload extends AbstractCreate implements RunnableTask { @Schema( - title = "The file to copy" + title = "The file URI to copy" ) - @PluginProperty(dynamic = true) @NotNull - private String from; + private Property from; @Schema( title = "The file id to update", description = "If not provided, it will create a new file" ) - @PluginProperty(dynamic = true) - private String fileId; + private Property fileId; @Schema( title = "The content-type of the file.", description = "a valid [RFC2045](https://datatracker.ietf.org/doc/html/rfc2045) like `text/csv`, `application/msword`, ... " ) - @PluginProperty(dynamic = true) @NotNull - private String contentType; + private Property contentType; @Override public Output run(RunContext runContext) throws Exception { Drive service = this.connection(runContext); Logger logger = runContext.logger(); - URI from = URI.create(runContext.render(this.from)); + URI from = URI.create(runContext.render(this.from).as(String.class).orElseThrow()); File fileMetadata = this.file(runContext); @@ -90,13 +88,13 @@ public Output run(RunContext runContext) throws Exception { IOUtils.copy(runContext.storage().getFile(from), out); } - FileContent fileContent = new FileContent(runContext.render(contentType), tempFile); + FileContent fileContent = new FileContent(runContext.render(contentType).as(String.class).orElseThrow(), tempFile); File file; if (this.fileId != null) { file = service .files() - .update(runContext.render(this.fileId), fileMetadata, fileContent) + .update(runContext.render(this.fileId).as(String.class).orElseThrow(), fileMetadata, fileContent) .setFields("id, name, version, createdTime, parents, trashed, mimeType") .setSupportsTeamDrives(true) .execute(); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractLoad.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractLoad.java index dce19de..c66577f 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractLoad.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractLoad.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; import io.swagger.v3.oas.annotations.media.Schema; @@ -29,14 +30,13 @@ public abstract class AbstractLoad extends AbstractSheet { title = "The spreadsheet unique id." ) @NotNull - @PluginProperty(dynamic = true) - protected String spreadsheetId; + protected Property spreadsheetId; @Builder.Default @Schema( title = "Specifies if the first line should be the header (default: false)." ) - protected final Boolean header = false; + protected final Property header = Property.of(false); @Schema( title = "Csv parsing options (Optional)." @@ -48,33 +48,32 @@ public abstract class AbstractLoad extends AbstractSheet { title = "Schema for avro objects (Optional).", description = "If provided, the task will read avro objects using this schema." ) - @PluginProperty(dynamic = true) - private String avroSchema; + private Property avroSchema; @Schema( title = "Format of the input file.", description = "If not provided, the task will programmatically try to find the correct format based on the extension." ) - @PluginProperty(dynamic = true) - private Format format; + private Property format; protected List> parse(RunContext runContext, URI from) throws Exception { Format format; if (this.format == null) { format = Format.getFromFile(from.toString()); } else { - format = this.format; + format = runContext.render(this.format).as(Format.class).orElseThrow(); } try (InputStream inputStream = runContext.storage().getFile(from)) { DataParser parser = new DataParser(runContext); + var headerValue = runContext.render(header).as(Boolean.class).orElseThrow(); return switch (format) { - case ION -> parser.parseThroughMapper(inputStream, ION_MAPPER, header); - case JSON -> parser.parseThroughMapper(inputStream, JSON_MAPPER, header); + case ION -> parser.parseThroughMapper(inputStream, ION_MAPPER, headerValue); + case JSON -> parser.parseThroughMapper(inputStream, JSON_MAPPER, headerValue); case CSV -> parser.parseCsv(inputStream, csvOptions); - case AVRO -> parser.parseAvro(inputStream, header, runContext.render(this.avroSchema)); - case PARQUET -> parser.parseParquet(inputStream, header); - case ORC -> parser.parseORC(inputStream, header); + case AVRO -> parser.parseAvro(inputStream, headerValue, runContext.render(this.avroSchema).as(String.class).orElse(null)); + case PARQUET -> parser.parseParquet(inputStream, headerValue); + case ORC -> parser.parseORC(inputStream, headerValue); }; } } @@ -116,8 +115,7 @@ public static class CsvOptions { title = "The separator for fields in a CSV file." ) @Builder.Default - @PluginProperty(dynamic = true) - private String fieldDelimiter = ","; + private Property fieldDelimiter = Property.of(","); @Schema( title = "The number of rows at the top of a CSV file that will be skipped when reading the data.", @@ -125,20 +123,18 @@ public static class CsvOptions { " that should be skipped." ) @PluginProperty - private Long skipLeadingRows; + private Property skipLeadingRows; @Schema( title = "The quote character in a CSV file." ) - @PluginProperty(dynamic = true) - private String quote; + private Property quote; @Schema( title = "The file encoding of CSV file." ) @Builder.Default - @PluginProperty(dynamic = true) - private String encoding = "UTF-8"; + private Property encoding = Property.of("UTF-8"); } } diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractRead.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractRead.java index 19dda6b..244eae5 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractRead.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractRead.java @@ -1,6 +1,7 @@ package io.kestra.plugin.googleworkspace.sheets; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.exceptions.IllegalVariableEvaluationException; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.FileSerde; import io.swagger.v3.oas.annotations.media.Schema; @@ -28,8 +29,7 @@ public abstract class AbstractRead extends AbstractSheet { title = "The spreadsheet unique id" ) @NotNull - @PluginProperty(dynamic = true) - protected String spreadsheetId; + protected Property spreadsheetId; @Schema( title = "Determines how values should be rendered in the output.", @@ -37,8 +37,7 @@ public abstract class AbstractRead extends AbstractSheet { ) @NotNull @Builder.Default - @PluginProperty(dynamic = false) - protected ValueRender valueRender = ValueRender.UNFORMATTED_VALUE; + protected Property valueRender = Property.of(ValueRender.UNFORMATTED_VALUE); @Schema( title = "How dates, times, and durations should be represented in the output.", @@ -47,37 +46,34 @@ public abstract class AbstractRead extends AbstractSheet { ) @NotNull @Builder.Default - @PluginProperty(dynamic = false) - protected DateTimeRender dateTimeRender = DateTimeRender.FORMATTED_STRING; + protected Property dateTimeRender = Property.of(DateTimeRender.FORMATTED_STRING); @Builder.Default @Schema( title = "Specifies if the first line should be the header (default: false)" ) - protected final Boolean header = true; + protected final Property header = Property.of(true); @Schema( title = "Whether to Fetch the data from the query result to the task output" ) - @PluginProperty(dynamic = false) @Builder.Default - protected final boolean fetch = false; + protected final Property fetch = Property.of(false); @Schema( title = "Whether to store the data from the query result into an ion serialized data file" ) - @PluginProperty(dynamic = false) @Builder.Default - protected final boolean store = true; + protected final Property store = Property.of(true); - protected List transform(List> values) { + protected List transform(List> values, RunContext runContext) throws IllegalVariableEvaluationException { List result = new ArrayList<>(); if (values == null || values.isEmpty()) { return result; } - if (this.header) { + if (runContext.render(this.header).as(Boolean.class).orElseThrow()) { List headers = values.get(0); for (int i = 1; i < values.size(); i++) { diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractSheet.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractSheet.java index a4e1fbb..b92cae4 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractSheet.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/AbstractSheet.java @@ -5,6 +5,7 @@ import com.google.auth.http.HttpCredentialsAdapter; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.plugin.googleworkspace.AbstractTask; import lombok.*; @@ -21,8 +22,7 @@ @NoArgsConstructor public abstract class AbstractSheet extends AbstractTask { @Builder.Default - @PluginProperty(dynamic = true) - protected List scopes = List.of(SheetsScopes.SPREADSHEETS); + protected Property> scopes = Property.of(List.of(SheetsScopes.SPREADSHEETS)); protected Sheets connection(RunContext runContext) throws IllegalVariableEvaluationException, IOException, GeneralSecurityException { HttpCredentialsAdapter credentials = this.credentials(runContext); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheet.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheet.java index 5c1cdd0..1372a6a 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheet.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheet.java @@ -8,6 +8,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.models.tasks.Task; import io.kestra.core.runners.RunContext; @@ -36,11 +37,11 @@ code = """ id: googleworkspace_sheets_create namespace: company.team - + inputs: - id: serviceAccount type: STRING - + tasks: - id: create_spreadsheet type: io.kestra.plugin.googleworkspace.sheets.CreateSpreadsheet @@ -55,8 +56,7 @@ public class CreateSpreadsheet extends AbstractSheet implements RunnableTask title; @Override public Output run(RunContext runContext) throws Exception { @@ -64,7 +64,7 @@ public Output run(RunContext runContext) throws Exception { Logger logger = runContext.logger(); SpreadsheetProperties properties = new SpreadsheetProperties() - .setTitle(runContext.render(this.title)); + .setTitle(runContext.render(this.title).as(String.class).orElseThrow()); Spreadsheet spreadsheet = new Spreadsheet() .setProperties(properties); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/DataParser.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/DataParser.java index 47a8716..79c335d 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/DataParser.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/DataParser.java @@ -39,7 +39,7 @@ public List> parseCsv(InputStream inputStream, AbstractLoad.CsvOpti List> result = new ArrayList<>(); Charset charset = Charset.forName( - runContext.render(csvOptions.getEncoding()) + runContext.render(csvOptions.getEncoding()).as(String.class).orElse(null) ); InputStreamReader reader = new InputStreamReader(inputStream, charset); @@ -239,18 +239,18 @@ private CSVFormat getCsvFormat(AbstractLoad.CsvOptions csvOptions) throws Illega return CSVFormat.Builder.create() .setDelimiter( csvOptions.getFieldDelimiter() != null ? - runContext.render(csvOptions.getFieldDelimiter()) : + this.runContext.render(csvOptions.getFieldDelimiter()).as(String.class).orElseThrow() : CSVFormat.DEFAULT.getDelimiterString() ) .setQuote( csvOptions.getQuote() != null ? - runContext.render(csvOptions.getQuote()).charAt(0) : + this.runContext.render(csvOptions.getQuote()).as(String.class).orElseThrow().charAt(0) : CSVFormat.DEFAULT.getQuoteCharacter() ) .setRecordSeparator(CSVFormat.DEFAULT.getRecordSeparator()) .setIgnoreEmptyLines(true) .setAllowDuplicateHeaderNames(false) - .setSkipHeaderRecord(csvOptions.getSkipLeadingRows() != null && csvOptions.getSkipLeadingRows() > 0) + .setSkipHeaderRecord(csvOptions.getSkipLeadingRows() != null && runContext.render(csvOptions.getSkipLeadingRows()).as(Long.class).orElseThrow() > 0) .build(); } diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheet.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheet.java index edd74d7..e5026b7 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheet.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheet.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.googleworkspace.drive.Delete; @@ -29,11 +30,11 @@ code = """ id: googleworkspace_sheets_delete namespace: company.team - + inputs: - id: serviceAccount type: STRING - + tasks: - id: delete_spreadsheet type: io.kestra.plugin.googleworkspace.sheets.DeleteSpreadsheet @@ -49,14 +50,13 @@ public class DeleteSpreadsheet extends AbstractSheet implements RunnableTask spreadsheetId; @Override public Output run(RunContext runContext) throws Exception { Sheets services = this.connection(runContext); - String spreadsheetId = runContext.render(this.spreadsheetId); + String spreadsheetId = runContext.render(this.spreadsheetId).as(String.class).orElseThrow(); try { services.spreadsheets() @@ -67,8 +67,8 @@ public Output run(RunContext runContext) throws Exception { } Delete delete = Delete.builder() - .serviceAccount(runContext.render(this.serviceAccount)) - .fileId(spreadsheetId) + .serviceAccount(this.serviceAccount) + .fileId(Property.of(spreadsheetId)) .build(); Delete.Output output = delete.run(runContext); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/Load.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/Load.java index 68127f6..42fd2b7 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/Load.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/Load.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -57,27 +58,28 @@ public class Load extends AbstractLoad implements RunnableTask { @Schema( title = "The URI of the Kestra's internal storage file." ) - @PluginProperty(dynamic = true) - private URI from; + private Property from; @Schema( title = "The sheet name or range to select." ) @Builder.Default - @PluginProperty(dynamic = true) - private String range = "Sheet1"; + private Property range = Property.of("Sheet1"); @Override public Output run(RunContext runContext) throws Exception { Sheets service = this.connection(runContext); Logger logger = runContext.logger(); + URI from = URI.create(runContext.render(this.from).as(String.class).orElseThrow()); List> values = this.parse(runContext, from); ValueRange body = new ValueRange().setValues(values); UpdateValuesResponse response = service.spreadsheets().values() - .update(runContext.render(this.spreadsheetId), runContext.render(this.range), body) + .update(runContext.render(this.spreadsheetId).as(String.class).orElseThrow(), + runContext.render(this.range).as(String.class).orElseThrow(), + body) .setValueInputOption("RAW") .execute(); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/Read.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/Read.java index 2523d4e..6866cc9 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/Read.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/Read.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.utils.ListUtils; @@ -20,7 +21,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotNull; import static io.kestra.core.utils.Rethrow.throwFunction; @@ -55,7 +55,7 @@ public class Read extends AbstractRead implements RunnableTask { title = "The sheet title to be included", description = "If not provided all the sheets will be included." ) - private List selectedSheetsTitle; + private Property> selectedSheetsTitle; @Override public Read.Output run(RunContext runContext) throws Exception { @@ -63,10 +63,10 @@ public Read.Output run(RunContext runContext) throws Exception { Logger logger = runContext.logger(); Spreadsheet spreadsheet = service.spreadsheets() - .get(runContext.render(spreadsheetId)) + .get(runContext.render(spreadsheetId).as(String.class).orElseThrow()) .execute(); - List includedSheetsTitle = ListUtils.emptyOnNull(this.selectedSheetsTitle) + List includedSheetsTitle = runContext.render(this.selectedSheetsTitle).asList(String.class) .stream() .map(throwFunction(runContext::render)) .collect(Collectors.toList()); @@ -89,10 +89,10 @@ public Read.Output run(RunContext runContext) throws Exception { // batch get all ranges BatchGetValuesResponse batchGet = service.spreadsheets().values() - .batchGet(runContext.render(spreadsheetId)) + .batchGet(runContext.render(spreadsheetId).as(String.class).orElseThrow()) .setRanges(ranges) - .set("valueRenderOption", this.valueRender.name()) - .set("dateTimeRenderOption", this.dateTimeRender.name()) + .set("valueRenderOption", runContext.render(this.valueRender).as(ValueRender.class).orElseThrow().name()) + .set("dateTimeRenderOption", runContext.render(this.dateTimeRender).as(DateTimeRender.class).orElseThrow().name()) .execute(); // read values @@ -107,9 +107,9 @@ public Read.Output run(RunContext runContext) throws Exception { logger.info("Fetch {} rows from range '{}'", valueRange.getValues().size(), valueRange.getRange()); rowsCount.addAndGet(valueRange.getValues().size()); - List values = this.transform(valueRange.getValues()); + List values = this.transform(valueRange.getValues(), runContext); - if (this.fetch) { + if (runContext.render(this.fetch).as(Boolean.class).orElseThrow()) { rows.put(sheet.getProperties().getTitle(), values); } else { uris.put(sheet.getProperties().getTitle(), runContext.storage().putFile(this.store(runContext, values))); @@ -119,7 +119,7 @@ public Read.Output run(RunContext runContext) throws Exception { Output.OutputBuilder builder = Output.builder() .size(rowsCount.get()); - if (this.fetch) { + if (runContext.render(this.fetch).as(Boolean.class).orElseThrow()) { builder.rows(rows); } else { builder.uris(uris); diff --git a/src/main/java/io/kestra/plugin/googleworkspace/sheets/ReadRange.java b/src/main/java/io/kestra/plugin/googleworkspace/sheets/ReadRange.java index 61f2df3..30139c2 100644 --- a/src/main/java/io/kestra/plugin/googleworkspace/sheets/ReadRange.java +++ b/src/main/java/io/kestra/plugin/googleworkspace/sheets/ReadRange.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.executions.metrics.Counter; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -46,7 +47,7 @@ public class ReadRange extends AbstractRead implements RunnableTask range; @Override public ReadRange.Output run(RunContext runContext) throws Exception { @@ -56,21 +57,21 @@ public ReadRange.Output run(RunContext runContext) throws Exception { ValueRange response = service.spreadsheets() .values() .get( - runContext.render(spreadsheetId), - runContext.render(range) + runContext.render(spreadsheetId).as(String.class).orElseThrow(), + runContext.render(range).as(String.class).orElse(null) ) - .set("valueRenderOption", this.valueRender.name()) - .set("dateTimeRenderOption", this.dateTimeRender.name()) + .set("valueRenderOption", runContext.render(this.valueRender).as(ValueRender.class).orElseThrow().name()) + .set("dateTimeRenderOption", runContext.render(this.dateTimeRender).as(DateTimeRender.class).orElseThrow().name()) .execute(); logger.info("Fetch {} rows from range '{}'", response.getValues().size(), response.getRange()); - List values = this.transform(response.getValues()); + List values = this.transform(response.getValues(), runContext); Output.OutputBuilder builder = Output.builder() .size(values.size()); - if (this.fetch) { + if (runContext.render(this.fetch).as(Boolean.class).orElseThrow()) { builder.rows(values); } else { builder.uri(runContext.storage().putFile(this.store(runContext, values))); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/calendar/InsertEventTest.java b/src/test/java/io/kestra/plugin/googleworkspace/calendar/InsertEventTest.java index 06aa7fe..74c6c48 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/calendar/InsertEventTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/calendar/InsertEventTest.java @@ -5,6 +5,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import io.kestra.core.models.property.Property; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIf; @@ -33,16 +34,20 @@ class InsertEventTest { void run() throws Exception { RunContext runContext = runContextFactory.of(); - CalendarTime startTime = CalendarTime.builder().dateTime("2024-11-28T09:00:00+05:30").timeZone("Asia/Calcutta").build(); - CalendarTime endTime = CalendarTime.builder().dateTime("2024-11-28T10:00:00+05:30").timeZone("Asia/Calcutta").build(); + CalendarTime startTime = CalendarTime.builder() + .dateTime(Property.of("2024-11-28T09:00:00+05:30")) + .timeZone(Property.of("Asia/Calcutta")).build(); + CalendarTime endTime = CalendarTime.builder() + .dateTime(Property.of("2024-11-28T10:00:00+05:30")) + .timeZone(Property.of("Asia/Calcutta")).build(); InsertEvent task = InsertEvent.builder() - .calendarId("primary") - .summary("New Calendar Event") + .calendarId(Property.of("primary")) + .summary(Property.of("New Calendar Event")) .description("This is a new calendar event generated by Kestra.") .startTime(startTime) .endTime(endTime) - .serviceAccount(UtilsTest.serviceAccount()) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); InsertEvent.Output runOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/drive/ListTest.java b/src/test/java/io/kestra/plugin/googleworkspace/drive/ListTest.java index 7e02737..c803c97 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/drive/ListTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/drive/ListTest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.googleworkspace.drive; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.TestsUtils; import io.kestra.plugin.googleworkspace.UtilsTest; @@ -22,8 +23,8 @@ void run() throws Exception { List task = List.builder() .id(ListTest.class.getSimpleName()) .type(List.class.getName()) - .query("'1YgHpphjepA8gAme1J04ftxVf7j80XABU' in parents") - .serviceAccount(UtilsTest.serviceAccount()) + .query(Property.of("'1YgHpphjepA8gAme1J04ftxVf7j80XABU' in parents")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); List.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, Map.of())); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/drive/SuiteTest.java b/src/test/java/io/kestra/plugin/googleworkspace/drive/SuiteTest.java index 2b1e0ae..8b98cb0 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/drive/SuiteTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/drive/SuiteTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.googleworkspace.drive; import com.google.common.io.CharStreams; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.storages.StorageInterface; import io.kestra.core.utils.IdUtils; @@ -54,10 +55,10 @@ void run() throws Exception { Create create = Create.builder() .id(SuiteTest.class.getSimpleName()) .type(Create.class.getName()) - .name(IdUtils.create()) - .parents(List.of("1gkUuyf7CmVjEz7QR-Hl9Xx5kdmbk5Lwo")) - .mimeType("application/vnd.google-apps.folder") - .serviceAccount(UtilsTest.serviceAccount()) + .name(Property.of(IdUtils.create())) + .parents(Property.of(List.of("1gkUuyf7CmVjEz7QR-Hl9Xx5kdmbk5Lwo"))) + .mimeType(Property.of("application/vnd.google-apps.folder")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Create.Output createRun = create.run(TestsUtils.mockRunContext(runContextFactory, create, Map.of())); @@ -67,12 +68,12 @@ void run() throws Exception { Upload upload = Upload.builder() .id(SuiteTest.class.getSimpleName()) .type(Upload.class.getName()) - .from(source.toString()) - .parents(List.of(createRun.getFile().getId())) - .name(IdUtils.create()) - .contentType("text/csv") - .mimeType("application/vnd.google-apps.spreadsheet") - .serviceAccount(UtilsTest.serviceAccount()) + .from(Property.of(source.toString())) + .parents(Property.of(List.of(createRun.getFile().getId()))) + .name(Property.of(IdUtils.create())) + .contentType(Property.of("text/csv")) + .mimeType(Property.of("application/vnd.google-apps.spreadsheet")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Upload.Output uploadRun = upload.run(TestsUtils.mockRunContext(runContextFactory, upload, Map.of())); @@ -82,9 +83,9 @@ void run() throws Exception { Export export = Export.builder() .id(SuiteTest.class.getSimpleName()) .type(Export.class.getName()) - .fileId(uploadRun.getFile().getId()) - .contentType("text/csv") - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(uploadRun.getFile().getId())) + .contentType(Property.of("text/csv")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Export.Output exportRun = export.run(TestsUtils.mockRunContext(runContextFactory, export, Map.of())); @@ -107,12 +108,12 @@ void run() throws Exception { Upload upload2 = Upload.builder() .id(SuiteTest.class.getSimpleName()) .type(Upload.class.getName()) - .from(source2.toString()) - .fileId(uploadRun.getFile().getId()) - .name(IdUtils.create()) - .contentType("text/csv") - .mimeType("application/vnd.google-apps.spreadsheet") - .serviceAccount(UtilsTest.serviceAccount()) + .from(Property.of(source2.toString())) + .fileId(Property.of(uploadRun.getFile().getId())) + .name(Property.of(IdUtils.create())) + .contentType(Property.of("text/csv")) + .mimeType(Property.of("application/vnd.google-apps.spreadsheet")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Upload.Output upload2Run = upload2.run(TestsUtils.mockRunContext(runContextFactory, upload, Map.of())); @@ -122,9 +123,9 @@ void run() throws Exception { Export export2 = Export.builder() .id(SuiteTest.class.getSimpleName()) .type(Export.class.getName()) - .fileId(uploadRun.getFile().getId()) - .contentType("text/csv") - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(uploadRun.getFile().getId())) + .contentType(Property.of("text/csv")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Export.Output exportRun2 = export2.run(TestsUtils.mockRunContext(runContextFactory, export, Map.of())); @@ -147,8 +148,8 @@ void run() throws Exception { Delete delete = Delete.builder() .id(SuiteTest.class.getSimpleName()) .type(Delete.class.getName()) - .fileId(uploadRun.getFile().getId()) - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(uploadRun.getFile().getId())) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Delete.Output deleteRun = delete.run(TestsUtils.mockRunContext(runContextFactory, delete, Map.of())); @@ -158,8 +159,8 @@ void run() throws Exception { Delete deleteFolder = Delete.builder() .id(SuiteTest.class.getSimpleName()) .type(Delete.class.getName()) - .fileId(createRun.getFile().getId()) - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(createRun.getFile().getId())) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Delete.Output deleteFolderRun = deleteFolder.run(TestsUtils.mockRunContext(runContextFactory, deleteFolder, Map.of())); @@ -187,11 +188,11 @@ void binary() throws Exception { Upload upload = Upload.builder() .id(SuiteTest.class.getSimpleName()) .type(Upload.class.getName()) - .from(source.toString()) - .parents(List.of("1gkUuyf7CmVjEz7QR-Hl9Xx5kdmbk5Lwo")) - .name(IdUtils.create()) - .contentType("application/zip") - .serviceAccount(UtilsTest.serviceAccount()) + .from(Property.of(source.toString())) + .parents(Property.of(List.of("1gkUuyf7CmVjEz7QR-Hl9Xx5kdmbk5Lwo"))) + .name(Property.of(IdUtils.create())) + .contentType(Property.of("application/zip")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Upload.Output uploadRun = upload.run(TestsUtils.mockRunContext(runContextFactory, upload, Map.of())); @@ -201,8 +202,8 @@ void binary() throws Exception { Download download = Download.builder() .id(SuiteTest.class.getSimpleName()) .type(Export.class.getName()) - .fileId(uploadRun.getFile().getId()) - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(uploadRun.getFile().getId())) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Download.Output downloadRun = download.run(TestsUtils.mockRunContext(runContextFactory, upload, Map.of())); @@ -225,8 +226,8 @@ void binary() throws Exception { Delete delete = Delete.builder() .id(SuiteTest.class.getSimpleName()) .type(Delete.class.getName()) - .fileId(uploadRun.getFile().getId()) - .serviceAccount(UtilsTest.serviceAccount()) + .fileId(Property.of(uploadRun.getFile().getId())) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); Delete.Output deleteRun = delete.run(TestsUtils.mockRunContext(runContextFactory, upload, Map.of())); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheetTest.java b/src/test/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheetTest.java index ba1dc1c..2f45384 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheetTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/sheets/CreateSpreadsheetTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.googleworkspace.sheets; import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.plugin.googleworkspace.UtilsTest; @@ -27,8 +28,8 @@ void run() throws Exception { CreateSpreadsheet createTask = CreateSpreadsheet.builder() .id(CreateSpreadsheetTest.class.getSimpleName()) - .title("CSV Test Spreadsheet") - .serviceAccount(UtilsTest.serviceAccount()) + .title(Property.of("CSV Test Spreadsheet")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); CreateSpreadsheet.Output createOutput = createTask.run(runContext); @@ -38,8 +39,8 @@ void run() throws Exception { DeleteSpreadsheet deleteTask = DeleteSpreadsheet.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(UtilsTest.serviceAccount()) - .spreadsheetId(createOutput.getSpreadsheetId()) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .spreadsheetId(Property.of(createOutput.getSpreadsheetId())) .build(); DeleteSpreadsheet.Output deleteOutput = deleteTask.run(runContext); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheetTest.java b/src/test/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheetTest.java index acdd5c0..fa70d95 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheetTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/sheets/DeleteSpreadsheetTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.googleworkspace.sheets; import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -29,8 +30,8 @@ void deleteExisting() throws Exception { CreateSpreadsheet createTask = CreateSpreadsheet.builder() .id(DeleteSpreadsheetTest.class.getSimpleName()) - .title("CSV Test Spreadsheet") - .serviceAccount(UtilsTest.serviceAccount()) + .title(Property.of("CSV Test Spreadsheet")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) .build(); CreateSpreadsheet.Output createOutput = createTask.run(runContext); @@ -40,8 +41,8 @@ void deleteExisting() throws Exception { DeleteSpreadsheet deleteTask = DeleteSpreadsheet.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(UtilsTest.serviceAccount()) - .spreadsheetId(createOutput.getSpreadsheetId()) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .spreadsheetId(Property.of(createOutput.getSpreadsheetId())) .build(); DeleteSpreadsheet.Output deleteOutput = deleteTask.run(runContext); @@ -55,8 +56,8 @@ void deleteNotExisting() throws Exception { DeleteSpreadsheet deleteTask = DeleteSpreadsheet.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(UtilsTest.serviceAccount()) - .spreadsheetId(IdUtils.create()) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .spreadsheetId(Property.of(IdUtils.create())) .build(); assertThrows(IllegalArgumentException.class, () -> deleteTask.run(runContext)); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/sheets/LoadTest.java b/src/test/java/io/kestra/plugin/googleworkspace/sheets/LoadTest.java index 5d79e2d..a694bac 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/sheets/LoadTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/sheets/LoadTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.googleworkspace.sheets; import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.storages.StorageInterface; @@ -49,9 +50,9 @@ void loadCSV() throws Exception { Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(getSource(".csv")) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(getSource(".csv").toString())) .build(); Load.Output run = task.run(runContext); @@ -71,9 +72,9 @@ void loadJSON() throws Exception { URI source = getSource(".json"); Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) .build(); Load.Output run = task.run(runContext); @@ -94,10 +95,10 @@ void loadJSONWithHeader() throws Exception { Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) - .header(true) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) + .header(Property.of(true)) .build(); Load.Output run = task.run(runContext); @@ -117,9 +118,9 @@ void loadAVRO() throws Exception { URI source = getSource(".avro"); Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) .build(); Load.Output run = task.run(runContext); @@ -140,10 +141,10 @@ void loadAVROWithHeader() throws Exception { Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) - .header(true) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) + .header(Property.of(true)) .build(); Load.Output run = task.run(runContext); @@ -163,9 +164,9 @@ void loadORC() throws Exception { URI source = getSource(".orc"); Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) .build(); Load.Output run = task.run(runContext); @@ -186,10 +187,10 @@ void loadORCWithHeader() throws Exception { Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) - .header(true) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) + .header(Property.of(true)) .build(); Load.Output run = task.run(runContext); @@ -209,9 +210,9 @@ void loadPARQUET() throws Exception { URI source = getSource(".parquet"); Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) .build(); Load.Output run = task.run(runContext); @@ -232,10 +233,10 @@ void loadPARQUETWithHeader() throws Exception { Load task = Load.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) - .from(source) - .header(true) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) + .from(Property.of(source.toString())) + .header(Property.of(true)) .build(); Load.Output run = task.run(runContext); @@ -261,8 +262,8 @@ private URI getSource(String extension) throws IOException, URISyntaxException { private String createSpreadsheet(RunContext runContext) throws Exception { CreateSpreadsheet createTask = CreateSpreadsheet.builder() .id(LoadTest.class.getSimpleName()) - .title("CSV Test Spreadsheet") - .serviceAccount(serviceAccount) + .title(Property.of("CSV Test Spreadsheet")) + .serviceAccount(Property.of(serviceAccount)) .build(); CreateSpreadsheet.Output createOutput = createTask.run(runContext); @@ -275,8 +276,8 @@ private String createSpreadsheet(RunContext runContext) throws Exception { private void deleteSpreadsheet(RunContext runContext, String spreadsheetId) throws Exception { DeleteSpreadsheet deleteTask = DeleteSpreadsheet.builder() .id(LoadTest.class.getSimpleName()) - .serviceAccount(serviceAccount) - .spreadsheetId(spreadsheetId) + .serviceAccount(Property.of(serviceAccount)) + .spreadsheetId(Property.of(spreadsheetId)) .build(); DeleteSpreadsheet.Output deleteOutput = deleteTask.run(runContext); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadRangeTest.java b/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadRangeTest.java index 0c9d6a3..824f1a6 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadRangeTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadRangeTest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.googleworkspace.sheets; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.serializers.FileSerde; import io.kestra.core.storages.StorageInterface; @@ -32,10 +33,10 @@ void rangeFetch() throws Exception { ReadRange task = ReadRange.builder() .id(ReadRangeTest.class.getSimpleName()) .type(ReadRange.class.getName()) - .spreadsheetId("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE") - .serviceAccount(UtilsTest.serviceAccount()) - .range("Class Data!A1:I") - .fetch(true) + .spreadsheetId(Property.of("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .range(Property.of("Class Data!A1:I")) + .fetch(Property.of(true)) .build(); ReadRange.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, Map.of())); @@ -50,9 +51,9 @@ void rangeStore() throws Exception { ReadRange task = ReadRange.builder() .id(ReadRangeTest.class.getSimpleName()) .type(ReadRange.class.getName()) - .spreadsheetId("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE") - .serviceAccount(UtilsTest.serviceAccount()) - .range("Second One!A1:I") + .spreadsheetId(Property.of("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .range(Property.of("Second One!A1:I")) .build(); ReadRange.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, Map.of())); diff --git a/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadTest.java b/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadTest.java index 77bfe9d..828eafd 100644 --- a/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadTest.java +++ b/src/test/java/io/kestra/plugin/googleworkspace/sheets/ReadTest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.googleworkspace.sheets; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.TestsUtils; import io.kestra.plugin.googleworkspace.UtilsTest; @@ -25,9 +26,9 @@ void run() throws Exception { Read task = Read.builder() .id(ReadTest.class.getSimpleName()) .type(ReadRange.class.getName()) - .spreadsheetId("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE") - .serviceAccount(UtilsTest.serviceAccount()) - .fetch(true) + .spreadsheetId(Property.of("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .fetch(Property.of(true)) .build(); Read.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, Map.of())); @@ -48,10 +49,10 @@ void selected() throws Exception { Read task = Read.builder() .id(ReadTest.class.getSimpleName()) .type(ReadRange.class.getName()) - .spreadsheetId("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE") - .serviceAccount(UtilsTest.serviceAccount()) - .selectedSheetsTitle(List.of("Second One")) - .fetch(true) + .spreadsheetId(Property.of("1Dkd7W0OQo-wxz9rrORLP7YGSj6EBLEg73fiTdbJUIQE")) + .serviceAccount(Property.of(UtilsTest.serviceAccount())) + .selectedSheetsTitle(Property.of(List.of("Second One"))) + .fetch(Property.of(true)) .build(); Read.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, Map.of()));