From dc55a071a198718eced06ee78307a16c4829f94b Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 04:21:15 +0000 Subject: [PATCH 1/6] =?UTF-8?q?rest=20client=20class=E3=81=ABOpportunity,O?= =?UTF-8?q?pportunityRole=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../marketo/rest/MarketoRESTEndpoint.java | 4 +++- .../input/marketo/rest/MarketoRestClient.java | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java b/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java index 6cebe20..41a5ffb 100644 --- a/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +++ b/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java @@ -33,7 +33,9 @@ public enum MarketoRESTEndpoint START_PROGRAM_MEMBERS_EXPORT_JOB("/bulk/v1/program/members/export/${export_id}/enqueue.json"), GET_PROGRAM_MEMBERS_EXPORT_STATUS("/bulk/v1/program/members/export/${export_id}/status.json"), GET_PROGRAM_MEMBERS_EXPORT_RESULT("/bulk/v1/program/members/export/${export_id}/file.json"), - GET_FOLDERS("/rest/asset/v1/folders.json"); + GET_FOLDERS("/rest/asset/v1/folders.json"), + GET_OPPORTUNITIES("/rest/v1/opportunities.json"), + GET_OPPORTUNITY_ROLES("/rest/v1/opportunities/roles.json"); private final String endpoint; MarketoRESTEndpoint(String endpoint) diff --git a/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java b/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java index 4db44a4..b004716 100644 --- a/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +++ b/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java @@ -643,4 +643,24 @@ public RecordPagingIterable getFolders(Optional root, int ma } return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_FOLDERS.getEndpoint(), builder.build(), ObjectNode.class); } + + public RecordPagingIterable getOpportunities(String filterType, String filterValue) + { + Multimap params = new ImmutableListMultimap + .Builder() + .put("filterType", StringUtils.trimToEmpty(filterType)) + .put("filterValues", StringUtils.trimToEmpty(filterValue)) + .put(BATCH_SIZE, MAX_BATCH_SIZE).build(); + return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_OPPORTUNITIES.getEndpoint(), params, ObjectNode.class); + } + + public RecordPagingIterable getOpportunityRoles(String filterType, String filterValue) + { + Multimap params = new ImmutableListMultimap + .Builder() + .put("filterType", StringUtils.trimToEmpty(filterType)) + .put("filterValues", StringUtils.trimToEmpty(filterValue)) + .put(BATCH_SIZE, MAX_BATCH_SIZE).build(); + return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_OPPORTUNITY_ROLES.getEndpoint(), params, ObjectNode.class); + } } From 46a91199b663244a767ba406364221ec5cf3ce83 Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 04:21:54 +0000 Subject: [PATCH 2/6] =?UTF-8?q?service=20class=E3=81=ABOpportunity,Opportu?= =?UTF-8?q?nityRole=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../embulk/input/marketo/MarketoService.java | 5 ++++ .../input/marketo/MarketoServiceImpl.java | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/main/java/org/embulk/input/marketo/MarketoService.java b/src/main/java/org/embulk/input/marketo/MarketoService.java index 5e016d0..ae3e442 100644 --- a/src/main/java/org/embulk/input/marketo/MarketoService.java +++ b/src/main/java/org/embulk/input/marketo/MarketoService.java @@ -52,4 +52,9 @@ public interface MarketoService File extractProgramMembers(String exportID); Iterable getFolders(Optional rootId, RootType rootType, Integer maxDepth, Optional workspace); + + Iterable getOpportunities(String filterType, Set filterValues); + + Iterable getOpportunityRoles(String filterType, Set filterValues); } + diff --git a/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java b/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java index 1141b57..bd6e45d 100644 --- a/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +++ b/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java @@ -286,4 +286,27 @@ public Iterable getFolders(Optional rootId, RootType rootType, Optional root = rootId.isPresent() ? Optional.of(String.format("{\"id\": %d, \"type\": \"%s\"}", rootId.get(), type)) : Optional.empty(); return marketoRestClient.getFolders(root, maxDepth, workspace); } + + @Override + public Iterable getOpportunities(String filterType, Set filterValues) + { + // Marketo allows maximum 300 (comma separated) filterValues per request. Split the input and process by chunk. + List> partitionedFilterValues = Lists.partition(Lists.newArrayList(filterValues), 300); + return MarketoUtils.flatMap(partitionedFilterValues, (part) -> { + String strFilterValues = StringUtils.join(part, ","); + return marketoRestClient.getOpportunities(filterType, strFilterValues); + }); + } + + @Override + public Iterable getOpportunityRoles(String filterType, Set filterValues) + { + // Marketo allows maximum 300 (comma separated) filterValues per request. Split the input and process by chunk. + List> partitionedFilterValues = Lists.partition(Lists.newArrayList(filterValues), 300); + return MarketoUtils.flatMap(partitionedFilterValues, (part) -> { + String strFilterValues = StringUtils.join(part, ","); + return marketoRestClient.getOpportunityRoles(filterType, strFilterValues); + }); + } } + From 331e15ea002f9cd5d36579f4f25969f41f00d6b6 Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 04:23:22 +0000 Subject: [PATCH 3/6] add Opportunity,OpportunityRole Input Plugins --- .../marketo/MarketoInputPluginDelegate.java | 10 ++- .../delegate/OpportunityInputPlugin.java | 76 ++++++++++++++++++ .../delegate/OpportunityRoleInputPlugin.java | 77 +++++++++++++++++++ 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java create mode 100644 src/main/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPlugin.java diff --git a/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java b/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java index ae4c8cf..fc18953 100644 --- a/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +++ b/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java @@ -15,6 +15,8 @@ import org.embulk.input.marketo.delegate.LeadWithListInputPlugin; import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin; import org.embulk.input.marketo.delegate.ListInputPlugin; +import org.embulk.input.marketo.delegate.OpportunityInputPlugin; +import org.embulk.input.marketo.delegate.OpportunityRoleInputPlugin; import org.embulk.input.marketo.delegate.ProgramInputPlugin; import org.embulk.input.marketo.delegate.ProgramMembersBulkExtractInputPlugin; import org.embulk.input.marketo.rest.MarketoRestClient; @@ -39,7 +41,9 @@ public interface PluginTask ProgramMembersBulkExtractInputPlugin.PluginTask, ListInputPlugin.PluginTask, ActivityTypeInputPlugin.PluginTask, - FolderInputPlugin.PluginTask + FolderInputPlugin.PluginTask, + OpportunityInputPlugin.PluginTask, + OpportunityRoleInputPlugin.PluginTask { @Config("target") Target getTarget(); @@ -88,7 +92,9 @@ public enum Target PROGRAM_MEMBERS(new ProgramMembersBulkExtractInputPlugin()), LIST(new ListInputPlugin()), ACTIVITY_TYPE(new ActivityTypeInputPlugin()), - FOLDER(new FolderInputPlugin()); + FOLDER(new FolderInputPlugin()), + OPPORTUNITY(new OpportunityInputPlugin()), + OPPORTUNITY_ROLE(new OpportunityRoleInputPlugin()); private final RestClientInputPluginDelegate restClientInputPluginDelegate; diff --git a/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java b/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java new file mode 100644 index 0000000..e60ab62 --- /dev/null +++ b/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java @@ -0,0 +1,76 @@ +package org.embulk.input.marketo.delegate; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.FluentIterable; +import org.apache.commons.lang3.StringUtils; +import org.embulk.base.restclient.ServiceResponseMapper; +import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper; +import org.embulk.base.restclient.record.ServiceRecord; +import org.embulk.base.restclient.record.ValueLocator; +import org.embulk.config.ConfigException; +import org.embulk.input.marketo.MarketoService; +import org.embulk.input.marketo.MarketoUtils; +import org.embulk.spi.type.Types; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; + +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class OpportunityInputPlugin extends MarketoBaseInputPluginDelegate +{ + public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask + { + @Config("opportunity_filter_type") + @ConfigDefault("\"\"") + String getOpportunityFilterType(); + + @Config("opportunity_filter_values") + @ConfigDefault("null") + Optional getOpportunityFilterValues(); + } + + public OpportunityInputPlugin() + { + } + + @Override + public void validateInputTask(PluginTask task) + { + super.validateInputTask(task); + if (StringUtils.isBlank(task.getOpportunityFilterType())) { + throw new ConfigException("`opportunity_filter_type` cannot be empty"); + } + if (refineFilterValues(task.getOpportunityFilterValues().orElse("")).isEmpty()) { + throw new ConfigException("`opportunity_filter_values` cannot contain empty values only"); + } + } + + private Set refineFilterValues(String filterValues) + { + return Stream.of(StringUtils.split(filterValues, ",")).map(StringUtils::trimToEmpty).filter(StringUtils::isNotBlank).collect(Collectors.toSet()); + } + + @Override + protected Iterator getServiceRecords(MarketoService marketoService, PluginTask task) + { + Set refinedValues = refineFilterValues(task.getOpportunityFilterValues().get()); + Iterable responseObj = marketoService.getOpportunities(task.getOpportunityFilterType(), refinedValues); + return FluentIterable.from(responseObj).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator(); + } + + @Override + public ServiceResponseMapper buildServiceResponseMapper(PluginTask task) + { + JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder(); + builder.add("marketoGUID", Types.STRING) + .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) + .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) + .add("externalOpportunityId", Types.STRING); + return builder.build(); + } +} + diff --git a/src/main/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPlugin.java b/src/main/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPlugin.java new file mode 100644 index 0000000..090da12 --- /dev/null +++ b/src/main/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPlugin.java @@ -0,0 +1,77 @@ +package org.embulk.input.marketo.delegate; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.FluentIterable; +import org.apache.commons.lang3.StringUtils; +import org.embulk.base.restclient.ServiceResponseMapper; +import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper; +import org.embulk.base.restclient.record.ServiceRecord; +import org.embulk.base.restclient.record.ValueLocator; +import org.embulk.config.ConfigException; +import org.embulk.input.marketo.MarketoService; +import org.embulk.input.marketo.MarketoUtils; +import org.embulk.spi.type.Types; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; + +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class OpportunityRoleInputPlugin extends MarketoBaseInputPluginDelegate +{ + public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask + { + @Config("opportunity_role_filter_type") + @ConfigDefault("\"\"") + String getOpportunityRoleFilterType(); + + @Config("opportunity_role_filter_values") + @ConfigDefault("null") + Optional getOpportunityRoleFilterValues(); + } + + public OpportunityRoleInputPlugin() + { + } + + @Override + public void validateInputTask(PluginTask task) + { + super.validateInputTask(task); + if (StringUtils.isBlank(task.getOpportunityRoleFilterType())) { + throw new ConfigException("`opportunity_role_filter_type` cannot be empty"); + } + if (refineFilterValues(task.getOpportunityRoleFilterValues().orElse("")).isEmpty()) { + throw new ConfigException("`opportunity_role_filter_values` cannot contain empty values only"); + } + } + + private Set refineFilterValues(String filterValues) + { + return Stream.of(StringUtils.split(filterValues, ",")).map(StringUtils::trimToEmpty).filter(StringUtils::isNotBlank).collect(Collectors.toSet()); + } + + @Override + protected Iterator getServiceRecords(MarketoService marketoService, PluginTask task) + { + Set refinedValues = refineFilterValues(task.getOpportunityRoleFilterValues().get()); + Iterable responseObj = marketoService.getOpportunityRoles(task.getOpportunityRoleFilterType(), refinedValues); + return FluentIterable.from(responseObj).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator(); + } + + @Override + public ServiceResponseMapper buildServiceResponseMapper(PluginTask task) + { + JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder(); + builder.add("marketoGUID", Types.STRING) + .add("externalOpportunityId", Types.STRING) + .add("leadId", Types.LONG) + .add("role", Types.STRING) + .add("isPrimary", Types.BOOLEAN); + return builder.build(); + } +} + From cdc628876e5341bfa0ef31c88a5450e7d63e30f2 Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 06:52:56 +0000 Subject: [PATCH 4/6] fix whitespaces --- .../embulk/input/marketo/delegate/OpportunityInputPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java b/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java index e60ab62..10b7809 100644 --- a/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java +++ b/src/main/java/org/embulk/input/marketo/delegate/OpportunityInputPlugin.java @@ -67,8 +67,8 @@ public ServiceResponseMapper buildServiceResponseMapper( { JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder(); builder.add("marketoGUID", Types.STRING) - .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) - .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) + .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) + .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT) .add("externalOpportunityId", Types.STRING); return builder.build(); } From 32240734bc2d31412f0683457e9212a6fa10a75b Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 07:26:35 +0000 Subject: [PATCH 5/6] add OpportunityInputPluginTest --- .../delegate/OpportunityInputPluginTest.java | 118 ++++++++++++++++++ src/test/resources/config/opportunity.yaml | 6 + .../fixtures/opportunity_response.json | 24 ++++ 3 files changed, 148 insertions(+) create mode 100644 src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java create mode 100644 src/test/resources/config/opportunity.yaml create mode 100644 src/test/resources/fixtures/opportunity_response.json diff --git a/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java b/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java new file mode 100644 index 0000000..4768d37 --- /dev/null +++ b/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java @@ -0,0 +1,118 @@ +package org.embulk.input.marketo.delegate; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.embulk.EmbulkTestRuntime; +import org.embulk.base.restclient.ServiceResponseMapper; +import org.embulk.base.restclient.record.RecordImporter; +import org.embulk.base.restclient.record.ValueLocator; +import org.embulk.config.ConfigException; +import org.embulk.config.ConfigLoader; +import org.embulk.config.ConfigSource; +import org.embulk.input.marketo.rest.MarketoRestClient; +import org.embulk.input.marketo.rest.RecordPagingIterable; +import org.embulk.spi.PageBuilder; +import org.embulk.spi.Schema; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.List; + +import static org.embulk.input.marketo.MarketoUtilsTest.CONFIG_MAPPER; +import static org.embulk.input.marketo.delegate.OpportunityInputPlugin.PluginTask; +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class OpportunityInputPluginTest +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Rule + public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime(); + + private OpportunityInputPlugin opportunityInputPlugin; + + private ConfigSource configSource; + + private MarketoRestClient mockMarketoRestClient; + + @Before + public void setUp() throws Exception + { + opportunityInputPlugin = spy(new OpportunityInputPlugin()); + ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class); + configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/opportunity.yaml")); + mockMarketoRestClient = mock(MarketoRestClient.class); + doReturn(mockMarketoRestClient).when(opportunityInputPlugin).createMarketoRestClient(any(PluginTask.class)); + } + + @Test + public void testValidPluginTask() + { + PluginTask pluginTask = mapTask(configSource); + opportunityInputPlugin.validateInputTask(pluginTask); + } + + @Test + public void testCustomObjectFilterTypeError() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_filter_type", "")); + Assert.assertThrows(ConfigException.class, () -> opportunityInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testEmptyStringFilterValues() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_filter_values", "")); + Assert.assertThrows(ConfigException.class, () -> opportunityInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testAllEmptyStringFilterValues() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_filter_values", ",, , ")); + Assert.assertThrows(ConfigException.class, () -> opportunityInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testRun() throws IOException + { + RecordPagingIterable mockRecordPagingIterable = mock(RecordPagingIterable.class); + JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class); + List objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/opportunity_response.json"), javaType); + when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator()); + when(mockMarketoRestClient.getOpportunities(anyString(), anyString())).thenReturn(mockRecordPagingIterable); + + PluginTask task = mapTask(configSource); + ServiceResponseMapper mapper = opportunityInputPlugin.buildServiceResponseMapper(task); + RecordImporter recordImporter = mapper.createRecordImporter(); + PageBuilder mockPageBuilder = mock(PageBuilder.class); + opportunityInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder); + verify(mockMarketoRestClient, times(1)).getOpportunities(anyString(), anyString()); + + Schema schema = mapper.getEmbulkSchema(); + ArgumentCaptor stringArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(mockPageBuilder, times(3)).setString(eq(schema.lookupColumn("externalOpportunityId")), stringArgumentCaptor.capture()); + List allValues = stringArgumentCaptor.getAllValues(); + assertArrayEquals(new String[]{"externalOpportunityId_1", "externalOpportunityId_2", "externalOpportunityId_3"}, allValues.toArray()); + } + + private PluginTask mapTask(ConfigSource config) + { + return CONFIG_MAPPER.map(config, PluginTask.class); + } +} + diff --git a/src/test/resources/config/opportunity.yaml b/src/test/resources/config/opportunity.yaml new file mode 100644 index 0000000..6c366c7 --- /dev/null +++ b/src/test/resources/config/opportunity.yaml @@ -0,0 +1,6 @@ +account_id: account_id +client_id: client_id +client_secret: client_secret +opportunity_filter_type: opportunity_filter_type +opportunity_filter_values: opportunity_filter_values + diff --git a/src/test/resources/fixtures/opportunity_response.json b/src/test/resources/fixtures/opportunity_response.json new file mode 100644 index 0000000..0909d98 --- /dev/null +++ b/src/test/resources/fixtures/opportunity_response.json @@ -0,0 +1,24 @@ +[ + { + "seq": 0, + "marketoGUID": "marketoGUID_1", + "externalOpportunityId": "externalOpportunityId_1", + "updatedAt": "2023-07-20T06:53:20Z", + "createdAt": "2022-12-06T04:11:12Z" + }, + { + "seq": 1, + "marketoGUID": "marketoGUID_2", + "externalOpportunityId": "externalOpportunityId_2", + "updatedAt": "2023-01-20T06:53:20Z", + "createdAt": "2023-01-06T04:11:12Z" + }, + { + "seq": 2, + "marketoGUID": "marketoGUID_3", + "externalOpportunityId": "externalOpportunityId_3", + "updatedAt": "2024-02-20T06:53:20Z", + "createdAt": "2024-02-06T04:11:12Z" + } +] + From 175c372a449dded22c7d359be12fd85d3589b17b Mon Sep 17 00:00:00 2001 From: maeken Date: Wed, 6 Mar 2024 07:44:18 +0000 Subject: [PATCH 6/6] add OpportunityRoleInputPluginTest --- .../delegate/OpportunityInputPluginTest.java | 2 +- .../OpportunityRoleInputPluginTest.java | 118 ++++++++++++++++++ .../resources/config/opportunity_role.yaml | 6 + .../fixtures/opportunity_role_response.json | 30 +++++ 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPluginTest.java create mode 100644 src/test/resources/config/opportunity_role.yaml create mode 100644 src/test/resources/fixtures/opportunity_role_response.json diff --git a/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java b/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java index 4768d37..e868cd3 100644 --- a/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java +++ b/src/test/java/org/embulk/input/marketo/delegate/OpportunityInputPluginTest.java @@ -67,7 +67,7 @@ public void testValidPluginTask() } @Test - public void testCustomObjectFilterTypeError() + public void testOpportunityFilterTypeError() { PluginTask pluginTask = mapTask(configSource.set("opportunity_filter_type", "")); Assert.assertThrows(ConfigException.class, () -> opportunityInputPlugin.validateInputTask(pluginTask)); diff --git a/src/test/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPluginTest.java b/src/test/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPluginTest.java new file mode 100644 index 0000000..64d8717 --- /dev/null +++ b/src/test/java/org/embulk/input/marketo/delegate/OpportunityRoleInputPluginTest.java @@ -0,0 +1,118 @@ +package org.embulk.input.marketo.delegate; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.embulk.EmbulkTestRuntime; +import org.embulk.base.restclient.ServiceResponseMapper; +import org.embulk.base.restclient.record.RecordImporter; +import org.embulk.base.restclient.record.ValueLocator; +import org.embulk.config.ConfigException; +import org.embulk.config.ConfigLoader; +import org.embulk.config.ConfigSource; +import org.embulk.input.marketo.rest.MarketoRestClient; +import org.embulk.input.marketo.rest.RecordPagingIterable; +import org.embulk.spi.PageBuilder; +import org.embulk.spi.Schema; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.List; + +import static org.embulk.input.marketo.MarketoUtilsTest.CONFIG_MAPPER; +import static org.embulk.input.marketo.delegate.OpportunityRoleInputPlugin.PluginTask; +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class OpportunityRoleInputPluginTest +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Rule + public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime(); + + private OpportunityRoleInputPlugin opportunityRoleInputPlugin; + + private ConfigSource configSource; + + private MarketoRestClient mockMarketoRestClient; + + @Before + public void setUp() throws Exception + { + opportunityRoleInputPlugin = spy(new OpportunityRoleInputPlugin()); + ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class); + configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/opportunity_role.yaml")); + mockMarketoRestClient = mock(MarketoRestClient.class); + doReturn(mockMarketoRestClient).when(opportunityRoleInputPlugin).createMarketoRestClient(any(PluginTask.class)); + } + + @Test + public void testValidPluginTask() + { + PluginTask pluginTask = mapTask(configSource); + opportunityRoleInputPlugin.validateInputTask(pluginTask); + } + + @Test + public void testOpportunityRoleFilterTypeError() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_role_filter_type", "")); + Assert.assertThrows(ConfigException.class, () -> opportunityRoleInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testEmptyStringFilterValues() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_role_filter_values", "")); + Assert.assertThrows(ConfigException.class, () -> opportunityRoleInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testAllEmptyStringFilterValues() + { + PluginTask pluginTask = mapTask(configSource.set("opportunity_role_filter_values", ",, , ")); + Assert.assertThrows(ConfigException.class, () -> opportunityRoleInputPlugin.validateInputTask(pluginTask)); + } + + @Test + public void testRun() throws IOException + { + RecordPagingIterable mockRecordPagingIterable = mock(RecordPagingIterable.class); + JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class); + List objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/opportunity_role_response.json"), javaType); + when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator()); + when(mockMarketoRestClient.getOpportunityRoles(anyString(), anyString())).thenReturn(mockRecordPagingIterable); + + PluginTask task = mapTask(configSource); + ServiceResponseMapper mapper = opportunityRoleInputPlugin.buildServiceResponseMapper(task); + RecordImporter recordImporter = mapper.createRecordImporter(); + PageBuilder mockPageBuilder = mock(PageBuilder.class); + opportunityRoleInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder); + verify(mockMarketoRestClient, times(1)).getOpportunityRoles(anyString(), anyString()); + + Schema schema = mapper.getEmbulkSchema(); + ArgumentCaptor stringArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(mockPageBuilder, times(3)).setString(eq(schema.lookupColumn("externalOpportunityId")), stringArgumentCaptor.capture()); + List allValues = stringArgumentCaptor.getAllValues(); + assertArrayEquals(new String[]{"externalOpportunityId_1", "externalOpportunityId_2", "externalOpportunityId_3"}, allValues.toArray()); + } + + private PluginTask mapTask(ConfigSource config) + { + return CONFIG_MAPPER.map(config, PluginTask.class); + } +} + diff --git a/src/test/resources/config/opportunity_role.yaml b/src/test/resources/config/opportunity_role.yaml new file mode 100644 index 0000000..8430ec3 --- /dev/null +++ b/src/test/resources/config/opportunity_role.yaml @@ -0,0 +1,6 @@ +account_id: account_id +client_id: client_id +client_secret: client_secret +opportunity_role_filter_type: opportunity_filter_type +opportunity_role_filter_values: opportunity_filter_values + diff --git a/src/test/resources/fixtures/opportunity_role_response.json b/src/test/resources/fixtures/opportunity_role_response.json new file mode 100644 index 0000000..9780b0b --- /dev/null +++ b/src/test/resources/fixtures/opportunity_role_response.json @@ -0,0 +1,30 @@ +[ + { + "seq": 0, + "marketoGUID": "marketoGUID_1", + "role": null, + "externalOpportunityId": "externalOpportunityId_1", + "createdAt": "2022-12-06T04:11:14Z", + "updatedAt": "2022-12-06T04:11:14Z", + "leadId": 1 + }, + { + "seq": 1, + "marketoGUID": "marketoGUID_2", + "role": "Closing", + "externalOpportunityId": "externalOpportunityId_2", + "createdAt": "2022-12-06T04:11:14Z", + "updatedAt": "2022-12-06T04:11:14Z", + "leadId": 22 + }, + { + "seq": 2, + "marketoGUID": "marketoGUID_3", + "role": "Introducing", + "externalOpportunityId": "externalOpportunityId_3", + "createdAt": "2022-12-06T04:11:14Z", + "updatedAt": "2022-12-06T04:11:14Z", + "leadId": 33 + } +] +