From 69402951dbbdd0ad1c8d0fd32e49db05a039b4bf Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Thu, 22 Nov 2018 22:03:21 -0300 Subject: [PATCH 1/7] First implementation of SlicingDice backend --- .../slicingdice/SlicingDiceBackend.java | 4 ++ .../slicingdice/SlicingDiceBackendImpl.java | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java create mode 100644 cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java new file mode 100644 index 000000000..c4c0e3778 --- /dev/null +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java @@ -0,0 +1,4 @@ +package com.telefonica.iot.cygnus.backends.slicingdice; + +public interface SlicingDiceBackend { +} diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java new file mode 100644 index 000000000..81f2a5918 --- /dev/null +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java @@ -0,0 +1,37 @@ +package com.telefonica.iot.cygnus.backends.slicingdice; + +import com.telefonica.iot.cygnus.backends.http.HttpBackend; + +public class SlicingDiceBackendImpl extends HttpBackend implements SlicingDiceBackend { + + // this is the SlicingDice host, the user will not be able to change it + private static final String SLICING_DICE_HOST = "api.slicingdice.com"; + + // this is the SlicingDice port, the user will not be able to change it + private static final String SLICING_DICE_PORT = "443"; + + private static final boolean IS_SSL = true; + + // max connections used by SlicingDice + private static final int MAX_CONNECTIONS = 50; + + // database key used to access SlicingDice API + private final String databaseKey; + + // boolean that indicates if we can auto create columns and dimensions if needed + private final boolean autoCreate; + + /** + * Constructor for the SlicingDice backend. + * + * @param databaseKey - the api key used to connect to the SlicingDice account + * @param autoCreate - if true we will auto create fields and dimensions when the request + * arrives + */ + public SlicingDiceBackendImpl(final String databaseKey, final boolean autoCreate) { + super(SLICING_DICE_HOST, SLICING_DICE_PORT, IS_SSL, false, null, null, null, null, MAX_CONNECTIONS, MAX_CONNECTIONS); + + this.databaseKey = databaseKey; + this.autoCreate = autoCreate; + } +} From 673e7846ebafdabe8b008d1d04b6865146c7b4e9 Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Sat, 24 Nov 2018 02:07:50 -0300 Subject: [PATCH 2/7] Add SlicingDice sink and backend support --- .../slicingdice/SlicingDiceBackend.java | 23 + .../slicingdice/SlicingDiceBackendImpl.java | 60 ++- .../SlicingDiceBackendImplTest.java | 88 ++++ .../iot/cygnus/sinks/NGSISlicingDiceSink.java | 466 ++++++++++++++++++ .../iot/cygnus/utils/NGSICharsets.java | 45 ++ .../iot/cygnus/utils/NGSIConstants.java | 6 +- .../iot/cygnus/utils/NGSIUtils.java | 9 +- .../cygnus/sinks/NGSISlicingDiceSinkTest.java | 230 +++++++++ 8 files changed, 918 insertions(+), 9 deletions(-) create mode 100644 cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java create mode 100644 cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java create mode 100644 cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java index c4c0e3778..7dc399cd5 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java @@ -1,4 +1,27 @@ package com.telefonica.iot.cygnus.backends.slicingdice; +import com.telefonica.iot.cygnus.errors.CygnusBadContextData; +import com.telefonica.iot.cygnus.errors.CygnusPersistenceError; +import com.telefonica.iot.cygnus.errors.CygnusRuntimeError; + public interface SlicingDiceBackend { + + /** + * Creates the necessary columns on SlicingDice dimension. + * @param fieldNames + * @throws CygnusRuntimeError + * @throws CygnusPersistenceError + */ + void createColumns(String fieldNames) throws CygnusRuntimeError, CygnusPersistenceError; + + /** + * Insert already processed context data into the given dimension. + * @param valuesForInsert + * @throws com.telefonica.iot.cygnus.errors.CygnusBadContextData + * @throws com.telefonica.iot.cygnus.errors.CygnusRuntimeError + * @throws com.telefonica.iot.cygnus.errors.CygnusPersistenceError + */ + void insertContextData(String valuesForInsert) + throws CygnusBadContextData, CygnusRuntimeError, CygnusPersistenceError; + } diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java index 81f2a5918..ccdf4037d 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java @@ -1,9 +1,20 @@ package com.telefonica.iot.cygnus.backends.slicingdice; import com.telefonica.iot.cygnus.backends.http.HttpBackend; +import com.telefonica.iot.cygnus.backends.http.JsonResponse; +import com.telefonica.iot.cygnus.errors.CygnusBadContextData; +import com.telefonica.iot.cygnus.errors.CygnusPersistenceError; +import com.telefonica.iot.cygnus.errors.CygnusRuntimeError; +import com.telefonica.iot.cygnus.log.CygnusLogger; +import java.util.ArrayList; +import org.apache.http.Header; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; public class SlicingDiceBackendImpl extends HttpBackend implements SlicingDiceBackend { + private static final CygnusLogger LOGGER = new CygnusLogger(SlicingDiceBackendImpl.class); + // this is the SlicingDice host, the user will not be able to change it private static final String SLICING_DICE_HOST = "api.slicingdice.com"; @@ -18,20 +29,55 @@ public class SlicingDiceBackendImpl extends HttpBackend implements SlicingDiceBa // database key used to access SlicingDice API private final String databaseKey; - // boolean that indicates if we can auto create columns and dimensions if needed - private final boolean autoCreate; - /** * Constructor for the SlicingDice backend. * * @param databaseKey - the api key used to connect to the SlicingDice account - * @param autoCreate - if true we will auto create fields and dimensions when the request - * arrives */ - public SlicingDiceBackendImpl(final String databaseKey, final boolean autoCreate) { + public SlicingDiceBackendImpl(final String databaseKey) { super(SLICING_DICE_HOST, SLICING_DICE_PORT, IS_SSL, false, null, null, null, null, MAX_CONNECTIONS, MAX_CONNECTIONS); this.databaseKey = databaseKey; - this.autoCreate = autoCreate; } + + @Override + public void createColumns(final String columnsToCreate) throws CygnusRuntimeError, CygnusPersistenceError { + final String urlPath = "/column/"; + + // do the SlicingDice request + final JsonResponse res = doSlicingDiceRequest("POST", urlPath, columnsToCreate); + + // check the status + if (res.getStatusCode() == 200) { + LOGGER.debug("Successful column creation"); + } else { + throw new CygnusPersistenceError("Could not create the columns, " + + "statusCode=" + res.getStatusCode() + ")"); + } // if else + } // createColumns + + @Override + public void insertContextData(final String valuesForInsert) throws CygnusBadContextData, CygnusRuntimeError, CygnusPersistenceError { + final String urlPath = "/insert/"; + + // do the SlicingDice request + final JsonResponse res = doSlicingDiceRequest("POST", urlPath, valuesForInsert); + + // check the status + if (res.getStatusCode() == 200) { + LOGGER.debug("Successful inserted data on SlicingDice"); + } else { + throw new CygnusPersistenceError("Could not create the columns, " + + "statusCode=" + res.getStatusCode() + ")"); + } // if else + } // insertContextData + + private JsonResponse doSlicingDiceRequest(final String method, final String urlPath, + final String jsonString) + throws CygnusPersistenceError, CygnusRuntimeError { + ArrayList
headers = new ArrayList<>(); + headers.add(new BasicHeader("Authorization", databaseKey)); + headers.add(new BasicHeader("Content-Type", "application/json")); + return doRequest(method, urlPath, true, headers, new StringEntity(jsonString, "UTF-8")); + } // doSlicingDiceRequest } diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java new file mode 100644 index 000000000..92ceb1232 --- /dev/null +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java @@ -0,0 +1,88 @@ +package com.telefonica.iot.cygnus.backends.slicingdice; + +import org.apache.http.ProtocolVersion; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHttpResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; + +/** + * + * @author joaosimbiose + */ +@RunWith(MockitoJUnitRunner.class) +public class SlicingDiceBackendImplTest { + + // constants + private static final String DATABASE_KEY = "oiasdiondasidasndasomn"; + private static final String COLUMNS_TO_CREATE = "[" + + " {" + + " \"name\": \"temperature\"," + + " \"api-name\": \"temperature\"," + + " \"type\": \"decimal-event\"," + + " \"decimal-places\": 5" + + " }" + + "]"; + + private static final String VALUES_TO_INSERT = "{" + + " \"ROOM1\": {" + + " \"temperature\": {" + + " \"value\": 15.5, " + + " \"date\": \"2018-10-21T00:00:00,000Z\"" + + " }" + + " }," + + " \"auto-create\": [\"dimension\", \"column\"]" + + "}"; + + // instance to be tested + private SlicingDiceBackendImpl backend; + + // mocks + @Mock + private HttpClient mockHttpClient; + + @Before + public void setUp() throws Exception { + backend = new SlicingDiceBackendImpl(DATABASE_KEY); + final BasicHttpResponse response = new BasicHttpResponse(new ProtocolVersion("http", 1, 1), 200, "ok"); + response.setEntity(new StringEntity("{\"result\": {\"whatever\":\"whatever\"}}")); + when(mockHttpClient.execute(Mockito.any(HttpUriRequest.class))).thenReturn(response); + } + + @Test + public void testCreateColumns() { + System.out.println("Testing SlicingDiceBackendImpl.createColumns"); + + try { + backend.setHttpClient(mockHttpClient); + backend.createColumns(COLUMNS_TO_CREATE); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } finally { + Assert.assertTrue(true); + } + } + + @Test + public void testInsertContextData() { + System.out.println("Testing SlicingDiceBackendImpl.insertContextData"); + + try { + backend.setHttpClient(mockHttpClient); + backend.insertContextData(VALUES_TO_INSERT); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } finally { + Assert.assertTrue(true); + } + } +} diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java new file mode 100644 index 000000000..3d0d7c8bc --- /dev/null +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java @@ -0,0 +1,466 @@ +package com.telefonica.iot.cygnus.sinks; + +import com.telefonica.iot.cygnus.backends.slicingdice.SlicingDiceBackend; +import com.telefonica.iot.cygnus.backends.slicingdice.SlicingDiceBackendImpl; +import com.telefonica.iot.cygnus.containers.NotifyContextRequest; +import com.telefonica.iot.cygnus.errors.CygnusBadConfiguration; +import com.telefonica.iot.cygnus.errors.CygnusBadContextData; +import com.telefonica.iot.cygnus.errors.CygnusCappingError; +import com.telefonica.iot.cygnus.errors.CygnusExpiratingError; +import com.telefonica.iot.cygnus.errors.CygnusPersistenceError; +import com.telefonica.iot.cygnus.errors.CygnusRuntimeError; +import com.telefonica.iot.cygnus.interceptors.NGSIEvent; +import com.telefonica.iot.cygnus.log.CygnusLogger; +import com.telefonica.iot.cygnus.utils.CommonConstants; +import com.telefonica.iot.cygnus.utils.CommonUtils; +import com.telefonica.iot.cygnus.utils.NGSICharsets; +import com.telefonica.iot.cygnus.utils.NGSIConstants; +import com.telefonica.iot.cygnus.utils.NGSIUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.flume.Context; + +/** + * + * @author joaosimbiose + * + * SlicingDice (https://www.slicingdice.com) sink for Orion Context Broker. + * + */ +public class NGSISlicingDiceSink extends NGSISink { + + private static final CygnusLogger LOGGER = new CygnusLogger(NGSISlicingDiceSink.class); + + private String databaseKey; + private boolean autoCreate; + private SlicingDiceBackend persistenceBackend; + + public String getDatabaseKey() { + return databaseKey; + } // getDatabaseKey + + public boolean isAutoCreate() { + return autoCreate; + } // isAutoCreate + + @Override + public void configure(final Context context) { + super.configure(context); + this.databaseKey = context.getString("database_key"); + + if (this.databaseKey == null) { + invalidConfiguration = true; + LOGGER.debug("[" + this.getName() + "] Invalid configuration (database_key=" + databaseKey + ")" + + " -- Must be different than null"); + } else { + LOGGER.debug("[" + this.getName() + "] Reading configuration (api_key=" + databaseKey + ")"); + } + this.autoCreate = context.getBoolean("auto_create", false); + LOGGER.debug("[" + this.getName() + "] Reading configuration (auto_create=" + autoCreate + ")"); + } // configure + + @Override + void persistBatch(final NGSIBatch batch) throws CygnusBadConfiguration, CygnusBadContextData, CygnusRuntimeError, CygnusPersistenceError { + if (batch == null) { + LOGGER.debug("[" + this.getName() + "] Null batch, nothing to do"); + return; + } // if + + // Iterate on the destinations + batch.startIterator(); + + while (batch.hasNext()) { + String destination = batch.getNextDestination(); + LOGGER.debug("[" + this.getName() + "] Processing sub-batch regarding the " + + destination + " destination"); + + // Get the events within the current sub-batch + ArrayList events = batch.getNextEvents(); + + SlicingDiceAggregator aggregator = getAggregator(); + aggregator.initialize(events.get(0)); + + for (NGSIEvent event : events) { + aggregator.aggregate(event); + } // for + + // Persist the aggregation + persistAggregation(aggregator); + batch.setNextPersisted(true); + } // while + } // persistBatch + + @Override + void capRecords(final NGSIBatch batch, final long maxRecords) throws CygnusCappingError { + } + + @Override + void expirateRecords(final long expirationTime) throws CygnusExpiratingError { + } + + @Override + public void start() { + try { + this.persistenceBackend = new SlicingDiceBackendImpl(databaseKey); + LOGGER.debug("[" + this.getName() + "] SlicingDice persistence backend created"); + } catch (Exception e) { + LOGGER.error("Error while creating the SlicingDice persistence backend. Details=" + + e.getMessage()); + } // try catch + + super.start(); + } // start + + /** + * Class for aggregating. + */ + private abstract class SlicingDiceAggregator { + + // command to create string and integer columns on SlicingDice + private static final String COLUMN_CREATION_COMMAND = + "{\"name\": \"%s\", \"api-name\": \"%s\", \"type\": \"%s\", " + + "\"description\": \"Created using CYGNUS.\"}"; + + // command to create decimal columns on SlicingDice + private static final String DECIMAL_COLUMN_CREATION_COMMAND = + "{\"name\": \"%s\", \"api-name\": \"%s\", \"type\": \"%s\", " + + "\"description\": \"Created using CYGNUS.\", \"decimal-places\": 5}"; + + // object containing the aggregted data + protected LinkedHashMap> aggregation; + protected LinkedHashMap fieldToType; + protected ArrayList timeSeries; + + protected String service; + protected String servicePathForData; + protected String servicePathForNaming; + protected String entityForNaming; + protected String attribute; + protected String dimensionName; + + public SlicingDiceAggregator() { + aggregation = new LinkedHashMap<>(); + fieldToType = new LinkedHashMap<>(); + timeSeries = new ArrayList<>(); + } // SlicingDiceAggregator + + public String getTableName(boolean enableLowercase) { + if (enableLowercase) { + return dimensionName.toLowerCase(); + } else { + return dimensionName; + } // if else + } // getTableName + + public String getValuesForInsert() { + final StringBuilder valuesForInsert = new StringBuilder("{"); + final int numEvents = aggregation.get(NGSIConstants.FIWARE_SERVICE_PATH).size(); + + for (int i = 0; i < numEvents; i++) { + // get entity id + final String entityId = aggregation.get(NGSIConstants.ENTITY_ID).get(i); + + if (i == 0) { + valuesForInsert.append("\"").append(entityId).append("\": {"); + } else { + valuesForInsert.append(",").append(" \"").append(entityId).append("\": {"); + } // if else + + // add dimension to the entity + valuesForInsert.append("\"dimension\": \"").append(dimensionName).append("\""); + + for (final Map.Entry> aggregationEntrySet : aggregation.entrySet()) { + final String columnName = aggregationEntrySet.getKey(); + + // we already used saved the entity id below so we can safely ignore it now + if (columnName.equals(NGSIConstants.ENTITY_ID)) { + continue; + } // if + + final String value = aggregationEntrySet.getValue().get(i); + final boolean isString = fieldToType.get(columnName).equals("string-event"); + final String date = timeSeries.get(i); + if (isString) { + valuesForInsert.append(", \"").append(columnName).append("\": [") + .append("{\"value\": \"").append(value).append("\", \"date\": \"") + .append(date).append("\"}]"); + } else { + valuesForInsert.append(", \"").append(columnName).append("\": [") + .append("{\"value\": ").append(value).append(", \"date\": \"") + .append(date).append("\"}]"); + } // if else + } // for + valuesForInsert.append("}"); + } // for + + if (autoCreate) { + valuesForInsert.append("\"auto-create\": [\"dimension\", \"column\"]"); + } else { + valuesForInsert.append("\"auto-create\": [\"dimension\"]"); + } // if else + + return valuesForInsert + "}"; + } // getValuesForInsert + + public String getFieldsForCreate() { + boolean first = true; + final StringBuilder fieldsForCreate = new StringBuilder("["); + + for (final Map.Entry fieldToTypeEntry : fieldToType.entrySet()) { + final String columnName = fieldToTypeEntry.getKey(); + final String type = fieldToTypeEntry.getValue(); + + final String columnCommand; + if (type.contains("decimal")) { + columnCommand = String.format(DECIMAL_COLUMN_CREATION_COMMAND, columnName, + columnName, type); + } else { + columnCommand = String.format(COLUMN_CREATION_COMMAND, columnName, columnName, + type); + } + + if (first) { + fieldsForCreate.append(columnCommand); + first = false; + } else { + fieldsForCreate.append(",").append(columnCommand); + } // if else + } // for + + return fieldsForCreate + "]"; + } // getFieldsForCreate + + public void initialize(NGSIEvent event) throws CygnusBadConfiguration { + service = event.getServiceForNaming(enableNameMappings); + servicePathForData = event.getServicePathForData(); + servicePathForNaming = event.getServicePathForNaming(enableGrouping, enableNameMappings); + entityForNaming = event.getEntityForNaming(enableGrouping, enableNameMappings, enableEncoding); + attribute = event.getAttributeForNaming(enableNameMappings); + dimensionName = buildDimensionName(servicePathForNaming, entityForNaming, attribute); + } // initialize + + public abstract void aggregate(NGSIEvent cygnusEvent); + + } + + /** + * Class for aggregating batches in column mode. + */ + private class ColumnAggregator extends NGSISlicingDiceSink.SlicingDiceAggregator { + + @Override + public void initialize(NGSIEvent cygnusEvent) throws CygnusBadConfiguration { + super.initialize(cygnusEvent); + + // particular initialization + aggregation.put(NGSIConstants.FIWARE_SERVICE_PATH, new ArrayList()); + aggregation.put(NGSIConstants.ENTITY_ID, new ArrayList()); + + // iterate on all this context element attributes, if there are attributes + ArrayList contextAttributes = cygnusEvent.getContextElement().getAttributes(); + + if (contextAttributes == null || contextAttributes.isEmpty()) { + return; + } // if + + for (NotifyContextRequest.ContextAttribute contextAttribute : contextAttributes) { + String attrName = contextAttribute.getName(); + aggregation.put(attrName, new ArrayList()); + } // for + } // initialize + + @Override + public void aggregate(NGSIEvent event) { + // Number of previous values + int numPreviousValues = aggregation.get(NGSIConstants.FIWARE_SERVICE_PATH).size(); + + // Get the event headers + long recvTimeTs = event.getRecvTimeTs(); + String recvTime = CommonUtils.getHumanReadable(recvTimeTs, false); + + // get the event body + NotifyContextRequest.ContextElement contextElement = event.getContextElement(); + String entityId = contextElement.getId(); + String entityType = contextElement.getType(); + LOGGER.debug("[" + getName() + "] Processing context element (id=" + entityId + ", type=" + + entityType + ")"); + + // Iterate on all this context element attributes, if there are attributes + ArrayList contextAttributes = contextElement.getAttributes(); + + if (contextAttributes == null || contextAttributes.isEmpty()) { + LOGGER.warn("No attributes within the notified entity, nothing is done (id=" + entityId + + ", type=" + entityType + ")"); + return; + } // if + + timeSeries.add(recvTime); + aggregation.get(NGSIConstants.FIWARE_SERVICE_PATH).add(servicePathForData); + aggregation.get(NGSIConstants.ENTITY_ID).add(entityId); + + for (NotifyContextRequest.ContextAttribute contextAttribute : contextAttributes) { + String attrName = encode(contextAttribute.getName(), false, true); + String attrType = contextAttribute.getType(); + String slicingDiceType = translateType(attrType); + String attrValue = contextAttribute.getContextValue(false); + LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type=" + + attrType + ")"); + + fieldToType.put(attrName, slicingDiceType); + + // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, + // add an empty value for all previous rows + if (aggregation.containsKey(attrName)) { + aggregation.get(attrName).add(attrValue); + } else { + ArrayList values = new ArrayList<>(Collections.nCopies(numPreviousValues, "")); + values.add(attrValue); + aggregation.put(attrName, values); + } // if else + } // for + + // Iterate on all the aggregations, checking for not updated attributes; add an empty value if missing + for (String key : aggregation.keySet()) { + ArrayList values = aggregation.get(key); + + if (values.size() == numPreviousValues) { + values.add(""); + } // if + } // for + } // aggregate + + /** + * Translate the attribute type to the SlicingDice format so we can create the column on + * SlicingDice. + * @param type - the type to be translated + * @return the translated type to the SlicingDice format + */ + private String translateType(final String type) { + final String lowerCaseType = type.toLowerCase(); + switch (lowerCaseType) { + case "float": + case "number": + return "decimal-event"; + case "integer": + return "integer-event"; + default: + return "string-event"; + } + } // translateType + + } // ColumnAggregator + + private SlicingDiceAggregator getAggregator() { + return new NGSISlicingDiceSink.ColumnAggregator(); + } // getAggregator + + private void persistAggregation(NGSISlicingDiceSink.SlicingDiceAggregator aggregator) + throws CygnusPersistenceError, CygnusRuntimeError, CygnusBadContextData { + String fieldsForCreate = aggregator.getFieldsForCreate(); + String valuesForInsert = aggregator.getValuesForInsert(); + String tableName = aggregator.getTableName(enableLowercase); + + LOGGER.info("[" + this.getName() + "] Persisting data at NGSIMySQLSink. Table (" + + tableName + "), Fields (" + fieldsForCreate + "), Values (" + + valuesForInsert + ")"); + + // creating the needed columns on SlicingDice, the dimension will be automatically created + if (!autoCreate) { + persistenceBackend.createColumns(fieldsForCreate); + } // if + + persistenceBackend.insertContextData(valuesForInsert); + } // persistAggregation + + /** + * Creates a SlicingDice dimension name given the FIWARE service path, the entity and the attribute. + * @param servicePath + * @param entity + * @param attribute + * @return The SlicingDice dimension name + * @throws CygnusBadConfiguration + */ + protected String buildDimensionName(final String servicePath, final String entity, + final String attribute) throws CygnusBadConfiguration { + String name; + + if (enableEncoding) { + switch(dataModel) { + case DMBYSERVICEPATH: + name = NGSICharsets.encodeSlicingDice(servicePath); + break; + case DMBYENTITY: + name = NGSICharsets.encodeSlicingDice(servicePath) + + CommonConstants.CONCATENATOR + + NGSICharsets.encodeSlicingDice(entity); + break; + case DMBYATTRIBUTE: + name = NGSICharsets.encodeSlicingDice(servicePath) + + CommonConstants.CONCATENATOR + + NGSICharsets.encodeSlicingDice(entity) + + CommonConstants.CONCATENATOR + + NGSICharsets.encodeSlicingDice(attribute); + break; + default: + throw new CygnusBadConfiguration("Unknown data model '" + dataModel.toString() + + "'. Please, use dm-by-service-path, dm-by-entity or dm-by-attribute"); + } // switch + } else { + switch(dataModel) { + case DMBYSERVICEPATH: + if (servicePath.equals("/")) { + throw new CygnusBadConfiguration("Default service path '/' cannot be used with " + + "dm-by-service-path data model"); + } // if + + name = encode(servicePath, true, false); + break; + case DMBYENTITY: + String truncatedServicePath = encode(servicePath, true, false); + name = (truncatedServicePath.isEmpty() ? "" : truncatedServicePath + '-') + + encode(entity, false, true); + break; + case DMBYATTRIBUTE: + truncatedServicePath = encode(servicePath, true, false); + name = (truncatedServicePath.isEmpty() ? "" : truncatedServicePath + '-') + + encode(entity, false, true) + + '-' + encode(attribute, false, true); + break; + default: + throw new CygnusBadConfiguration("Unknown data model '" + dataModel.toString() + + "'. Please, use DMBYSERVICEPATH, DMBYENTITY or DMBYATTRIBUTE"); + } // switch + } // if else + + if (name.length() > NGSIConstants.SLICINGDICE_MAX_NAME_LEN) { + throw new CygnusBadConfiguration("Building table name '" + name + + "' and its length is greater than " + NGSIConstants.SLICINGDICE_MAX_NAME_LEN); + } // if + + return name; + } // buildTableName + + /** + * Encodes a string replacing all the non alphanumeric characters by '-' (except by '-' and '.'). + * This should be only called when building a persistence element name, such as table names, file paths, etc. + * We had to create this specific method instead of using the existent one on NSGIUtils because + * SlicingDice doesn't accept slashes, only '-'. + * + * @param in + * @param deleteSlash + * @param encodeSlash + * @return The encoded version of the input string. + */ + public static String encode(String in, boolean deleteSlash, boolean encodeSlash) { + if (deleteSlash) { + return NGSIUtils.getENCODEPATTERN().matcher(in.substring(1)).replaceAll("-"); + } else if (encodeSlash) { + return NGSIUtils.getENCODEPATTERN().matcher(in).replaceAll("-"); + } else { + return NGSIUtils.getENCODEPATTERNSLASH().matcher(in).replaceAll("-"); + } // if else + } // encode + +} diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java index 59ba33ff0..93c6bccb8 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java @@ -162,6 +162,51 @@ public static String encodeCKAN(String in) { return out; } // encodeCKAN + + /** + * Encodes a string for SlicingDice. Only lowercase alphanumerics and - are allowed. + * @param in + * @return The encoded string + */ + public static String encodeSlicingDice(String in) { + String out = ""; + + for (int i = 0; i < in.length(); i++) { + char c = in.charAt(i); + int code = c; + + if (code >= 97 && code <= 119) { // a-w --> a-w + out += c; + } else if (c == 'x') { + String next4; + + if (i + 4 < in.length()) { + next4 = in.substring(i + 1, i + 5); + } else { + next4 = "WXYZ"; // whatever except a unicode + } // if else + + if (next4.matches("^[0-9a-fA-F]{4}$")) { // x --> xx + out += "xx"; + } else { // x --> x + out += c; + } // if else + } else if (code == 121 || code == 122) { // yz --> yz + out += c; + } else if (code >= 48 && code <= 57) { // 0-9 --> 0-9 + out += c; + } else if (c == '-') { // - --> - + out += c; + } else if (c == '=') { // = --> xffff + out += "xffff"; + } else { // --> xUNICODE + String hex = Integer.toHexString(code); + out += "x" + ("0000" + hex).substring(hex.length()); + } // else + } // for + + return out; + } // encodeSlicingDice /** * Encodes a string for MySQL. diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java index bf9c6100e..b53de79c6 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java @@ -88,7 +88,11 @@ private NGSIConstants() { //NGSIMySQLSink specific constants // http://dev.mysql.com/doc/refman/5.7/en/identifiers.html public static final int MYSQL_MAX_NAME_LEN = 64; - + + // NGSISlicingDiceSink specific constants + // https://docs.slicingdice.com/v1/docs/api-errors + public static final int SLICINGDICE_MAX_NAME_LEN = 64; + // NGSIPostgreSQLSink specific constants // http://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS public static final int POSTGRESQL_MAX_NAME_LEN = 63; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java index d4cac829c..c42af1210 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java @@ -143,5 +143,12 @@ public static ImmutablePair getGeometry(String attrValue, Strin // The attribute was not related to a geolocation return new ImmutablePair(attrValue, false); } // getGeometry - + + public static Pattern getENCODEPATTERN() { + return ENCODEPATTERN; + } + + public static Pattern getENCODEPATTERNSLASH() { + return ENCODEPATTERNSLASH; + } } // NGSIUtils diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java new file mode 100644 index 000000000..122b510b7 --- /dev/null +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java @@ -0,0 +1,230 @@ +package com.telefonica.iot.cygnus.sinks; + +import com.telefonica.iot.cygnus.errors.CygnusBadConfiguration; +import org.apache.flume.Context; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; + +import static com.telefonica.iot.cygnus.utils.CommonUtilsForTests.getTestTraceHead; +import static org.junit.Assert.assertTrue; + +/** + * + * @autor joaosimbiose + */ +@RunWith(MockitoJUnitRunner.class) +public class NGSISlicingDiceSinkTest { + + /** + * Constructor + */ + public NGSISlicingDiceSinkTest() { + LogManager.getRootLogger().setLevel(Level.FATAL); + } // NGSISlicingDiceSinkTest + + @Test + public void testConfigureGetConfiguration() { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "false"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + Assert.assertEquals(databaseKey, sink.getDatabaseKey()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'database_key=oasdisadnasoi' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'database_key=oasdisadnasoi' was not configured"); + throw e; + } // try catch + + try { + Assert.assertFalse(sink.isAutoCreate()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'auto_create=false' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'auto_create=false' was not configured"); + throw e; + } // try catch + } // testConfigureGetConfiguration + + @Test + public void testConfigureAutoCreateTrue() { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + Assert.assertTrue(sink.isAutoCreate()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'auto_create=false' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'auto_create=false' was not configured"); + throw e; + } // try catch + } // testConfigureAutoCreateTrue + + @Test + public void testInvalidConfiguration() { + final String databaseKey = null; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + assertTrue(sink.getInvalidConfiguration()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'database_key=null' detected"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - database_key=null' was not detected"); + throw e; + } // try catch + } // testConfigureAutoCreateTrue + + @Test + public void testBuildDimensionNameOnlyServicePath() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = null; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("users-a-x-15--", sink.buildDimensionName(servicePath, entity, + attribute)); + } // testBuildDimensionNameOnlyServicePath + + @Test + public void testBuildDimensionNameEntity() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-entity"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = "some_entity_x"; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("users-a-x-15---some-entity-x", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameEntity + + @Test + public void testBuildDimensionNameAttribute() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-attribute"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = "some_entity_x"; // irrelevant for this test + String attribute = "some_attribute_%"; // irrelevant for this test + Assert.assertEquals("users-a-x-15---some-entity-x-some-attribute--", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameAttribute + + @Test + public void testBuildDimensionNameEncodingOnlyServicePath() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x_15_$"; + String entity = null; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("x002fusersx005fax005fxx005f15x005fx0024", sink.buildDimensionName(servicePath, entity, + attribute)); + } // testBuildDimensionNameOnlyServicePath + + @Test + public void testBuildDimensionNameEncodingEntity() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-entity"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x"; + String entity = "some"; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("x002fusersx005fax005fxxffffsome", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameEntity + + @Test + public void testBuildDimensionNameEncodingAttribute() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-attribute"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_"; + String entity = "some_"; // irrelevant for this test + String attribute = "attribute"; // irrelevant for this test + Assert.assertEquals("x002fusersx005fxffffsomex005fxffffattribute", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameAttribute + + @Test + public void testStringEncode() { + final String[][] tests = new String[][]{ + new String[]{"temperature", "temperature"}, + new String[]{"code_review", "code-review"}, + new String[]{"columns-123", "columns-123"}, + new String[]{"columns_123", "columns-123"}, + new String[]{"some_path", "some-path"} + }; + + // with slash + for (final String[] test : tests) { + final String result = NGSISlicingDiceSink.encode("/" + test[0], true, false); + Assert.assertEquals(test[1], result); + } + + // without slash + for (final String[] test : tests) { + final String result = NGSISlicingDiceSink.encode(test[0], false, true); + Assert.assertEquals(test[1], result); + } + } // testStringEncodeWithoutSlash + + private Context createContext(final String databaseKey, final String autoCreate, + final String dataModel, final String encoding) { + Context context = new Context(); + context.put("database_key", databaseKey); + context.put("auto_create", autoCreate); + context.put("data_model", dataModel); + context.put("enable_encoding", encoding); + return context; + } // createContext +} From 9415761a828ca9de3d73557921eb12e164ac4042 Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Sat, 24 Nov 2018 14:34:42 -0300 Subject: [PATCH 3/7] Add unit tests for SlicingDice backend and sink --- .../slicingdice/SlicingDiceBackendImpl.java | 21 +- .../SlicingDiceBackendImplTest.java | 66 ++ .../iot/cygnus/sinks/NGSISlicingDiceSink.java | 23 +- .../cygnus/sinks/NGSISlicingDiceSinkTest.java | 734 +++++++++++++----- 4 files changed, 644 insertions(+), 200 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java index ccdf4037d..634d2d479 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java @@ -8,8 +8,11 @@ import com.telefonica.iot.cygnus.log.CygnusLogger; import java.util.ArrayList; import org.apache.http.Header; +import org.apache.http.HttpResponse; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; public class SlicingDiceBackendImpl extends HttpBackend implements SlicingDiceBackend { @@ -50,6 +53,17 @@ public void createColumns(final String columnsToCreate) throws CygnusRuntimeErro // check the status if (res.getStatusCode() == 200) { LOGGER.debug("Successful column creation"); + } else if (res.getStatusCode() == 400) { + final JSONArray errors = (JSONArray) res.getJsonObject().get("errors"); + final JSONObject error = (JSONObject) errors.get(0); + final Long code = (Long) error.get("code"); + + if (code == 3003) { + LOGGER.debug("Column already exists"); + } else { + throw new CygnusPersistenceError("Could not create the columns, " + + "statusCode=" + res.getStatusCode() + ")"); + } } else { throw new CygnusPersistenceError("Could not create the columns, " + "statusCode=" + res.getStatusCode() + ")"); @@ -72,7 +86,7 @@ public void insertContextData(final String valuesForInsert) throws CygnusBadCont } // if else } // insertContextData - private JsonResponse doSlicingDiceRequest(final String method, final String urlPath, + JsonResponse doSlicingDiceRequest(final String method, final String urlPath, final String jsonString) throws CygnusPersistenceError, CygnusRuntimeError { ArrayList
headers = new ArrayList<>(); @@ -80,4 +94,9 @@ private JsonResponse doSlicingDiceRequest(final String method, final String urlP headers.add(new BasicHeader("Content-Type", "application/json")); return doRequest(method, urlPath, true, headers, new StringEntity(jsonString, "UTF-8")); } // doSlicingDiceRequest + + @Override + protected JsonResponse createJsonResponse(final HttpResponse httpRes) throws CygnusRuntimeError { + return super.createJsonResponse(httpRes); + } } diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java index 92ceb1232..be9a63a9d 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java @@ -1,10 +1,19 @@ package com.telefonica.iot.cygnus.backends.slicingdice; +import com.telefonica.iot.cygnus.backends.http.JsonResponse; +import com.telefonica.iot.cygnus.errors.CygnusPersistenceError; +import com.telefonica.iot.cygnus.errors.CygnusRuntimeError; +import java.io.IOException; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseFactory; +import org.apache.http.HttpVersion; import org.apache.http.ProtocolVersion; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.DefaultHttpResponseFactory; import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -13,6 +22,9 @@ import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** @@ -72,6 +84,60 @@ public void testCreateColumns() { } } + @Test + public void testColumnAlreadyExist() throws IOException, CygnusRuntimeError, CygnusPersistenceError { + System.out.println("Testing SlicingDiceBackendImpl.createColumns"); + + HttpResponseFactory factory = new DefaultHttpResponseFactory(); + HttpResponse response = factory + .newHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, 400, null), null); + response.setHeader("Content-Type", "application/json"); + String responseStr = "{\"errors\": [{\"code\": 3003, \"message\": \"Column: Column already exists.\"}]}"; + response.setEntity(new StringEntity(responseStr)); + final JsonResponse jsonRes = backend.createJsonResponse(response); + + final SlicingDiceBackendImpl mockedBackend = mock(SlicingDiceBackendImpl.class); + when(mockedBackend.doSlicingDiceRequest(anyString(), anyString(), anyString())).thenReturn( + jsonRes); + doCallRealMethod().when(mockedBackend).createColumns(anyString()); + + try { + mockedBackend.setHttpClient(mockHttpClient); + mockedBackend.createColumns(COLUMNS_TO_CREATE); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } finally { + Assert.assertTrue(true); + } + } + + @Test + public void testColumnError400() throws IOException, CygnusRuntimeError, CygnusPersistenceError { + System.out.println("Testing SlicingDiceBackendImpl.createColumns"); + + HttpResponseFactory factory = new DefaultHttpResponseFactory(); + HttpResponse response = factory + .newHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, 400, null), null); + response.setHeader("Content-Type", "application/json"); + String responseStr = "{\"errors\": [{\"code\": 4026, \"message\": \"Query: Invalid query format. Must be a list.\"}]}"; + response.setEntity(new StringEntity(responseStr)); + final JsonResponse jsonRes = backend.createJsonResponse(response); + + final SlicingDiceBackendImpl mockedBackend = mock(SlicingDiceBackendImpl.class); + when(mockedBackend.doSlicingDiceRequest(anyString(), anyString(), anyString())).thenReturn( + jsonRes); + doCallRealMethod().when(mockedBackend).createColumns(anyString()); + + try { + mockedBackend.setHttpClient(mockHttpClient); + mockedBackend.createColumns(COLUMNS_TO_CREATE); + Assert.fail(); + } catch (final Exception e) { + // this response should call an exception because isn't a normal behavior + Assert.assertTrue(true); + } + } + @Test public void testInsertContextData() { System.out.println("Testing SlicingDiceBackendImpl.insertContextData"); diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java index 3d0d7c8bc..1ce11463f 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java @@ -121,12 +121,13 @@ private abstract class SlicingDiceAggregator { // command to create string and integer columns on SlicingDice private static final String COLUMN_CREATION_COMMAND = "{\"name\": \"%s\", \"api-name\": \"%s\", \"type\": \"%s\", " + - "\"description\": \"Created using CYGNUS.\"}"; + "\"description\": \"Created using CYGNUS.\", \"dimension\": \"%s\"}"; // command to create decimal columns on SlicingDice private static final String DECIMAL_COLUMN_CREATION_COMMAND = "{\"name\": \"%s\", \"api-name\": \"%s\", \"type\": \"%s\", " + - "\"description\": \"Created using CYGNUS.\", \"decimal-places\": 5}"; + "\"description\": \"Created using CYGNUS.\", \"decimal-places\": 5, " + + "\"dimension\": \"%s\"}"; // object containing the aggregted data protected LinkedHashMap> aggregation; @@ -180,7 +181,13 @@ public String getValuesForInsert() { } // if final String value = aggregationEntrySet.getValue().get(i); - final boolean isString = fieldToType.get(columnName).equals("string-event"); + final boolean isString; + + if (columnName.equals(NGSIConstants.FIWARE_SERVICE_PATH)) { + isString = true; + } else { + isString = fieldToType.get(columnName).equals("string-event"); + } final String date = timeSeries.get(i); if (isString) { valuesForInsert.append(", \"").append(columnName).append("\": [") @@ -215,10 +222,10 @@ public String getFieldsForCreate() { final String columnCommand; if (type.contains("decimal")) { columnCommand = String.format(DECIMAL_COLUMN_CREATION_COMMAND, columnName, - columnName, type); + columnName, type, dimensionName); } else { columnCommand = String.format(COLUMN_CREATION_COMMAND, columnName, columnName, - type); + type, dimensionName); } if (first) { @@ -297,7 +304,10 @@ public void aggregate(NGSIEvent event) { } // if timeSeries.add(recvTime); + aggregation.get(NGSIConstants.FIWARE_SERVICE_PATH).add(servicePathForData); + fieldToType.put(NGSIConstants.FIWARE_SERVICE_PATH, "string-event"); + aggregation.get(NGSIConstants.ENTITY_ID).add(entityId); for (NotifyContextRequest.ContextAttribute contextAttribute : contextAttributes) { @@ -463,4 +473,7 @@ public static String encode(String in, boolean deleteSlash, boolean encodeSlash) } // if else } // encode + public void setPersistenceBackend(final SlicingDiceBackend persistenceBackend) { + this.persistenceBackend = persistenceBackend; + } } diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java index 122b510b7..227b9eae2 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java @@ -1,22 +1,41 @@ package com.telefonica.iot.cygnus.sinks; +import com.telefonica.iot.cygnus.backends.slicingdice.SlicingDiceBackendImpl; +import com.telefonica.iot.cygnus.containers.NotifyContextRequest; import com.telefonica.iot.cygnus.errors.CygnusBadConfiguration; +import com.telefonica.iot.cygnus.interceptors.NGSIEvent; +import com.telefonica.iot.cygnus.utils.CommonConstants; +import com.telefonica.iot.cygnus.utils.NGSIConstants; +import com.telefonica.iot.cygnus.utils.NGSIUtilsForTests; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.flume.Context; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import static com.telefonica.iot.cygnus.utils.CommonUtilsForTests.getTestTraceHead; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * * @autor joaosimbiose */ -@RunWith(MockitoJUnitRunner.class) +@RunWith(Enclosed.class) public class NGSISlicingDiceSinkTest { /** @@ -26,200 +45,527 @@ public NGSISlicingDiceSinkTest() { LogManager.getRootLogger().setLevel(Level.FATAL); } // NGSISlicingDiceSinkTest - @Test - public void testConfigureGetConfiguration() { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "false"; - final String dataModel = "dm-by-service-path"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - try { - Assert.assertEquals(databaseKey, sink.getDatabaseKey()); - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- OK - 'database_key=oasdisadnasoi' was configured"); - } catch (AssertionError e) { - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- FAIL - 'database_key=oasdisadnasoi' was not configured"); - throw e; - } // try catch - - try { - Assert.assertFalse(sink.isAutoCreate()); - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- OK - 'auto_create=false' was configured"); - } catch (AssertionError e) { - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- FAIL - 'auto_create=false' was not configured"); - throw e; - } // try catch - } // testConfigureGetConfiguration - - @Test - public void testConfigureAutoCreateTrue() { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-service-path"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - try { - Assert.assertTrue(sink.isAutoCreate()); - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- OK - 'auto_create=false' was configured"); - } catch (AssertionError e) { - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- FAIL - 'auto_create=false' was not configured"); - throw e; - } // try catch - } // testConfigureAutoCreateTrue - - @Test - public void testInvalidConfiguration() { - final String databaseKey = null; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-service-path"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - try { - assertTrue(sink.getInvalidConfiguration()); - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- OK - 'database_key=null' detected"); - } catch (AssertionError e) { - System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") - + "- FAIL - database_key=null' was not detected"); - throw e; - } // try catch - } // testConfigureAutoCreateTrue - - @Test - public void testBuildDimensionNameOnlyServicePath() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-service-path"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - String servicePath = "/users_a_x_15_$"; - String entity = null; // irrelevant for this test - String attribute = null; // irrelevant for this test - Assert.assertEquals("users-a-x-15--", sink.buildDimensionName(servicePath, entity, - attribute)); - } // testBuildDimensionNameOnlyServicePath - - @Test - public void testBuildDimensionNameEntity() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-entity"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - String servicePath = "/users_a_x_15_$"; - String entity = "some_entity_x"; // irrelevant for this test - String attribute = null; // irrelevant for this test - Assert.assertEquals("users-a-x-15---some-entity-x", sink.buildDimensionName( - servicePath, entity, attribute)); - } // testBuildDimensionNameEntity - - @Test - public void testBuildDimensionNameAttribute() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-attribute"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); - - String servicePath = "/users_a_x_15_$"; - String entity = "some_entity_x"; // irrelevant for this test - String attribute = "some_attribute_%"; // irrelevant for this test - Assert.assertEquals("users-a-x-15---some-entity-x-some-attribute--", sink.buildDimensionName( - servicePath, entity, attribute)); - } // testBuildDimensionNameAttribute - - @Test - public void testBuildDimensionNameEncodingOnlyServicePath() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-service-path"; - final String encoding = "true"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); - - String servicePath = "/users_a_x_15_$"; - String entity = null; // irrelevant for this test - String attribute = null; // irrelevant for this test - Assert.assertEquals("x002fusersx005fax005fxx005f15x005fx0024", sink.buildDimensionName(servicePath, entity, - attribute)); - } // testBuildDimensionNameOnlyServicePath - - @Test - public void testBuildDimensionNameEncodingEntity() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-entity"; - final String encoding = "true"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); - - String servicePath = "/users_a_x"; - String entity = "some"; // irrelevant for this test - String attribute = null; // irrelevant for this test - Assert.assertEquals("x002fusersx005fax005fxxffffsome", sink.buildDimensionName( - servicePath, entity, attribute)); - } // testBuildDimensionNameEntity - - @Test - public void testBuildDimensionNameEncodingAttribute() throws CygnusBadConfiguration { - final String databaseKey = "oasdisadnasoi"; - final String isAutoCreate = "true"; - final String dataModel = "dm-by-attribute"; - final String encoding = "true"; - - final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); - sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); - - String servicePath = "/users_"; - String entity = "some_"; // irrelevant for this test - String attribute = "attribute"; // irrelevant for this test - Assert.assertEquals("x002fusersx005fxffffsomex005fxffffattribute", sink.buildDimensionName( - servicePath, entity, attribute)); - } // testBuildDimensionNameAttribute - - @Test - public void testStringEncode() { - final String[][] tests = new String[][]{ - new String[]{"temperature", "temperature"}, - new String[]{"code_review", "code-review"}, - new String[]{"columns-123", "columns-123"}, - new String[]{"columns_123", "columns-123"}, - new String[]{"some_path", "some-path"} - }; - - // with slash - for (final String[] test : tests) { - final String result = NGSISlicingDiceSink.encode("/" + test[0], true, false); - Assert.assertEquals(test[1], result); + public static class ConfigureTest { + @Test + public void testConfigureGetConfiguration() { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "false"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + Assert.assertEquals(databaseKey, sink.getDatabaseKey()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'database_key=oasdisadnasoi' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'database_key=oasdisadnasoi' was not configured"); + throw e; + } // try catch + + try { + Assert.assertFalse(sink.isAutoCreate()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'auto_create=false' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'auto_create=false' was not configured"); + throw e; + } // try catch + } // testConfigureGetConfiguration + + @Test + public void testConfigureAutoCreateTrue() { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + Assert.assertTrue(sink.isAutoCreate()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'auto_create=false' was configured"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - 'auto_create=false' was not configured"); + throw e; + } // try catch + } // testConfigureAutoCreateTrue + + @Test + public void testInvalidConfiguration() { + final String databaseKey = null; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + try { + assertTrue(sink.getInvalidConfiguration()); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- OK - 'database_key=null' detected"); + } catch (AssertionError e) { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.configure]") + + "- FAIL - database_key=null' was not detected"); + throw e; + } // try catch + } // testConfigureAutoCreateTrue + } // ConfigureTest + + public static class AuxiliaryMethodsTest { + @Test + public void testBuildDimensionNameOnlyServicePath() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = null; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("users-a-x-15--", sink.buildDimensionName(servicePath, entity, + attribute)); + } // testBuildDimensionNameOnlyServicePath + + @Test + public void testBuildDimensionNameEntity() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-entity"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = "some_entity_x"; + String attribute = null; // irrelevant for this test + Assert.assertEquals("users-a-x-15---some-entity-x", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameEntity + + @Test + public void testBuildDimensionNameAttribute() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-attribute"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, null)); + + String servicePath = "/users_a_x_15_$"; + String entity = "some_entity_x"; + String attribute = "some_attribute_%"; + Assert.assertEquals("users-a-x-15---some-entity-x-some-attribute--", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameAttribute + + @Test + public void testBuildDimensionNameEncodingOnlyServicePath() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x_15_$"; + String entity = null; // irrelevant for this test + String attribute = null; // irrelevant for this test + Assert.assertEquals("x002fusersx005fax005fxx005f15x005fx0024", sink.buildDimensionName(servicePath, entity, + attribute)); + } // testBuildDimensionNameEncodingOnlyServicePath + + @Test + public void testBuildDimensionNameEncodingEntity() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-entity"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x"; + String entity = "some"; + String attribute = null; // irrelevant for this test + Assert.assertEquals("x002fusersx005fax005fxxffffsome", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameEncodingEntity + + @Test + public void testBuildDimensionNameEncodingAttribute() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-attribute"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_"; + String entity = "some_"; + String attribute = "attribute"; + Assert.assertEquals("x002fusersx005fxffffsomex005fxffffattribute", sink.buildDimensionName( + servicePath, entity, attribute)); + } // testBuildDimensionNameEncodingAttribute + + @Test + public void testBuildDimensionNameLengthOnlyServicePath() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x_15__some_length_name_for_the_service_path"; + String entity = null; // irrelevant for this test + String attribute = null; // irrelevant for this test + try { + sink.buildDimensionName(servicePath, entity, attribute); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- FAIL - A table name length greater than 64 characters has not been detected"); + Assert.fail(); + } catch (final Exception e) { + Assert.assertTrue(true); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- OK - A table name length greater than 64 characters has been detected"); + } + } // testBuildDimensionNameLengthOnlyServicePath + + @Test + public void testBuildDimensionNameLengthEntity() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-entity"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_a_x"; + String entity = "some_giant_name_on_the_entity"; + String attribute = null; // irrelevant for this test + try { + sink.buildDimensionName(servicePath, entity, attribute); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- FAIL - A table name length greater than 64 characters has not been detected"); + Assert.fail(); + } catch (final Exception e) { + Assert.assertTrue(true); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- OK - A table name length greater than 64 characters has been detected"); + } + } // testBuildDimensionNameLengthEntity + + @Test + public void testBuildDimensionNameLengthAttribute() throws CygnusBadConfiguration { + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-attribute"; + final String encoding = "true"; + + final NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + String servicePath = "/users_"; + String entity = "some_"; + String attribute = "some_giant_attribute_name_for_attribute"; + try { + sink.buildDimensionName(servicePath, entity, attribute); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- FAIL - A table name length greater than 64 characters has not been detected"); + Assert.fail(); + } catch (final Exception e) { + Assert.assertTrue(true); + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.buildDimensionName]") + + "- OK - A table name length greater than 64 characters has been detected"); + } + } // testBuildDimensionNameLengthAttribute + + @Test + public void testStringEncode() { + final String[][] tests = new String[][]{ + new String[]{"temperature", "temperature"}, + new String[]{"code_review", "code-review"}, + new String[]{"columns-123", "columns-123"}, + new String[]{"columns_123", "columns-123"}, + new String[]{"some_path", "some-path"} + }; + + // with slash + for (final String[] test : tests) { + final String result = NGSISlicingDiceSink.encode("/" + test[0], true, false); + Assert.assertEquals(test[1], result); + } + + // without slash + for (final String[] test : tests) { + final String result = NGSISlicingDiceSink.encode(test[0], false, true); + Assert.assertEquals(test[1], result); + } + } // testStringEncodeWithoutSlash + } // AuxiliaryMethodsTest + + public static class PersistBatchTest { + + private static final String DATABASE_KEY = "oiasdiondasidasndasomn"; + + @Mock + private SlicingDiceBackendImpl mockBackend; + + @Captor + private ArgumentCaptor idxCaptor; + + private final String contextElementStr1 = "" + + "{" + + "\"attributes\" : [" + + "{" + + "\"name\" : \"temperature\"," + + "\"type\" : \"number\"," + + "\"value\" : \"26.5\"" + + "}" + + "]," + + "\"type\" : \"Room\"," + + "\"isPattern\" : \"false\"," + + "\"id\" : \"Room1\"" + + "}"; + private final String contextElementStr2 = "" + + "{" + + "\"attributes\" : [" + + "{" + + "\"name\" : \"temperature\"," + + "\"type\" : \"number\"," + + "\"value\" : \"26.5\"" + + "}," + + "{" + + "\"name\" : \"roomtype\"," + + "\"type\" : \"string\"," + + "\"value\" : \"\"" + + "}" + + "]," + + "\"type\" : \"Room\"," + + "\"isPattern\" : \"false\"," + + "\"id\" : \"Room1\"" + + "}"; + private final String contextElementStr3 = "" + + "{" + + "\"attributes\" : [" + + "{" + + "\"name\" : \"temperature\"," + + "\"type\" : \"number\"," + + "\"value\" : \"26.5\"," + + "\"metadatas\" : [" + + "{" + + "\"name\" : \"TimeInstant\"," + + "\"type\" : \"ISO8601\"," + + "\"value\" : \"2018-01-02T03:04:05.678+0900\"" + + "}" + + "]" + + "}," + + "{" + + "\"name\" : \"roomtype\"," + + "\"type\" : \"string\"," + + "\"value\" : \"single\"," + + "\"metadatas\" : [" + + "{" + + "\"name\" : \"TimeInstant\"," + + "\"type\" : \"ISO8601\"," + + "\"value\" : \"2018-01-02T03:04:05.678+0900\"" + + "}" + + "]" + + "}," + + "{" + + "\"name\" : \"smoking\"," + + "\"type\" : \"boolean\"," + + "\"value\" : \"true\"," + + "\"metadatas\" : [" + + "{" + + "\"name\" : \"TimeInstant\"," + + "\"type\" : \"ISO8601\"," + + "\"value\" : \"2018-01-02T03:04:05.678+0900\"" + + "}" + + "]" + + "}" + + "]," + + "\"type\" : \"Room\"," + + "\"isPattern\" : \"false\"," + + "\"id\" : \"Room1\"" + + "}"; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + doNothing().when(mockBackend).createColumns(anyString()); + doNothing().when(mockBackend).insertContextData(anyString()); } - // without slash - for (final String[] test : tests) { - final String result = NGSISlicingDiceSink.encode(test[0], false, true); - Assert.assertEquals(test[1], result); - } - } // testStringEncodeWithoutSlash - - private Context createContext(final String databaseKey, final String autoCreate, - final String dataModel, final String encoding) { + @Test + public void testPersistBatchWithoutData() { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.persistBatch]")); + + NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.setPersistenceBackend(mockBackend); + + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "true"; + final String dataModel = "dm-by-service-path"; + final String encoding = null; + + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + try { + sink.persistBatch(null); + verify(mockBackend, times(0)).createColumns(anyString()); + verify(mockBackend, times(0)).insertContextData(anyString()); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } // try catch + } // testPersistBatchWithoutData + + @Test + public void testPersistBatchWithFirstExample() throws Exception { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.persistBatch]")); + + NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.setPersistenceBackend(mockBackend); + + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "false"; + final String dataModel = "dm-by-service-path"; + final String encoding = null; + + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + final NGSIBatch batch = createBatch(sink, contextElementStr1); + sink.persistBatch(batch); + + verify(mockBackend, times(1)).createColumns(idxCaptor.capture()); + Assert.assertEquals("[{\"name\": \"fiwareServicePath\", \"api-name\": " + + "\"fiwareServicePath\", \"type\": \"string-event\", \"description\": " + + "\"Created using CYGNUS.\", \"dimension\": \"room-service-path\"},{\"name\": " + + "\"temperature\", \"api-name\": \"temperature\", \"type\": \"decimal-event\", " + + "\"description\": \"Created using CYGNUS.\", \"decimal-places\": 5, " + + "\"dimension\": \"room-service-path\"}]", idxCaptor.getValue()); + + verify(mockBackend, times(1)).insertContextData(idxCaptor.capture()); + Assert.assertEquals("{\"Room1\": {\"dimension\": \"room-service-path\", " + + "\"fiwareServicePath\": [{\"value\": \"/room_service_path\", \"date\": " + + "\"1970-01-15T06:56:07.890\"}], \"temperature\": [{\"value\": 26.5, \"date\": " + + "\"1970-01-15T06:56:07.890\"}]}\"auto-create\": [\"dimension\"]}", idxCaptor.getValue()); + } // testPersistBatchWithOneAttribute + + @Test + public void testPersistBatchWithSecondExample() throws Exception { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.persistBatch]")); + + NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.setPersistenceBackend(mockBackend); + + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "false"; + final String dataModel = "dm-by-entity"; + final String encoding = null; + + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + final NGSIBatch batch = createBatch(sink, contextElementStr2); + sink.persistBatch(batch); + + verify(mockBackend, times(1)).createColumns(idxCaptor.capture()); + Assert.assertEquals("[{\"name\": \"fiwareServicePath\", \"api-name\": " + + "\"fiwareServicePath\", \"type\": \"string-event\", \"description\": " + + "\"Created using CYGNUS.\", \"dimension\": \"room-service-path-Room1-Room\"}," + + "{\"name\": \"temperature\", \"api-name\": \"temperature\", \"type\": " + + "\"decimal-event\", \"description\": \"Created using CYGNUS.\", " + + "\"decimal-places\": 5, \"dimension\": \"room-service-path-Room1-Room\"}," + + "{\"name\": \"roomtype\", \"api-name\": \"roomtype\", \"type\": " + + "\"string-event\", \"description\": \"Created using CYGNUS.\", \"dimension\": " + + "\"room-service-path-Room1-Room\"}]", idxCaptor.getValue()); + + verify(mockBackend, times(1)).insertContextData(idxCaptor.capture()); + Assert.assertEquals("{\"Room1\": {\"dimension\": \"room-service-path-Room1" + + "-Room\", \"fiwareServicePath\": [{\"value\": \"/room_service_path\", \"date\":" + + " \"1970-01-15T06:56:07.890\"}], \"temperature\": [{\"value\": 26.5, \"date\":" + + " \"1970-01-15T06:56:07.890\"}], \"roomtype\": [{\"value\": \"\", \"date\":" + + " \"1970-01-15T06:56:07.890\"}]}\"auto-create\": [\"dimension\"]}", + idxCaptor.getValue()); + } // testPersistBatchWithSecondExample + + @Test + public void testPersistBatchWithThirdExample() throws Exception { + System.out.println(getTestTraceHead("[NGSISlicingDiceSink.persistBatch]")); + + NGSISlicingDiceSink sink = new NGSISlicingDiceSink(); + sink.setPersistenceBackend(mockBackend); + + final String databaseKey = "oasdisadnasoi"; + final String isAutoCreate = "false"; + final String dataModel = "dm-by-entity"; + final String encoding = null; + + sink.configure(createContext(databaseKey, isAutoCreate, dataModel, encoding)); + + final NGSIBatch batch = createBatch(sink, contextElementStr3); + sink.persistBatch(batch); + + verify(mockBackend, times(1)).createColumns(idxCaptor.capture()); + Assert.assertEquals("[{\"name\": \"fiwareServicePath\", \"api-name\": " + + "\"fiwareServicePath\", \"type\": \"string-event\", \"description\": " + + "\"Created using CYGNUS.\", \"dimension\": \"room-service-path-Room1-Room\"}," + + "{\"name\": \"temperature\", \"api-name\": \"temperature\", \"type\": " + + "\"decimal-event\", \"description\": \"Created using CYGNUS.\", " + + "\"decimal-places\": 5, \"dimension\": \"room-service-path-Room1-Room\"}," + + "{\"name\": \"roomtype\", \"api-name\": \"roomtype\", \"type\": " + + "\"string-event\", \"description\": \"Created using CYGNUS.\", \"dimension\": " + + "\"room-service-path-Room1-Room\"},{\"name\": \"smoking\", \"api-name\": " + + "\"smoking\", \"type\": \"string-event\", \"description\": " + + "\"Created using CYGNUS.\", \"dimension\": \"room-service-path-Room1-Room\"}]", + idxCaptor.getValue()); + + verify(mockBackend, times(1)).insertContextData(idxCaptor.capture()); + Assert.assertEquals("{\"Room1\": {\"dimension\": \"room-service-path-Room1-" + + "Room\", \"fiwareServicePath\": [{\"value\": \"/room_service_path\", " + + "\"date\": \"1970-01-15T06:56:07.890\"}], \"temperature\": [{\"value\":" + + " 26.5, \"date\": \"1970-01-15T06:56:07.890\"}], \"roomtype\": " + + "[{\"value\": \"single\", \"date\": \"1970-01-15T06:56:07.890\"}], " + + "\"smoking\": [{\"value\": \"true\", \"date\": \"1970-01-15T06:56:" + + "07.890\"}]}\"auto-create\": [\"dimension\"]}", + idxCaptor.getValue()); + } // testPersistBatchWithSecondExample + + @Ignore + private NGSIBatch createBatch(NGSISink sink, String contextElementStr) throws Exception { + final String timestamp = "1234567890"; + final String correlatorId = "1234567891"; + final String serviceStr = "room_service"; + final String servicePathStr = "/room_service_path"; + NGSISink.Accumulator acc = sink.new Accumulator(); + acc.initialize(new Date().getTime()); + Map headers = new ConcurrentHashMap<>(); + headers.put(NGSIConstants.FLUME_HEADER_TIMESTAMP, timestamp); + headers.put(CommonConstants.HEADER_CORRELATOR_ID, correlatorId); + headers.put(CommonConstants.HEADER_FIWARE_SERVICE, serviceStr); + headers.put(CommonConstants.HEADER_FIWARE_SERVICE_PATH, servicePathStr); + headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE, serviceStr); + headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE_PATH, servicePathStr); + NotifyContextRequest.ContextElement contextElement = NGSIUtilsForTests.createJsonContextElement(contextElementStr); + acc.accumulate(new NGSIEvent(headers, contextElement.toString().getBytes(), contextElement, contextElement)); + return acc.getBatch(); + } // createBatch + } + + private static Context createContext(final String databaseKey, final String autoCreate, + final String dataModel, final String encoding) { Context context = new Context(); context.put("database_key", databaseKey); context.put("auto_create", autoCreate); From fd273ee75fe12ead68bbd12e5352113384877d05 Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Sat, 24 Nov 2018 15:09:16 -0300 Subject: [PATCH 4/7] Add SlicingDice entries on Dockerfile and shell scripts --- docker/cygnus-ngsi/Dockerfile | 14 +++++++++++++ docker/cygnus-ngsi/agent.conf | 19 ++++++++++++++++- docker/cygnus-ngsi/cygnus-entrypoint.sh | 28 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docker/cygnus-ngsi/Dockerfile b/docker/cygnus-ngsi/Dockerfile index 4a5feff0f..691ef56a2 100644 --- a/docker/cygnus-ngsi/Dockerfile +++ b/docker/cygnus-ngsi/Dockerfile @@ -139,6 +139,20 @@ ENV CYGNUS_POSTGRESQL_BATCH_TIMEOUT "" ENV CYGNUS_POSTGRESQL_BATCH_TTL "" ENV CYGNUS_POSTGRESQL_ENABLE_CACHE "" +# SlicingDice options +ENV CYGNUS_SLICINGDICE_DATABASE_KEY "" +ENV CYGNUS_SLICINGDICE_AUTO_CREATE "" +ENV CYGNUS_SLICINGDICE_ENABLE_ENCODING "" +ENV CYGNUS_SLICINGDICE_ENABLE_GROUPING "" +ENV CYGNUS_SLICINGDICE_ENABLE_NAME_MAPPINGS "" +ENV CYGNUS_SLICINGDICE_ENABLE_LOWERCASE "" +ENV CYGNUS_SLICINGDICE_DATA_MODEL "" +ENV CYGNUS_SLICINGDICE_ATTR_PERSISTENCE "" +ENV CYGNUS_SLICINGDICE_BATCH_SIZE "" +ENV CYGNUS_SLICINGDICE_BATCH_TIMEOUT "" +ENV CYGNUS_SLICINGDICE_BATCH_TTL "" +ENV CYGNUS_SLICINGDICE_AUTO_CREATE "" + # Carto options # ENV CYGNUS_CARTO_USER "" # ENV CYGNUS_CARTO_KEY "" diff --git a/docker/cygnus-ngsi/agent.conf b/docker/cygnus-ngsi/agent.conf index aa136c651..48ed48e47 100644 --- a/docker/cygnus-ngsi/agent.conf +++ b/docker/cygnus-ngsi/agent.conf @@ -21,7 +21,7 @@ cygnus-ngsi.sinks = cygnus-ngsi.channels = cygnus-ngsi.sources.http-source.type = org.apache.flume.source.http.HTTPSource -cygnus-ngsi.sources.http-source.channels = mysql-channel mongo-channel sth-channel ckan-channel hdfs-channel cartodb-channel postgresql-channel orion-channel +cygnus-ngsi.sources.http-source.channels = mysql-channel mongo-channel sth-channel ckan-channel hdfs-channel cartodb-channel postgresql-channel orion-channel slicingdice-channel cygnus-ngsi.sources.http-source.port = 5050 cygnus-ngsi.sources.http-source.handler = com.telefonica.iot.cygnus.handlers.NGSIRestHandler cygnus-ngsi.sources.http-source.handler.notification_target = /notify @@ -180,6 +180,19 @@ cygnus-ngsi.sinks.orion-sink.keystone_ssl = false cygnus-ngsi.sinks.orion-sink.orion_fiware = cygnus-ngsi.sinks.orion-sink.orion_fiware_path = + +# SlicingDiceSink configuration +cygnus-ngsi.sinks.slicingdice.type = com.telefonica.iot.cygnus.sinks.NGSISlicingDiceSink +cygnus-ngsi.sinks.slicingdice-sink.channel = slicingdice-channel +#cygnus-ngsi.sinks.slicingdice-sink.enable_encoding = false +#cygnus-ngsi.sinks.slicingdice-sink.enable_grouping = false +#cygnus-ngsi.sinks.slicingdice-sink.enable_name_mappings = false +#cygnus-ngsi.sinks.slicingdice-sink.enable_lowercase = false +#cygnus-ngsi.sinks.slicingdice-sink.data_model = dm-by-entity +cygnus-ngsi.sinks.slicingdice-sink.database_key = +cygnus-ngsi.sinks.slicingdice-sink.auto_create = +#cygnus-ngsi.sinks.slicingdice-sink.attr_persistence = row + cygnus-ngsi.channels.mysql-channel.type = com.telefonica.iot.cygnus.channels.CygnusMemoryChannel cygnus-ngsi.channels.mysql-channel.capacity = 1000 cygnus-ngsi.channels.mysql-channel.transactionCapacity = 100 @@ -212,3 +225,7 @@ cygnus-ngsi.channels.cartodb-channel.transactionCapacity = 100 cygnus-ngsi.channels.orion-channel.type = com.telefonica.iot.cygnus.channels.CygnusMemoryChannel cygnus-ngsi.channels.orion-channel.capacity = 1000 cygnus-ngsi.channels.orion-channel.transactionCapacity = 100 + +cygnus-ngsi.channels.slicingdice-channel.type = com.telefonica.iot.cygnus.channels.CygnusMemoryChannel +cygnus-ngsi.channels.slicingdice-channel.capacity = 1000 +cygnus-ngsi.channels.slicingdice-channel.transactionCapacity = 100 diff --git a/docker/cygnus-ngsi/cygnus-entrypoint.sh b/docker/cygnus-ngsi/cygnus-entrypoint.sh index 550e1a770..c568b55a2 100755 --- a/docker/cygnus-ngsi/cygnus-entrypoint.sh +++ b/docker/cygnus-ngsi/cygnus-entrypoint.sh @@ -525,6 +525,34 @@ if [ "$CYGNUS_ORION_HOST" != "" ]; then fi +# Check if SLICINGDICE ENV vars +if [ "$CYGNUS_SLICINGDICE_DATABASE_KEY" != "" ]; then + sed -i 's/'${CYGNUS_AGENT_NAME}'.sinks =/'${CYGNUS_AGENT_NAME}'.sinks = slicingdice-sink /g' ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + sed -i 's/'${CYGNUS_AGENT_NAME}'.channels =/'${CYGNUS_AGENT_NAME}'.channels = slicingdice-channel /g' ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + sed -i '/'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.database_key/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.database_key = '${$CYGNUS_SLICINGDICE_DATABASE_KEY} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + # The following are optional and disabled by default + if [ "$CYGNUS_SLICINGDICE_ENABLE_ENCODING" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_encoding/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_encoding = '${CYGNUS_SLICINGDICE_ENABLE_ENCODING} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_ENABLE_GROUPING" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_grouping/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_grouping = '${CYGNUS_SLICINGDICE_ENABLE_GROUPING} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_ENABLE_NAME_MAPPINGS" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_name_mappings/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_name_mappings = '${CYGNUS_SLICINGDICE_ENABLE_NAME_MAPPINGS} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_ENABLE_LOWERCASE" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_lowercase/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.enable_lowercase = '${CYGNUS_SLICINGDICE_ENABLE_LOWERCASE} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_DATA_MODEL" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.data_model/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.data_model = '${CYGNUS_SLICINGDICE_DATA_MODEL} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_ATTR_PERSISTENCE" != "" ]; then + sed -i '/#'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.attr_persistence/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.attr_persistence = '${CYGNUS_SLICINGDICE_ATTR_PERSISTENCE} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi + if [ "$CYGNUS_SLICINGDICE_AUTO_CREATE" != "" ]; then + sed -i '/'${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.auto_create/c '${CYGNUS_AGENT_NAME}'.sinks.slicingdice-sink.auto_create = '${CYGNUS_SLICINGDICE_AUTO_CREATE} ${FLUME_HOME}/conf/${AGENT_CONF_FILE} + fi +fi if [ "${CYGNUS_MULTIAGENT,,}" == "false" ]; then From c1190a73d0717913f7fd286088dab17f3481c32a Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Sat, 24 Nov 2018 15:11:43 -0300 Subject: [PATCH 5/7] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index bccf9fe63..e3f56ce66 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -9,3 +9,4 @@ [cygnus-ngsi][KafkaSink] Using global connection to zookeeper instead of creating one each time an event arrives [cygnus-ngsi][NGSINameMappingsInterceptor] Now namemapping checks sevice, subervice and (type of entity and id entity) of EntityMapping (#1535) [cygnus-ngsi][NGSIEvent] Unable to deliver event: null pointer getAttributeForNaming (#1506) +[cygnus-ngsi][SlicingDiceSink] Added SlicingDice sink to cygnus From 11d92fe7a007abfe723a1a38ee60fb4460b2a843 Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Sat, 24 Nov 2018 15:23:21 -0300 Subject: [PATCH 6/7] Change default variables values --- .../iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java | 2 +- .../com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java index 634d2d479..261b30374 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java @@ -19,7 +19,7 @@ public class SlicingDiceBackendImpl extends HttpBackend implements SlicingDiceBa private static final CygnusLogger LOGGER = new CygnusLogger(SlicingDiceBackendImpl.class); // this is the SlicingDice host, the user will not be able to change it - private static final String SLICING_DICE_HOST = "api.slicingdice.com"; + private static final String SLICING_DICE_HOST = "api.slicingdice.com/v1"; // this is the SlicingDice port, the user will not be able to change it private static final String SLICING_DICE_PORT = "443"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java index 1ce11463f..477fe4573 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java @@ -57,7 +57,7 @@ public void configure(final Context context) { } else { LOGGER.debug("[" + this.getName() + "] Reading configuration (api_key=" + databaseKey + ")"); } - this.autoCreate = context.getBoolean("auto_create", false); + this.autoCreate = context.getBoolean("auto_create", true); LOGGER.debug("[" + this.getName() + "] Reading configuration (auto_create=" + autoCreate + ")"); } // configure From 1c6ba1729ff0f8ab7682f2d3a8d790a03b6eff5d Mon Sep 17 00:00:00 2001 From: joaosimbiose Date: Mon, 26 Nov 2018 11:31:04 -0300 Subject: [PATCH 7/7] Add license headers to created/modified files --- .../slicingdice/SlicingDiceBackend.java | 21 ++++++++++++++++ .../slicingdice/SlicingDiceBackendImpl.java | 21 ++++++++++++++++ .../SlicingDiceBackendImplTest.java | 25 ++++++++++++++++--- .../iot/cygnus/sinks/NGSISlicingDiceSink.java | 21 ++++++++++++++++ .../iot/cygnus/utils/NGSICharsets.java | 4 +++ .../iot/cygnus/utils/NGSIConstants.java | 3 +++ .../iot/cygnus/utils/NGSIUtils.java | 3 +++ .../cygnus/sinks/NGSISlicingDiceSinkTest.java | 25 ++++++++++++++++--- docker/cygnus-ngsi/Dockerfile | 2 ++ docker/cygnus-ngsi/agent.conf | 2 ++ docker/cygnus-ngsi/cygnus-entrypoint.sh | 2 ++ 11 files changed, 121 insertions(+), 8 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java index 7dc399cd5..929df724b 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackend.java @@ -1,3 +1,24 @@ +/** + * Copyright 2018 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cygnus (FIWARE project). + * + * fiware-cygnus is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cygnus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cygnus. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Authorship: SlicingDice + * + */ + package com.telefonica.iot.cygnus.backends.slicingdice; import com.telefonica.iot.cygnus.errors.CygnusBadContextData; diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java index 261b30374..29229e94b 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImpl.java @@ -1,3 +1,24 @@ +/** + * Copyright 2018 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cygnus (FIWARE project). + * + * fiware-cygnus is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cygnus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cygnus. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Authorship: SlicingDice + * + */ + package com.telefonica.iot.cygnus.backends.slicingdice; import com.telefonica.iot.cygnus.backends.http.HttpBackend; diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java index be9a63a9d..9cb67b1cc 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/slicingdice/SlicingDiceBackendImplTest.java @@ -1,3 +1,24 @@ +/** + * Copyright 2018 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cygnus (FIWARE project). + * + * fiware-cygnus is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cygnus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cygnus. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Authorship: SlicingDice + * + */ + package com.telefonica.iot.cygnus.backends.slicingdice; import com.telefonica.iot.cygnus.backends.http.JsonResponse; @@ -27,10 +48,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -/** - * - * @author joaosimbiose - */ @RunWith(MockitoJUnitRunner.class) public class SlicingDiceBackendImplTest { diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java index 477fe4573..769e82577 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSink.java @@ -1,3 +1,24 @@ +/** + * Copyright 2018 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cygnus (FIWARE project). + * + * fiware-cygnus is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cygnus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cygnus. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Authorship: SlicingDice + * + */ + package com.telefonica.iot.cygnus.sinks; import com.telefonica.iot.cygnus.backends.slicingdice.SlicingDiceBackend; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java index 93c6bccb8..67c349317 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSICharsets.java @@ -14,7 +14,11 @@ * http://www.gnu.org/licenses/. * * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Modified by: SlicingDice + * */ + package com.telefonica.iot.cygnus.utils; /** diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java index b53de79c6..d31533b57 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIConstants.java @@ -14,6 +14,9 @@ * http://www.gnu.org/licenses/. * * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Modified by: SlicingDice + * */ package com.telefonica.iot.cygnus.utils; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java index c42af1210..4cbf8e0c7 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java @@ -14,6 +14,9 @@ * http://www.gnu.org/licenses/. * * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Modified by: SlicingDice + * */ package com.telefonica.iot.cygnus.utils; diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java index 227b9eae2..61de62dec 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSISlicingDiceSinkTest.java @@ -1,3 +1,24 @@ +/** + * Copyright 2018 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-cygnus (FIWARE project). + * + * fiware-cygnus is free software: you can redistribute it and/or modify it under the terms of the GNU Affero + * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * fiware-cygnus is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License + * for more details. + * + * You should have received a copy of the GNU Affero General Public License along with fiware-cygnus. If not, see + * http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es + * + * Authorship: SlicingDice + * + */ + package com.telefonica.iot.cygnus.sinks; import com.telefonica.iot.cygnus.backends.slicingdice.SlicingDiceBackendImpl; @@ -31,10 +52,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -/** - * - * @autor joaosimbiose - */ @RunWith(Enclosed.class) public class NGSISlicingDiceSinkTest { diff --git a/docker/cygnus-ngsi/Dockerfile b/docker/cygnus-ngsi/Dockerfile index 691ef56a2..11a8cf2ba 100644 --- a/docker/cygnus-ngsi/Dockerfile +++ b/docker/cygnus-ngsi/Dockerfile @@ -15,6 +15,8 @@ # # For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es # +# Modified by: SlicingDice +# FROM centos:centos7.4.1708 diff --git a/docker/cygnus-ngsi/agent.conf b/docker/cygnus-ngsi/agent.conf index 48ed48e47..620bc6709 100644 --- a/docker/cygnus-ngsi/agent.conf +++ b/docker/cygnus-ngsi/agent.conf @@ -15,6 +15,8 @@ # # For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es # +# Modified by: SlicingDice +# cygnus-ngsi.sources = http-source cygnus-ngsi.sinks = diff --git a/docker/cygnus-ngsi/cygnus-entrypoint.sh b/docker/cygnus-ngsi/cygnus-entrypoint.sh index c568b55a2..5297a9714 100755 --- a/docker/cygnus-ngsi/cygnus-entrypoint.sh +++ b/docker/cygnus-ngsi/cygnus-entrypoint.sh @@ -16,6 +16,8 @@ # # For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es # +# Modified by: SlicingDice +# # Unpack two big jar files [[ -f ${FLUME_HOME}/plugins.d/cygnus/libext/cygnus-common-${CYGNUS_VERSION}-jar-with-dependencies.jar.pack.gz ]] && unpack200 -r ${FLUME_HOME}/plugins.d/cygnus/libext/cygnus-common-${CYGNUS_VERSION}-jar-with-dependencies.jar.pack.gz ${FLUME_HOME}/plugins.d/cygnus/libext/cygnus-common-${CYGNUS_VERSION}-jar-with-dependencies.jar