Skip to content

Commit

Permalink
HPCC4J-657 DFSClient: Reading indices with blobs fails (#767)
Browse files Browse the repository at this point in the history 
Signed-off-by: James McMullan [email protected]
  • Loading branch information
@jpmcmu
jpmcmu authored Oct 30, 2024
1 parent 0fdbfe5 commit 96d8009
Show file tree
Hide file tree
Showing 8 changed files with 224 additions and 27 deletions.
21 changes: 21 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class FieldDef implements Serializable
private long len = 0;
private boolean fixedLength = false;
private boolean isUnsigned = false;
private boolean isBlob = false;
private int additionalFlags = 0;

/**
Expand All @@ -59,6 +60,7 @@ public FieldDef(FieldDef rhs)
this.len = rhs.len;
this.fixedLength = rhs.fixedLength;
this.isUnsigned = rhs.isUnsigned;
this.isBlob = rhs.isBlob;
this.additionalFlags = rhs.additionalFlags;
}

Expand Down Expand Up @@ -330,6 +332,25 @@ public boolean isBiased()
return isNonStandardInt();
}

/**
* Is the field stored in a blob?
*
* @return true when blob
*/
public boolean isBlob()
{
return this.isBlob;
}

/**
* Sets the blob flag.
* @param blob is the field a blob?
*/
public void setIsBlob(boolean blob)
{
this.isBlob = blob;
}

/**
*
*
Expand Down
105 changes: 80 additions & 25 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ public class RecordDefinitionTranslator
private static final String NAME_KEY = "name";
private static final String TYPE_KEY = "type";
private static final String CHILD_KEY = "child";
private static final String XPATH_KEY = "xpath";
private static final String FLAGS_KEY = "flags";

private static final String ESP_TYPE_NAME_PREFIX = "ty";

private static final int BLOB_LENGTH = 8;
private static final int FLAG_UNSIGNED = 256;
private static final int FLAG_UNKNOWN_SIZE = 1024;
private static final int TYPE_ID_MASK = 0xff; // 0x7fff & ~FLAG_UNKNOWN_SIZE & ~FLAG_UNSIGNED;
Expand All @@ -48,6 +50,7 @@ public class RecordDefinitionTranslator
final private static int type_keyedint = 10; // Convert to integer
final private static int type_record = 13;
final private static int type_varstring = 14;
final private static int type_blob = 15;
final private static int type_data = 16;
final private static int type_table = 20;
final private static int type_set = 21;
Expand Down Expand Up @@ -264,28 +267,32 @@ public static String toECLRecord(FieldDef field) throws Exception
*/
private static String getEClTypeDefinition(FieldDef field, HashMap<String, String> recordDefinitionMap) throws Exception
{
String type = "";
switch (field.getFieldType())
{
case SET:
{
return "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
type = "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
break;
}
case DATASET:
{
return "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
type = "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
break;
}
case BINARY:
{
type = "DATA";
if (field.isFixed())
{
return "DATA" + field.getDataLen();
type += field.getDataLen();
}

return "DATA";
break;
}
case BOOLEAN:
{
return "BOOLEAN";
type = "BOOLEAN";
break;
}
case INTEGER:
{
Expand All @@ -300,7 +307,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported integer size: " + field.getDataLen() + " must 1-8.");
}

return root + field.getDataLen();
type = root + field.getDataLen();
break;
}
case FILEPOS:
{
Expand All @@ -314,7 +322,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported filepos size: " + field.getDataLen() + " must be 8.");
}

return "UNSIGNED8";
type = "UNSIGNED8";
break;
}
case DECIMAL:
{
Expand All @@ -323,28 +332,33 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
root = "U" + root;
}
return root + field.getPrecision() + "_" + field.getScale();
type = root + field.getPrecision() + "_" + field.getScale();
break;
}
case REAL:
{
if (field.getDataLen() == 4)
{
return "REAL4";
type = "REAL4";
}
else if (field.getDataLen() == 8)
{
return "REAL8";
type = "REAL8";
}
else
{
throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
break;

throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
case CHAR:
{
return "STRING1";
type = "STRING1";
break;
}
case STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -371,11 +385,10 @@ else if (srcType == HpccSrcType.QSTRING)
{
type += field.getDataLen();
}
return type;
break;
}
case VAR_STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -394,7 +407,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
{
type += field.getDataLen();
}
return type;
break;
}
case RECORD:
{
Expand All @@ -416,13 +429,21 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
String recordDefnName = "##" + hash + "##";

recordDefinitionMap.put(recordDefnName, definition);
return recordDefnName;
type = recordDefnName;
break;
}
default:
{
throw new Exception("Unable to generate ECL unknown field type: " + field.getFieldType().description());
}
}

if (field.isBlob())
{
type += " {blob}";
}

return type;
}

/**
Expand Down Expand Up @@ -648,7 +669,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
*/
private static int getTypeHash(FieldDef field) throws Exception
{
int numHashComponents = 4 + field.getNumDefs();
int numHashComponents = 5 + field.getNumDefs();
if (field.getFieldType() == FieldType.DECIMAL)
{
numHashComponents += 2;
Expand All @@ -659,8 +680,9 @@ private static int getTypeHash(FieldDef field) throws Exception
hashComponents[1] = field.getDataLen();
hashComponents[2] = field.getSourceType().ordinal();
hashComponents[3] = field.getAdditionalFlags();
hashComponents[4] = field.isBlob() ? 1 : 0;

int hashCompIndex = 4;
int hashCompIndex = 5;
for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
{
hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
Expand Down Expand Up @@ -698,6 +720,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
return typeHash;
}

if (field.isBlob())
{
FieldDef nonBlobField = new FieldDef(field);
nonBlobField.setIsBlob(false);

int nonBlobTypeHash = getJsonTypeDefinition(nonBlobField, typeDefinitionMap, typeDefinitions);
int nonBlobTypeIndex = typeDefinitionMap.get(nonBlobTypeHash);
String nonBlobTypeName = ESP_TYPE_NAME_PREFIX + (nonBlobTypeIndex + 1);

JSONObject typeDef = new JSONObject();
typeDef.put("fieldType", type_blob);
typeDef.put("length", BLOB_LENGTH);
typeDef.put("child", nonBlobTypeName);

int newTypeIndex = typeDefinitions.size();
typeDefinitions.add(typeDef);
typeDefinitionMap.put(typeHash, newTypeIndex);
return typeHash;
}

JSONObject typeDef = new JSONObject();
int typeID = getTypeID(field);
switch (field.getFieldType())
Expand Down Expand Up @@ -753,25 +795,30 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);

int childTypeID = getTypeID(childField);
if (childField.isBlob())
{
childTypeID = type_blob;
}

JSONObject childJson = new JSONObject();
childJson.put("name", childField.getFieldName());
childJson.put("type", childTypeName);
childJson.put(NAME_KEY, childField.getFieldName());
childJson.put(TYPE_KEY, childTypeName);

int flags = childTypeID | childField.getAdditionalFlags();
if (flags > 0)
{
childJson.put("flags", flags);
childJson.put(FLAGS_KEY, flags);
}

if (childField.getFieldType() == FieldType.DATASET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Row");
}
else if (childField.getFieldType() == FieldType.SET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Item");
}

fields.put(childJson);
Expand Down Expand Up @@ -954,6 +1001,14 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
int typeID = typeDef.getInt(FIELD_TYPE_KEY);
long length = typeDef.getLong(LENGTH_KEY);

if (typeID == type_blob)
{
String blobType = typeDef.getString(CHILD_KEY);
FieldDef def = getOrParseJsonTypeDefintion(blobType, jsonTypeDefinitions, protoTypeDefs);
def.setIsBlob(true);
return def;
}

FieldType fieldType = getFieldType(typeID);
switch (fieldType)
{
Expand Down
21 changes: 21 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,22 @@ public class TestFieldDefinitions
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

private static final String blobIndexDefinitionStr = "{\n \"ty6\": { \"vinit\": 2, \"length\": 8, \"fieldType\": 285 },\n"
+ " \"ty5\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty4\" },\n"
+ " \"length\": 36,\n"
+ " \"ty2\": { \"length\": 0, \"fieldType\": 1028 },\n"
+ " \"fields\": [\n"
+ " { \"name\": \"str12\", \"flags\": 4, \"type\": \"ty1\" },\n"
+ " { \"name\": \"content_string\", \"flags\": 65551, \"type\": \"ty3\" },\n"
+ " { \"name\": \"content_data\", \"flags\": 65551, \"type\": \"ty5\" },\n"
+ " { \"name\": \"__internal_fpos__\", \"flags\": 65821, \"type\": \"ty6\" }\n"
+ " ],\n"
+ " \"ty1\": { \"length\": 12, \"fieldType\": 4 },\n"
+ " \"fieldType\": 13,\n"
+ " \"ty4\": { \"length\": 0, \"fieldType\": 1040 },\n"
+ " \"ty3\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty2\" }\n"
+ "}\n";

/**
* Gets the complex record definition json.
*
Expand All @@ -79,6 +95,11 @@ public static String getAllTypesIndexRecordDefinitionJson()
return allTypesIndexRecordDefinitionStr;
}

public static String getBlobIndexDefinitionJson()
{
return blobIndexDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
3 changes: 2 additions & 1 deletion commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public void setup()
public void testJsonRecordParsing() throws Exception
{
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson(),
TestFieldDefinitions.getBlobIndexDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
try
Expand Down
30 changes: 30 additions & 0 deletions dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,36 @@ private Object parseRecord(FieldDef recordDef, IRecordBuilder recordBuilder, boo
for (int fieldIndex = 0; fieldIndex < recordDef.getNumDefs(); fieldIndex++)
{
FieldDef fd = recordDef.getDef(fieldIndex);
if (fd.isBlob())
{
// If we encounter a blob field, we only have access to the blob file location
// So read that location and construct a default value field
long blobFileLoc = (long) getUnsigned(8, true);
try
{
switch (fd.getFieldType())
{
case BINARY:
recordBuilder.setFieldValue(fieldIndex, new byte[0]);
continue;
case STRING:
case VAR_STRING:
recordBuilder.setFieldValue(fieldIndex, "");
continue;
case SET:
case DATASET:
recordBuilder.setFieldValue(fieldIndex, new ArrayList<Object>());
continue;
default:
throw new UnparsableContentException("Unexpected blob type: " + fd.getFieldType() + " for field: " + fd.getFieldName());
}
}
catch (IllegalAccessException e)
{
throw new UnparsableContentException("Unable to set field value for field: " + fd.getFieldName() + " with error: " + e.getMessage());
}
}

Object fieldValue = null;
switch (fd.getFieldType())
{
Expand Down
Loading

0 comments on commit 96d8009

Please sign in to comment.