Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC4J-657 DFSClient: Reading indices with blobs fails #767

Merged
merged 1 commit into from
Oct 30, 2024
Merged

HPCC4J-657 DFSClient: Reading indices with blobs fails #767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class FieldDef implements Serializable
private long len = 0;
private boolean fixedLength = false;
private boolean isUnsigned = false;
private boolean isBlob = false;
private int additionalFlags = 0;

/**
Expand All @@ -59,6 +60,7 @@ public FieldDef(FieldDef rhs)
this.len = rhs.len;
this.fixedLength = rhs.fixedLength;
this.isUnsigned = rhs.isUnsigned;
this.isBlob = rhs.isBlob;
this.additionalFlags = rhs.additionalFlags;
}

Expand Down Expand Up @@ -330,6 +332,25 @@ public boolean isBiased()
return isNonStandardInt();
}

/**
* Is the field stored in a blob?
*
* @return true when blob
*/
public boolean isBlob()
{
return this.isBlob;
}

/**
* Sets the blob flag.
* @param blob is the field a blob?
*/
public void setIsBlob(boolean blob)
{
this.isBlob = blob;
}

/**
*
*
Expand Down
105 changes: 80 additions & 25 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ public class RecordDefinitionTranslator
private static final String NAME_KEY = "name";
private static final String TYPE_KEY = "type";
private static final String CHILD_KEY = "child";
private static final String XPATH_KEY = "xpath";
private static final String FLAGS_KEY = "flags";

private static final String ESP_TYPE_NAME_PREFIX = "ty";

private static final int BLOB_LENGTH = 8;
private static final int FLAG_UNSIGNED = 256;
private static final int FLAG_UNKNOWN_SIZE = 1024;
private static final int TYPE_ID_MASK = 0xff; // 0x7fff & ~FLAG_UNKNOWN_SIZE & ~FLAG_UNSIGNED;
Expand All @@ -48,6 +50,7 @@ public class RecordDefinitionTranslator
final private static int type_keyedint = 10; // Convert to integer
final private static int type_record = 13;
final private static int type_varstring = 14;
final private static int type_blob = 15;
final private static int type_data = 16;
final private static int type_table = 20;
final private static int type_set = 21;
Expand Down Expand Up @@ -264,28 +267,32 @@ public static String toECLRecord(FieldDef field) throws Exception
*/
private static String getEClTypeDefinition(FieldDef field, HashMap<String, String> recordDefinitionMap) throws Exception
{
String type = "";
switch (field.getFieldType())
{
case SET:
{
return "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
type = "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
break;
}
case DATASET:
{
return "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
type = "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
break;
}
case BINARY:
{
type = "DATA";
if (field.isFixed())
{
return "DATA" + field.getDataLen();
type += field.getDataLen();
}

return "DATA";
break;
}
case BOOLEAN:
{
return "BOOLEAN";
type = "BOOLEAN";
break;
}
case INTEGER:
{
Expand All @@ -300,7 +307,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported integer size: " + field.getDataLen() + " must 1-8.");
}

return root + field.getDataLen();
type = root + field.getDataLen();
break;
}
case FILEPOS:
{
Expand All @@ -314,7 +322,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported filepos size: " + field.getDataLen() + " must be 8.");
}

return "UNSIGNED8";
type = "UNSIGNED8";
break;
}
case DECIMAL:
{
Expand All @@ -323,28 +332,33 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
root = "U" + root;
}
return root + field.getPrecision() + "_" + field.getScale();
type = root + field.getPrecision() + "_" + field.getScale();
break;
}
case REAL:
{
if (field.getDataLen() == 4)
{
return "REAL4";
type = "REAL4";
}
else if (field.getDataLen() == 8)
{
return "REAL8";
type = "REAL8";
}
else
{
throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
break;

throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
case CHAR:
{
return "STRING1";
type = "STRING1";
break;
}
case STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -371,11 +385,10 @@ else if (srcType == HpccSrcType.QSTRING)
{
type += field.getDataLen();
}
return type;
break;
}
case VAR_STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -394,7 +407,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
{
type += field.getDataLen();
}
return type;
break;
}
case RECORD:
{
Expand All @@ -416,13 +429,21 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
String recordDefnName = "##" + hash + "##";

recordDefinitionMap.put(recordDefnName, definition);
return recordDefnName;
type = recordDefnName;
break;
}
default:
{
throw new Exception("Unable to generate ECL unknown field type: " + field.getFieldType().description());
}
}

if (field.isBlob())
{
type += " {blob}";
rpastrana marked this conversation as resolved.
Show resolved Hide resolved
}

return type;
}

/**
Expand Down Expand Up @@ -648,7 +669,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
*/
private static int getTypeHash(FieldDef field) throws Exception
{
int numHashComponents = 4 + field.getNumDefs();
int numHashComponents = 5 + field.getNumDefs();
if (field.getFieldType() == FieldType.DECIMAL)
{
numHashComponents += 2;
Expand All @@ -659,8 +680,9 @@ private static int getTypeHash(FieldDef field) throws Exception
hashComponents[1] = field.getDataLen();
hashComponents[2] = field.getSourceType().ordinal();
hashComponents[3] = field.getAdditionalFlags();
hashComponents[4] = field.isBlob() ? 1 : 0;

int hashCompIndex = 4;
int hashCompIndex = 5;
for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
{
hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
Expand Down Expand Up @@ -698,6 +720,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
return typeHash;
}

if (field.isBlob())
{
FieldDef nonBlobField = new FieldDef(field);
nonBlobField.setIsBlob(false);

int nonBlobTypeHash = getJsonTypeDefinition(nonBlobField, typeDefinitionMap, typeDefinitions);
int nonBlobTypeIndex = typeDefinitionMap.get(nonBlobTypeHash);
String nonBlobTypeName = ESP_TYPE_NAME_PREFIX + (nonBlobTypeIndex + 1);

JSONObject typeDef = new JSONObject();
typeDef.put("fieldType", type_blob);
typeDef.put("length", BLOB_LENGTH);
typeDef.put("child", nonBlobTypeName);

int newTypeIndex = typeDefinitions.size();
typeDefinitions.add(typeDef);
typeDefinitionMap.put(typeHash, newTypeIndex);
return typeHash;
}

JSONObject typeDef = new JSONObject();
int typeID = getTypeID(field);
switch (field.getFieldType())
Expand Down Expand Up @@ -753,25 +795,30 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);

int childTypeID = getTypeID(childField);
if (childField.isBlob())
{
childTypeID = type_blob;
}

JSONObject childJson = new JSONObject();
childJson.put("name", childField.getFieldName());
childJson.put("type", childTypeName);
childJson.put(NAME_KEY, childField.getFieldName());
childJson.put(TYPE_KEY, childTypeName);

int flags = childTypeID | childField.getAdditionalFlags();
if (flags > 0)
{
childJson.put("flags", flags);
childJson.put(FLAGS_KEY, flags);
}

if (childField.getFieldType() == FieldType.DATASET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Row");
}
else if (childField.getFieldType() == FieldType.SET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Item");
}

fields.put(childJson);
Expand Down Expand Up @@ -954,6 +1001,14 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
int typeID = typeDef.getInt(FIELD_TYPE_KEY);
long length = typeDef.getLong(LENGTH_KEY);

if (typeID == type_blob)
{
String blobType = typeDef.getString(CHILD_KEY);
FieldDef def = getOrParseJsonTypeDefintion(blobType, jsonTypeDefinitions, protoTypeDefs);
def.setIsBlob(true);
return def;
}

FieldType fieldType = getFieldType(typeID);
switch (fieldType)
{
Expand Down
21 changes: 21 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,22 @@ public class TestFieldDefinitions
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

private static final String blobIndexDefinitionStr = "{\n \"ty6\": { \"vinit\": 2, \"length\": 8, \"fieldType\": 285 },\n"
+ " \"ty5\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty4\" },\n"
+ " \"length\": 36,\n"
+ " \"ty2\": { \"length\": 0, \"fieldType\": 1028 },\n"
+ " \"fields\": [\n"
+ " { \"name\": \"str12\", \"flags\": 4, \"type\": \"ty1\" },\n"
+ " { \"name\": \"content_string\", \"flags\": 65551, \"type\": \"ty3\" },\n"
+ " { \"name\": \"content_data\", \"flags\": 65551, \"type\": \"ty5\" },\n"
+ " { \"name\": \"__internal_fpos__\", \"flags\": 65821, \"type\": \"ty6\" }\n"
+ " ],\n"
+ " \"ty1\": { \"length\": 12, \"fieldType\": 4 },\n"
+ " \"fieldType\": 13,\n"
+ " \"ty4\": { \"length\": 0, \"fieldType\": 1040 },\n"
+ " \"ty3\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty2\" }\n"
+ "}\n";

/**
* Gets the complex record definition json.
*
Expand All @@ -79,6 +95,11 @@ public static String getAllTypesIndexRecordDefinitionJson()
return allTypesIndexRecordDefinitionStr;
}

public static String getBlobIndexDefinitionJson()
{
return blobIndexDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
3 changes: 2 additions & 1 deletion commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public void setup()
public void testJsonRecordParsing() throws Exception
{
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson(),
TestFieldDefinitions.getBlobIndexDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
try
Expand Down
30 changes: 30 additions & 0 deletions dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,36 @@ private Object parseRecord(FieldDef recordDef, IRecordBuilder recordBuilder, boo
for (int fieldIndex = 0; fieldIndex < recordDef.getNumDefs(); fieldIndex++)
{
FieldDef fd = recordDef.getDef(fieldIndex);
if (fd.isBlob())
{
// If we encounter a blob field, we only have access to the blob file location
// So read that location and construct a default value field
long blobFileLoc = (long) getUnsigned(8, true);
try
{
switch (fd.getFieldType())
{
case BINARY:
recordBuilder.setFieldValue(fieldIndex, new byte[0]);
continue;
case STRING:
case VAR_STRING:
recordBuilder.setFieldValue(fieldIndex, "");
continue;
case SET:
case DATASET:
recordBuilder.setFieldValue(fieldIndex, new ArrayList<Object>());
continue;
default:
throw new UnparsableContentException("Unexpected blob type: " + fd.getFieldType() + " for field: " + fd.getFieldName());
}
}
catch (IllegalAccessException e)
{
throw new UnparsableContentException("Unable to set field value for field: " + fd.getFieldName() + " with error: " + e.getMessage());
}
}

Object fieldValue = null;
switch (fd.getFieldType())
{
Expand Down
Loading
Loading