HPCC-30524 Parquet Strings cannot be converted to REAL

hpcc-systems · Oct 17, 2023 · aeeb64c · aeeb64c
1 parent a05215b
commit aeeb64c
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 5 deletions.
diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp
@@ -956,12 +956,42 @@ __int64 ParquetRowBuilder::getCurrIntValue(const RtlFieldInfo *field)
         {
             __int64 myint64 = 0;
             auto scalar = getCurrView(field);
-            handleDeserializeOutcome(tokenDeserializer.deserialize(scalar.data(), myint64), "signed", scalar.data());
+            handleDeserializeOutcome(tokenDeserializer.deserialize(StringBuffer(scalar.size(), scalar.data()).str(), myint64), "signed", scalar.data());
             return myint64;
         }
     }
 }
 
+double ParquetRowBuilder::getCurrRealValue(const RtlFieldInfo *field)
+{
+    switch ((*array_visitor)->type)
+    {
+        case BoolType:
+            return (*array_visitor)->bool_arr->Value(currArrayIndex());
+        case IntType:
+            return getSigned(array_visitor, currArrayIndex());
+        case UIntType:
+            return getUnsigned(array_visitor, currArrayIndex());
+        case RealType:
+            return getReal(array_visitor, currArrayIndex());
+        case DateType:
+            return (*array_visitor)->size == 32 ? (*array_visitor)->date32_arr->Value(currArrayIndex()) : (*array_visitor)->date64_arr->Value(currArrayIndex());
+        case TimestampType:
+            return (*array_visitor)->timestamp_arr->Value(currArrayIndex());
+        case TimeType:
+            return (*array_visitor)->size == 32 ? (*array_visitor)->time32_arr->Value(currArrayIndex()) : (*array_visitor)->time64_arr->Value(currArrayIndex());
+        case DurationType:
+            return (*array_visitor)->duration_arr->Value(currArrayIndex());
+        default:
+        {
+            double mydouble = 0.0;
+            auto scalar = getCurrView(field);
+            handleDeserializeOutcome(tokenDeserializer.deserialize(StringBuffer(scalar.size(), scalar.data()).str(), mydouble), "real", scalar.data());
+            return mydouble;
+        }
+    }
+}
+
 /**
  * @brief Gets a Boolean result for an ECL Row
  *
@@ -1020,10 +1050,7 @@ double ParquetRowBuilder::getRealResult(const RtlFieldInfo *field)
         return p.doubleResult;
     }
 
-    if ((*array_visitor)->type == RealType)
-        return getReal(array_visitor, currArrayIndex());
-    else
-        return getCurrIntValue(field);
+    return getCurrRealValue(field);
 }
 
 /**

diff --git a/plugins/parquet/parquetembed.hpp b/plugins/parquet/parquetembed.hpp
@@ -899,6 +899,7 @@ class ParquetRowBuilder : public CInterfaceOf<IFieldSource>
     const std::shared_ptr<arrow::Array> &getChunk(std::shared_ptr<arrow::ChunkedArray> *column);
     std::string_view getCurrView(const RtlFieldInfo *field);
     __int64 getCurrIntValue(const RtlFieldInfo *field);
+    double getCurrRealValue(const RtlFieldInfo *field);
     void nextField(const RtlFieldInfo *field);
     void nextFromStruct(const RtlFieldInfo *field);
     void xpathOrName(StringBuffer &outXPath, const RtlFieldInfo *field) const;