fixes to Parquet_write.ecl & Parquet_size.ecl & key files

hpcc-systems · Oct 2, 2024 · 84c24cc · 84c24cc
1 parent bb67c82
commit 84c24cc
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 3 deletions.
diff --git a/testing/regress/ecl/key/parquet_schema.xml b/testing/regress/ecl/key/parquet_schema.xml
@@ -0,0 +1,7 @@
+
+<Result>
+<Dataset name='ReadData'>
+ <Row><name>Alice</name><salary>50000.5</salary><id>1</id></Row>
+ <Row><name>Bob</name><salary>60000.75</salary><id>2</id></Row>
+</Dataset>
+</Result>
diff --git a/testing/regress/ecl/key/parquet_size.xml b/testing/regress/ecl/key/parquet_size.xml
@@ -2,9 +2,12 @@
  <Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
  <Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
  <Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
 </Dataset>
 <Dataset name='multiDataset'>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
  <Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
  <Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
  <Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
 </Dataset>
diff --git a/testing/regress/ecl/key/parquet_types.xml b/testing/regress/ecl/key/parquet_types.xml
@@ -0,0 +1,66 @@
+<Dataset name='BooleanTest'>
+ <Row><BooleanTest>Pass</BooleanTest></Row>
+</Dataset>
+<Dataset name='IntegerTest'>
+ <Row><IntegerTest>Pass</IntegerTest></Row>
+</Dataset>
+<Dataset name='UnsignedTest'>
+ <Row><UnsignedTest>Pass</UnsignedTest></Row>
+</Dataset>
+<Dataset name='RealTest'>
+ <Row><RealTest>Pass</RealTest></Row>
+</Dataset>
+<Dataset name='DecimalTest'>
+ <Row><DecimalTest>Pass</DecimalTest></Row>
+</Dataset>
+<Dataset name='StringTest'>
+ <Row><StringTest>Pass</StringTest></Row>
+</Dataset>
+<Dataset name='DataAsStringTest'>
+ <Row><DataAsStringTest>Fail</DataAsStringTest></Row>
+</Dataset>
+<Dataset name='VarStringTest'>
+ <Row><VarStringTest>Pass</VarStringTest></Row>
+</Dataset>
+<Dataset name='QStringTest'>
+ <Row><QStringTest>Pass</QStringTest></Row>
+</Dataset>
+<Dataset name='UTF8Test'>
+ <Row><UTF8Test>Fail: UTF8 data count mismatch</UTF8Test></Row>
+</Dataset>
+<Dataset name='UnicodeTest'>
+ <Row><UnicodeTest>Fail: Unicode data count mismatch</UnicodeTest></Row>
+</Dataset>
+<Dataset name='SetOfIntegerTest'>
+ <Row><SetOfIntegerTest>Pass</SetOfIntegerTest></Row>
+</Dataset>
+<Dataset name='Real8Test'>
+ <Row><Real8Test>Pass</Real8Test></Row>
+</Dataset>
+<Dataset name='SetOfStringTest'>
+ <Row><SetOfStringTest>Pass</SetOfStringTest></Row>
+</Dataset>
+<Dataset name='SetOfUnicodeTest'>
+ <Row><SetOfUnicodeTest>Pass</SetOfUnicodeTest></Row>
+</Dataset>
+<Dataset name='IntegerSizesTest'>
+ <Row><IntegerSizesTest>Pass</IntegerSizesTest></Row>
+</Dataset>
+<Dataset name='UnsignedSizesTest'>
+ <Row><UnsignedSizesTest>Pass</UnsignedSizesTest></Row>
+</Dataset>
+<Dataset name='Real4Test'>
+ <Row><Real4Test>Pass</Real4Test></Row>
+</Dataset>
+<Dataset name='Integer1Test'>
+ <Row><Integer1Test>Pass</Integer1Test></Row>
+</Dataset>
+<Dataset name='FixedSizeBinaryTest'>
+ <Row><FixedSizeBinaryTest>Pass</FixedSizeBinaryTest></Row>
+</Dataset>
+<Dataset name='LargeBinaryTest'>
+ <Row><LargeBinaryTest>Pass</LargeBinaryTest></Row>
+</Dataset>
+<Dataset name='LargeListTest'>
+ <Row><LargeListTest>Pass</LargeListTest></Row>
+</Dataset>
diff --git a/testing/regress/ecl/parquet_size.ecl b/testing/regress/ecl/parquet_size.ecl
@@ -1,15 +1,18 @@
 /*##############################################################################
     HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
+
        http://www.apache.org/licenses/LICENSE-2.0
+
     Unless required by applicable law or agreed to in writing, software
     distributed under the License is distributed on an "AS IS" BASIS,
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
-############################################################################## */
+##############################################################################*/
 
 //class=parquet
 
@@ -24,7 +27,7 @@ END;
 
 // Paths to the files
 singleFilePath := '/var/lib/HPCCSystems/mydropzone/single.parquet';
-multiFilePath := '/var/lib/HPCCSystems/mydropzone/multi*.parquet';  // wildcard for all 3 parts
+multiFilePath := '/var/lib/HPCCSystems/mydropzone/multi.parquet';
 
 // Reading the single and multi-part files
 singleDataset := ParquetIO.Read(recordLayout, singleFilePath);
@@ -34,4 +37,4 @@ multiDataset := ParquetIO.Read(recordLayout, multiFilePath);
 SEQUENTIAL(
     OUTPUT(singleDataset, NAMED('singleDataset')),  // Output for the single file
     OUTPUT(multiDataset, NAMED('multiDataset'))     // Output for the combined multi-part files
-);
+);