Skip to content

Commit

Permalink
Updates to size test
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilhan2316
ilhan2316 committed Sep 18, 2024
1 parent 92a925a commit e6bca6f
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 164 deletions.
160 changes: 8 additions & 152 deletions testing/regress/ecl/key/parquet_size.xml
View file
Original file line number Diff line number Diff line change
@@ -1,154 +1,10 @@
<Dataset name='small_dataset'>
<Row><id>1</id><name>B</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>2</id><name>C</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>3</id><name>D</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>4</id><name>E</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>5</id><name>F</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>6</id><name>G</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>7</id><name>H</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>8</id><name>I</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>9</id><name>J</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>10</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>11</id><name>B</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>12</id><name>C</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>13</id><name>D</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>14</id><name>E</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>15</id><name>F</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>16</id><name>G</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>17</id><name>H</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>18</id><name>I</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>19</id><name>J</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>20</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>21</id><name>B</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>22</id><name>C</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>23</id><name>D</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>24</id><name>E</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>25</id><name>F</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>26</id><name>G</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>27</id><name>H</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>28</id><name>I</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>29</id><name>J</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>30</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>31</id><name>B</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>32</id><name>C</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>33</id><name>D</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>34</id><name>E</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>35</id><name>F</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>36</id><name>G</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>37</id><name>H</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>38</id><name>I</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>39</id><name>J</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>40</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>41</id><name>B</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>42</id><name>C</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>43</id><name>D</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>44</id><name>E</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>45</id><name>F</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>46</id><name>G</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>47</id><name>H</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>48</id><name>I</name><price>10.0</price><isactive>1</isactive></Row>
<Row><id>49</id><name>J</name><price>10.0</price><isactive>0</isactive></Row>
<Row><id>50</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
<Dataset name='singleDataset'>
<Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
<Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
<Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
</Dataset>
<Dataset name='large_dataset'>
<Row><id>1</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>2</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>3</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>4</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>5</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>6</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>7</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>8</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>9</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>10</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>11</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>12</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>13</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>14</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>15</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>16</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>17</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>18</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>19</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>20</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>21</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>22</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>23</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>24</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>25</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>26</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>27</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>28</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>29</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>30</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>31</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>32</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>33</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>34</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>35</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>36</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>37</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>38</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>39</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>40</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>41</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>42</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>43</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>44</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>45</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>46</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>47</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>48</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>49</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>50</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>51</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>52</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>53</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>54</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>55</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>56</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>57</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>58</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>59</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>60</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>61</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>62</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>63</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>64</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>65</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>66</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>67</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>68</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>69</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>70</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>71</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>72</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>73</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>74</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>75</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>76</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>77</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>78</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>79</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>80</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>81</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>82</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>83</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>84</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>85</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>86</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>87</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>88</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>89</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>90</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>91</id><name>B</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>92</id><name>C</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>93</id><name>D</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>94</id><name>E</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>95</id><name>F</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>96</id><name>G</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>97</id><name>H</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>98</id><name>I</name><price>20.0</price><isactive>1</isactive></Row>
<Row><id>99</id><name>J</name><price>20.0</price><isactive>0</isactive></Row>
<Row><id>100</id><name>A</name><price>20.0</price><isactive>1</isactive></Row>
<Dataset name='multiDataset'>
<Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
<Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
<Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
</Dataset>
21 changes: 9 additions & 12 deletions testing/regress/ecl/parquet_size.ecl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,16 @@ recordLayout := RECORD
STRING isactive;
END;

smallFilePath := '/var/lib/HPCCSystems/mydropzone/small1.parquet';
mediumFilePath := '/var/lib/HPCCSystems/mydropzone/medium1.parquet';
// Paths to the files
singleFilePath := '/var/lib/HPCCSystems/mydropzone/single.parquet';
multiFilePath := '/var/lib/HPCCSystems/mydropzone/multi*.parquet'; // wildcard for all 3 parts

smallDataset := ParquetIO.Read(recordLayout, smallFilePath);
largeDataset := ParquetIO.Read(recordLayout, mediumFilePath);

largeDatasetPart1 := largeDataset[1..33];
largeDatasetPart2 := largeDataset[34..66];
largeDatasetPart3 := largeDataset[67..100];

combinedLargeDataset := largeDatasetPart1 + largeDatasetPart2 + largeDatasetPart3;
// Reading the single and multi-part files
singleDataset := ParquetIO.Read(recordLayout, singleFilePath);
multiDataset := ParquetIO.Read(recordLayout, multiFilePath);

// Output the datasets
SEQUENTIAL(
OUTPUT(smallDataset, NAMED('small_dataset')),
OUTPUT(combinedLargeDataset, NAMED('large_dataset'))
OUTPUT(singleDataset, NAMED('singleDataset')), // Output for the single file
OUTPUT(multiDataset, NAMED('multiDataset')) // Output for the combined multi-part files
);

0 comments on commit e6bca6f

Please sign in to comment.