Skip to content

Commit

Permalink
check payload s3 source
Browse files Browse the repository at this point in the history
Signed-off-by: tabVersion <[email protected]>
  • Loading branch information
tabVersion committed Sep 11, 2024
1 parent 22db564 commit c0a21ec
Showing 1 changed file with 15 additions and 1 deletion.
16 changes: 15 additions & 1 deletion e2e_test/s3/fs_source_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ def _encode():
name TEXT,
sex int,
mark int,
) WITH (
)
INCLUDE payload as rw_payload
WITH (
connector = 's3',
match_pattern = '{prefix}*.{fmt}',
s3.region_name = '{config['S3_REGION']}',
Expand Down Expand Up @@ -105,6 +107,18 @@ def _assert_eq(field, got, expect):
_assert_eq('sum(sex)', result[2], total_rows / 2)
_assert_eq('sum(mark)', result[3], 0)

# check rw_payload
print('Check rw_payload')
stmt = f"select id, name, sex, mark, rw_payload from {_table()} limit 1;"
cur.execute(stmt)
result = cur.fetchone()
print("Got one line with rw_payload: ", result)
payload = json.loads(result[4])
_assert_eq('id', payload['id'], result[0])
_assert_eq('name', payload['name'], result[1])
_assert_eq('sex', payload['sex'], result[2])
_assert_eq('mark', payload['mark'], result[3])

print('Test pass')

if need_drop_table:
Expand Down

0 comments on commit c0a21ec

Please sign in to comment.