Skip to content

Commit

Permalink
Introduce basic sanity test for MV used by Observability Integrations (…
Browse files Browse the repository at this point in the history 
…#995)

Signed-off-by: Chen Dai <[email protected]>
  • Loading branch information
@dai-chen
dai-chen authored Jan 16, 2025
1 parent 6b69197 commit be1df0f
Show file tree
Hide file tree
Showing 5 changed files with 635 additions and 0 deletions.
181 changes: 181 additions & 0 deletions integ-test/src/integration/resources/aws-logs/cloud_trail.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
CREATE TABLE {table_name} (
eventVersion STRING,
userIdentity STRUCT<
type:STRING,
principalId:STRING,
arn:STRING,
accountId:STRING,
invokedBy:STRING,
accessKeyId:STRING,
userName:STRING,
sessionContext:STRUCT<
attributes:STRUCT<
mfaAuthenticated:STRING,
creationDate:STRING
>,
sessionIssuer:STRUCT<
type:STRING,
principalId:STRING,
arn:STRING,
accountId:STRING,
userName:STRING
>,
ec2RoleDelivery:STRING,
webIdFederationData:MAP<STRING,STRING>
>
>,
eventTime STRING,
eventSource STRING,
eventName STRING,
awsRegion STRING,
sourceIPAddress STRING,
userAgent STRING,
errorCode STRING,
errorMessage STRING,
requestParameters STRING,
responseElements STRING,
additionalEventData STRING,
requestId STRING,
eventId STRING,
resources ARRAY<STRUCT<
arn:STRING,
accountId:STRING,
type:STRING
>>,
eventType STRING,
apiVersion STRING,
readOnly STRING,
recipientAccountId STRING,
serviceEventDetails STRING,
sharedEventId STRING,
vpcEndpointId STRING,
eventCategory STRING,
tlsDetails STRUCT<
tlsVersion:STRING,
cipherSuite:STRING,
clientProvidedHostHeader:STRING
>
)
USING json
OPTIONS (
recursivefilelookup='true',
multiline 'true'
);

INSERT INTO {table_name} VALUES
(
'1.08',
NAMED_STRUCT(
'type', 'IAMUser',
'principalId', 'AWS123456789012',
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
'accountId', '123456789012',
'invokedBy', null,
'accessKeyId', 'AKIA1234567890',
'userName', 'ExampleUser',
'sessionContext', NAMED_STRUCT(
'attributes', NAMED_STRUCT(
'mfaAuthenticated', 'true',
'creationDate', '2023-11-01T05:00:00Z'
),
'sessionIssuer', NAMED_STRUCT(
'type', 'Role',
'principalId', 'ARO123456789012',
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'userName', 'MyRole'
),
'ec2RoleDelivery', 'true',
'webIdFederationData', MAP()
)
),
'2023-11-01T05:00:00Z',
'sts.amazonaws.com',
'AssumeRole',
'us-east-1',
'198.51.100.45',
'AWS CLI',
null,
null,
null,
null,
null,
'request-id-1',
'event-id-1',
ARRAY(NAMED_STRUCT(
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'type', 'AWS::IAM::Role'
)),
'AwsApiCall',
'2015-03-31',
'true',
'123456789012',
null,
null,
null,
'Management',
NAMED_STRUCT(
'tlsVersion', 'TLSv1.2',
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
'clientProvidedHostHeader', null
)
),
(
'1.08',
NAMED_STRUCT(
'type', 'IAMUser',
'principalId', 'AWS123456789012',
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
'accountId', '123456789012',
'invokedBy', null,
'accessKeyId', 'AKIA1234567890',
'userName', 'ExampleUser',
'sessionContext', NAMED_STRUCT(
'attributes', NAMED_STRUCT(
'mfaAuthenticated', 'true',
'creationDate', '2023-11-01T05:06:00Z'
),
'sessionIssuer', NAMED_STRUCT(
'type', 'Role',
'principalId', 'ARO123456789012',
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'userName', 'MyRole'
),
'ec2RoleDelivery', 'true',
'webIdFederationData', MAP()
)
),
'2023-11-01T05:06:00Z',
'sts.amazonaws.com',
'AssumeRole',
'us-east-1',
'198.51.100.45',
'AWS CLI',
null,
null,
null,
null,
null,
'request-id-2',
'event-id-2',
ARRAY(NAMED_STRUCT(
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'type', 'AWS::IAM::Role'
)),
'AwsApiCall',
'2015-03-31',
'true',
'123456789012',
null,
null,
null,
'Management',
NAMED_STRUCT(
'tlsVersion', 'TLSv1.2',
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
'clientProvidedHostHeader', null
)
);
29 changes: 29 additions & 0 deletions integ-test/src/integration/resources/aws-logs/vpc_flow.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
CREATE TABLE {table_name} (
version INT,
accountId STRING,
interfaceId STRING,
srcAddr STRING,
dstAddr STRING,
srcPort INT,
dstPort INT,
protocol BIGINT,
packets BIGINT,
bytes BIGINT,
start BIGINT,
`end` BIGINT,
action STRING,
logStatus STRING
)
USING csv
OPTIONS (
sep = ' ',
recursiveFileLookup = 'true'
);

INSERT INTO {table_name} VALUES
(1, '123456789012', 'eni-abc123', '10.0.0.1', '10.0.0.2', 12345, 80, 6, 10, 200, 1698814800, 1698814860, 'ACCEPT', 'OK'), -- 05:00:00 to 05:01:00 UTC
(2, '123456789012', 'eni-def456', '10.0.0.1', '10.0.0.2', 12346, 443, 6, 5, 150, 1698814900, 1698814960, 'ACCEPT', 'OK'), -- 05:01:40 to 05:02:40 UTC
(3, '123456789013', 'eni-ghi789', '10.0.0.3', '10.0.0.4', 12347, 22, 6, 15, 300, 1698815400, 1698815460, 'ACCEPT', 'OK'), -- 05:10:00 to 05:11:00 UTC
(4, '123456789013', 'eni-jkl012', '10.0.0.5', '10.0.0.6', 12348, 21, 6, 20, 400, 1698815500, 1698815560, 'REJECT', 'OK'), -- 05:11:40 to 05:12:40 UTC
(5, '123456789014', 'eni-mno345', '10.0.0.7', '10.0.0.8', 12349, 25, 6, 25, 500, 1698816000, 1698816060, 'ACCEPT', 'OK') -- 05:20:00 to 05:21:00 UTC
;
91 changes: 91 additions & 0 deletions integ-test/src/integration/resources/aws-logs/waf.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
CREATE TABLE {table_name} (
timestamp STRING,
webaclId STRING,
action STRING,
formatVersion INT,
httpRequest STRUCT<
clientIp: STRING,
country: STRING,
headers: ARRAY<STRUCT<
name: STRING,
value: STRING
>>,
uri: STRING,
args: STRING,
httpVersion: STRING,
httpMethod: STRING,
requestId: STRING
>,
httpSourceId STRING,
httpSourceName STRING,
requestBodySize INT,
requestBodySizeInspectedByWAF INT,
terminatingRuleId STRING,
terminatingRuleType STRING,
ruleGroupList ARRAY<STRUCT<
ruleId: STRING,
ruleAction: STRING
>>,
rateBasedRuleList ARRAY<STRUCT<
ruleId: STRING
>>,
nonTerminatingMatchingRules ARRAY<STRUCT<
ruleId: STRING
>>
)
USING json
OPTIONS (
recursivefilelookup = 'true'
);

INSERT INTO {table_name} VALUES
(
1698814800000, -- 2023-11-01T05:00:00Z
'webacl-12345',
'ALLOW',
1,
NAMED_STRUCT(
'clientIp', '192.0.2.1',
'country', 'US',
'headers', ARRAY(NAMED_STRUCT('name', 'User-Agent', 'value', 'Mozilla/5.0')),
'uri', '/index.html',
'args', 'query=example',
'httpVersion', 'HTTP/1.1',
'httpMethod', 'GET',
'requestId', 'req-1'
),
'source-1',
'http-source',
500,
450,
'rule-1',
'REGULAR',
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-1', 'ruleAction', 'ALLOW')),
ARRAY(),
ARRAY()
),
(
1698815400000, -- 2023-11-01T05:10:00Z
'webacl-67890',
'BLOCK',
1,
NAMED_STRUCT(
'clientIp', '192.0.2.2',
'country', 'CA',
'headers', ARRAY(NAMED_STRUCT('name', 'Referer', 'value', 'example.com')),
'uri', '/login.html',
'args', '',
'httpVersion', 'HTTP/2',
'httpMethod', 'POST',
'requestId', 'req-2'
),
'source-2',
'http-source',
750,
600,
'rule-2',
'RATE_BASED',
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-2', 'ruleAction', 'BLOCK')),
ARRAY(),
ARRAY()
);
Loading

0 comments on commit be1df0f

Please sign in to comment.