Skip to content

Commit

Permalink
Run integration tests with the new dynamodb lock set up
Browse files Browse the repository at this point in the history
  • Loading branch information
rtyler committed Jan 6, 2024
1 parent d43cd51 commit 0f259a2
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 12 deletions.
7 changes: 3 additions & 4 deletions Dockerfile.setup
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
FROM python:3.10-alpine
FROM python:alpine

RUN apk add --no-cache bash openssh ca-certificates jq curl openssl perl git zip \
RUN apk add --no-cache aws-cli bash openssh ca-certificates jq curl openssl perl git zip \
&& apk add --no-cache --virtual .build-deps gcc make openssl-dev libffi-dev musl-dev linux-headers \
&& apk add --no-cache libintl icu-libs libc6-compat \
&& apk add --no-cache bash-completion \
&& update-ca-certificates
RUN pip3 install azure-cli
RUN pip3 install awscli
RUN az config set extension.use_dynamic_install=yes_without_prompt

ENTRYPOINT ["/bin/bash"]
ENTRYPOINT ["/bin/bash"]
15 changes: 8 additions & 7 deletions bin/localstack-setup_emails.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,11 @@ wait_for "DynamoDB" "aws dynamodb list-tables --endpoint-url=$ENDPOINT"

echo "Create delta-rs lock table in dynamo"
aws dynamodb delete-table --table-name locks --endpoint-url=$ENDPOINT > /dev/null 2>&1
aws dynamodb create-table --table-name locks --endpoint-url=$ENDPOINT \
--attribute-definitions \
AttributeName=key,AttributeType=S \
--key-schema \
AttributeName=key,KeyType=HASH \
--provisioned-throughput \
ReadCapacityUnits=10,WriteCapacityUnits=10 > /dev/null
aws dynamodb create-table \
--endpoint-url=$ENDPOINT \
--table-name locks \
--attribute-definitions AttributeName=tablePath,AttributeType=S \
AttributeName=fileName,AttributeType=S \
--key-schema AttributeName=tablePath,KeyType=HASH \
AttributeName=fileName,KeyType=RANGE \
--billing-mode PAY_PER_REQUEST
3 changes: 2 additions & 1 deletion tests/emails_s3_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ async fn when_rebalance_happens() {
}

async fn run_emails_s3_tests(initiate_rebalance: bool) {
deltalake_aws::register_handlers(None);
helpers::init_logger();
let topic = format!("emails_s3-{}", Uuid::new_v4());
let table = prepare_table(&topic).await;
Expand Down Expand Up @@ -80,7 +81,7 @@ fn create_options(name: &str) -> IngestOptions {
env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
env::set_var("AWS_REGION", "us-east-2");
env::set_var("AWS_STORAGE_ALLOW_HTTP", "true");
env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks");
env::set_var("DELTA_DYNAMO_TABLE_NAME", "locks");
env::set_var("DYNAMO_LOCK_OWNER_NAME", name);
env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", "emails_s3_tests");
env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100");
Expand Down
5 changes: 5 additions & 0 deletions tests/helpers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,11 @@ pub async fn create_and_run_kdi(
Runtime,
) {
init_logger();

#[cfg(feature = "s3")]
deltalake_aws::register_handlers(None);
#[cfg(feature = "azure")]
deltalake_azure::register_handlers(None);
println!("OPTS!: {:?}", opts);
let topic = format!("{}-{}", app_id, Uuid::new_v4());
let table = create_local_table(schema, delta_partitions, &topic);
Expand Down

0 comments on commit 0f259a2

Please sign in to comment.