From 78a0d013c1f78c2fa4dca7950f71d7064d987fe5 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Fri, 8 Sep 2023 13:41:16 +0800 Subject: [PATCH] one dir --- .../account-change-feature-demo/Dockerfile | 54 -------- .../account-change-feature-demo/README.md | 31 ----- .../docker-compose.yml | 119 ------------------ .../nyc-taxi-feature-store-demo/README.md | 26 +++- 4 files changed, 23 insertions(+), 207 deletions(-) delete mode 100644 integration_tests/account-change-feature-demo/Dockerfile delete mode 100644 integration_tests/account-change-feature-demo/README.md delete mode 100644 integration_tests/account-change-feature-demo/docker-compose.yml diff --git a/integration_tests/account-change-feature-demo/Dockerfile b/integration_tests/account-change-feature-demo/Dockerfile deleted file mode 100644 index 1ba804cb2262c..0000000000000 --- a/integration_tests/account-change-feature-demo/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM rust:1.67 as feature-store-server - -USER root - -ENV WORK_DIR /opt/feature-store -RUN mkdir -p $WORK_DIR - -WORKDIR $WORK_DIR - -RUN apt update -RUN apt install -y python3 python3-pip wget ca-certificates -RUN apt install -y postgresql-client - -ADD ../nyc-taxi-feature-store-demo/server/model/requirements.txt $WORK_DIR/model-pipreqs.txt -ADD ../nyc-taxi-feature-store-demo/generator/requirements.txt $WORK_DIR/generator-pipreqs.txt -RUN pip3 install --upgrade pip -RUN pip3 install -r $WORK_DIR/model-pipreqs.txt -RUN pip3 install -r $WORK_DIR/generator-pipreqs.txt -RUN pip3 install risingwave - -RUN apt install -y lsof curl openssl libssl-dev pkg-config build-essential -RUN apt install -y cmake librdkafka-dev - -# Install .NET 6.0 -RUN wget https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -O packages-microsoft-prod.deb -RUN dpkg -i packages-microsoft-prod.deb -RUN rm packages-microsoft-prod.deb -RUN apt-get update && apt-get install -y dotnet-sdk-6.0 -RUN apt install -y liblttng-ust0 - -# `cargo build` included in ./build -ADD ../nyc-taxi-feature-store-demo/server $WORK_DIR/build/server -ADD ../nyc-taxi-feature-store-demo/simulator $WORK_DIR/build/simulator -RUN cargo build --manifest-path $WORK_DIR/build/server/Cargo.toml --release -RUN cargo build --manifest-path $WORK_DIR/build/simulator/Cargo.toml --release - -RUN cp $WORK_DIR/build/server/target/release/server $WORK_DIR/feature-store-server -RUN cp $WORK_DIR/build/simulator/target/release/simulator $WORK_DIR/feature-store-simulator -RUN rm -rf $WORK_DIR/build - -ADD ../nyc-taxi-feature-store-demo/server/model $WORK_DIR/server/model -ADD ../nyc-taxi-feature-store-demo/server/udf.py $WORK_DIR/udf.py -ADD ../nyc-taxi-feature-store-demo/generator $WORK_DIR/generator -ADD ../nyc-taxi-feature-store-demo/taxi-start.sql $WORK_DIR/taxi-start.sql -ADD ../nyc-taxi-feature-store-demo/mfa-start.sql $WORK_DIR/mfa-start.sql -RUN mkdir $WORK_DIR/run-sh -ADD ../nyc-taxi-feature-store-demo/run.sh $WORK_DIR/run-sh/ -ADD ../nyc-taxi-feature-store-demo/run-mfa.sh $WORK_DIR/run-sh/ - -RUN cp $WORK_DIR/run-sh/run-mfa.sh $WORK_DIR/run.sh;\ - -RUN chmod +x $WORK_DIR/run.sh && rm -rf $WORK_DIR/run-sh - -CMD ["sh", "-c", "sleep 10 && ./run.sh"] \ No newline at end of file diff --git a/integration_tests/account-change-feature-demo/README.md b/integration_tests/account-change-feature-demo/README.md deleted file mode 100644 index 55e3b8420e122..0000000000000 --- a/integration_tests/account-change-feature-demo/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Description - -#### Feature store demo. - -We use `simulators` to simulate data input. - -Then all messages will be sent to the `server` and written in `Kafka` -> `RisingWave`. `RisingWave` will process the data based on pre-defined operations. - -We also utilize the `simulator` to simulate user queries to our `feature`. The `server` will receive requests -> query data -> and return results. - -If we intend to modify our business logic, we simply need to update the materialized view within our `RisingWave` by using SQL statements. - -#### Specific case: - -This chapter is a simple demo of feature extraction in `RisingWave`, primarily showcasing the real-time feature aggregation and updating capabilities of `RisingWave`. - -In this case, we need to calculate the frequency and count of user account changes over a period of time. We mainly use SQL aggregation functions and UDFs (User-Defined Functions) to achieve this. - -Due to the similarity between the code in this demo and another code, the implementation code is located in the `nyc-taxi-feature-store-demo` folder. - -## Installation - -Run it in local. - -1. Build docker. Kafka RisingWave and Feature Store. - -```docker compose up --build``` - -2. Then we can get the simulation results for Feature store in `.log`. - -```cat .log/simulator_log``` \ No newline at end of file diff --git a/integration_tests/account-change-feature-demo/docker-compose.yml b/integration_tests/account-change-feature-demo/docker-compose.yml deleted file mode 100644 index 48ad49d2d8d4e..0000000000000 --- a/integration_tests/account-change-feature-demo/docker-compose.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -version: "3" -services: - kafka: - image: confluentinc/cp-kafka:7.1.0 - platform: linux/amd64 - hostname: kafka - container_name: kafka - ports: - - "29092:29092" - - "9092:9092" - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR - depends_on: - [ zookeeper ] - healthcheck: - test: [ "CMD-SHELL", "kafka-topics --bootstrap-server kafka:9092 --list" ] - interval: 5s - timeout: 10s - retries: 5 - - init-kafka: - image: confluentinc/cp-kafka:7.1.0 - depends_on: - - kafka - entrypoint: [ '/bin/sh', '-c' ] - command: | - " - # blocks until kafka is reachable - kafka-topics --bootstrap-server kafka:9092 --list - echo -e 'Creating kafka topics' - kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic taxi --replication-factor 1 --partitions 1 - echo -e 'Creating kafka topics' - kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic mfa --replication-factor 1 --partitions 1 - - echo -e 'Successfully created the following topics:' - kafka-topics --bootstrap-server kafka:9092 --list - " - - zookeeper: - image: confluentinc/cp-zookeeper:7.1.0 - platform: linux/amd64 - hostname: zookeeper - container_name: zookeeper - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - compactor-0: - extends: - file: ../../docker/docker-compose.yml - service: compactor-0 - compute-node-0: - extends: - file: ../../docker/docker-compose.yml - service: compute-node-0 - volumes: - - "../nyc-taxi-feature-store-demo/server/udf.py:/udf.py" - - "../nyc-taxi-feature-store-demo/mfa-start.sql:/mfa-start.sql" - - "../nyc-taxi-feature-store-demo/mfa-mock.sql:/mfa-mock.sql" - feature-store: - image: rust:1.67 - build: - context: . - target: feature-store-server - depends_on: - [kafka,meta-node-0,frontend-node-0] - volumes: - - "./log:/opt/feature-store/.log" - etcd-0: - extends: - file: ../../docker/docker-compose.yml - service: etcd-0 - frontend-node-0: - extends: - file: ../../docker/docker-compose.yml - service: frontend-node-0 - grafana-0: - extends: - file: ../../docker/docker-compose.yml - service: grafana-0 - meta-node-0: - extends: - file: ../../docker/docker-compose.yml - service: meta-node-0 - ports: - - "8815:8815" - depends_on: - [kafka] - minio-0: - extends: - file: ../../docker/docker-compose.yml - service: minio-0 - prometheus-0: - extends: - file: ../../docker/docker-compose.yml - service: prometheus-0 - connector-node: - extends: - file: ../../docker/docker-compose.yml - service: connector-node -volumes: - etcd-0: - external: false - grafana-0: - external: false - minio-0: - external: false - prometheus-0: - external: false -name: risingwave-compose - diff --git a/integration_tests/nyc-taxi-feature-store-demo/README.md b/integration_tests/nyc-taxi-feature-store-demo/README.md index 30b312e18e627..02021f32e56af 100644 --- a/integration_tests/nyc-taxi-feature-store-demo/README.md +++ b/integration_tests/nyc-taxi-feature-store-demo/README.md @@ -1,6 +1,6 @@ # Description -#### Feature store demo. +Feature store demo. We use `simulators` to simulate data input. @@ -10,7 +10,8 @@ We also utilize the `simulator` to simulate user queries to our `feature`. The ` If we intend to modify our business logic, we simply need to update the materialized view within our `RisingWave` by using SQL statements. -#### Specific case: +# Nyc taxi feature store +#### Case Description The case in this chapter is a New York taxi fare prediction. We need to predict the taxi fare based on the starting and ending points of the trip. @@ -20,7 +21,7 @@ When a user needs to make a prediction using these features, they can retrieve a When a user needs to make a prediction using these features, they can provide their starting and ending points, query the corresponding features in `RisingWave`, and inject them into a machine learning model for real-time fare prediction. -## Installation +#### Installation 1. Build docker. Kafka RisingWave and Feature Store. @@ -39,4 +40,23 @@ The Feature Store system performs several tasks in sequence: 2. Then we can get the simulation results for Feature store in `.log`. +```cat .log/simulator_log``` + +# Account change feature store +#### Case Description + +This chapter is a simple demo of feature extraction in `RisingWave`, primarily showcasing the real-time feature aggregation and updating capabilities of `RisingWave`. + +In this case, we need to calculate the frequency and count of user account changes over a period of time. We mainly use SQL aggregation functions and UDFs (User-Defined Functions) to achieve this. + +Due to the similarity between the code in this demo and another code, the implementation code is located in the `nyc-taxi-feature-store-demo` folder. + +#### Installation + +1. Build docker. Kafka RisingWave and Feature Store. + +```docker compose build --build-arg BUILD_ARG=mfa``` + +2. Then we can get the simulation results for Feature store in `.log`. + ```cat .log/simulator_log``` \ No newline at end of file