-
Notifications
You must be signed in to change notification settings - Fork 16
/
Copy path.pipeline.yml
82 lines (69 loc) · 2.56 KB
/
.pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
pipeline:
create_cluster:
image: banzaicloud/pipeline-client:0.1.0
log_level: info
log_format: text
cluster_name: "spark-pi-cluster"
cluster_location: "eu-west-1"
cluster_state: "created"
node_image: ami-294ffd50
node_instance_type: m4.xlarge
node_spot_price: 0.2
node_min_count: 1
node_max_count: 2
master_image: ami-294ffd50
master_instance_type: m4.xlarge
deployment_name: "pipeline-cluster-monitor"
deployment_release_name: "monitor"
deployment_state: "created"
secrets: [ plugin_endpoint, plugin_username, plugin_password]
remote_checkout:
image: banzaicloud/k8s-proxy:0.1.0
original_image: plugins/git
remote_build:
image: banzaicloud/k8s-proxy:0.1.0
original_image: maven:3.5-jdk-8
original_commands:
- mvn clean package -s settings.xml
prepare_spark_deps:
image: banzaicloud/pipeline-client:0.1.0
log_level: info
log_format: text
cluster_name: "spark-pi-cluster"
cluster_location: "eu-west-1"
cluster_state: "created"
node_image: ami-294ffd50
node_instance_type: m4.xlarge
node_spot_price: 0.2
node_min_count: 1
node_max_count: 2
master_image: ami-294ffd50
master_instance_type: m4.xlarge
deployment_name: "spark"
deployment_release_name: "spark"
deployment_state: "created"
secrets: [ plugin_endpoint, plugin_username, plugin_password]
run:
image: banzaicloud/k8s-proxy:0.1.0
original_image: banzaicloud/spark-submit-k8s:0.1.0
pod_service_account: spark
pull: true
spark_deploy_mode: cluster
spark_class: banzaicloud.SparkPi
spark_kubernetes_local_deploy: true
spark_kubernetes_namespace: default
spark_app_name: sparkpi
spark_local_dir: /tmp/spark-local
spark_kubernetes_driver_docker_image: banzaicloud/spark-driver:v2.2.0-k8s-1.0.179
spark_kubernetes_executor_docker_image: banzaicloud/spark-executor:v2.2.0-k8s-1.0.179
spark_kubernetes_initcontainer_docker_image: banzaicloud/spark-init:v2.2.0-k8s-1.0.179
spark_dynamic_allocation: true
spark_kubernetes_resourcestagingserver_uri: http://spark-rss:10000
spark_kubernetes_resourcestagingserver_internal_uri: http://spark-rss:10000
spark_shuffle_service_enabled: true
spark_kubernetes_shuffle_namespace: default
spark_kubernetes_shuffle_labels: "app=spark-shuffle-service,spark-version=2.2.0"
spark_kubernetes_authenticate_driver_serviceaccount_name: "spark"
spark_metrics_conf: /opt/spark/conf/metrics.properties
spark_app_source: target/spark-pi-1.0-SNAPSHOT.jar
spark_app_args: 1000