-
Notifications
You must be signed in to change notification settings - Fork 45
/
setup.sh
executable file
·204 lines (160 loc) · 6.7 KB
/
setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
#! /bin/bash
start_dir=$PWD
echo "-- Start dir is ${start_dir}"
echo "-- Configure and optimize the OS"
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local
# add tuned optimization https://www.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_admin_performance.html
echo "vm.swappiness = 1" >> /etc/sysctl.conf
sysctl vm.swappiness=1
timedatectl set-timezone UTC
echo "-- Install Java OpenJDK8 and other tools"
yum install -y java-1.8.0-openjdk-devel vim wget curl git bind-utils rng-tools
yum install -y epel-release
yum install -y python-pip
cp /usr/lib/systemd/system/rngd.service /etc/systemd/system/
systemctl daemon-reload
systemctl start rngd
systemctl enable rngd
#echo "-- Installing requirements for Stream Messaging Manager"
#yum install -y gcc-c++ make
#curl -sL https://rpm.nodesource.com/setup_10.x | sudo -E bash -
#yum install nodejs -y
#npm install forever -g
# Check input parameters
case "$1" in
aws)
echo "server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4" >> /etc/chrony.conf
systemctl restart chronyd
;;
azure)
# default root disk in most Azure Centos images is often small so second disk may be needed for /opt
# if you are already using /opt before the CDH install you may need to adjust this step as appropriate
# the temp disk used in the Cloudera Centos image on Azure on /mnt/resource may be an option if not persisting image
# umount /mnt/resource
# mount /dev/sdb1 /opt
echo "server time.windows.com prefer iburst minpoll 4 maxpoll 4" >> /etc/chrony.conf
systemctl restart chronyd
;;
gcp)
;;
*)
echo $"Usage: $0 {aws|azure|gcp} template-file [docker-device]"
echo $"example: ./setup.sh azure templates/essential.json"
echo $"example: ./setup.sh aws template/cml.json /dev/xvdb"
exit 1
esac
TEMPLATE=$2
DOCKERDEVICE=$3
echo "-- Configure networking"
PUBLIC_IP=`curl https://api.ipify.org/`
hostnamectl set-hostname `hostname -f`
echo "`hostname -I` `hostname`" >> /etc/hosts
sed -i "s/HOSTNAME=.*/HOSTNAME=`hostname`/" /etc/sysconfig/network
systemctl disable firewalld
systemctl stop firewalld
setenforce 0
sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
echo "-- Install CM and MariaDB"
## CM 7
wget https://archive.cloudera.com/cm7/7.4.4/redhat7/yum/cloudera-manager-trial.repo -P /etc/yum.repos.d/
## MariaDB
cat - >/etc/yum.repos.d/MariaDB.repo <<EOF
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.5/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1
EOF
yum clean all
rm -rf /var/cache/yum/
yum repolist
## CM
yum install -y cloudera-manager-agent cloudera-manager-daemons cloudera-manager-server
## MariaDB
yum install -y MariaDB-server MariaDB-client
cat conf/mariadb.config > /etc/my.cnf
echo "--Enable and start MariaDB"
systemctl enable mariadb
systemctl start mariadb
echo "-- Install JDBC connector"
wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-5.1.49.tar.gz -P ~
tar zxf ~/mysql-connector-java-5.1.49.tar.gz -C ~
mkdir -p /usr/share/java/
cp ~/mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar /usr/share/java/mysql-connector-java.jar
rm -rf ~/mysql-connector-java-5.1.49*
# Install the MySQL Client - required for lastest versions of Hue with MySQL-MariaDB - needs a python 3.8 or similar install to work
#yum install -y xmlsec1 xmlsec1-openssl
#export PATH=/usr/local/bin:$PATH
#yum -y install mariadb-devel
#pip3.8 install mysqlclient
echo "-- Create DBs required by CM"
mysql -u root < scripts/create_db.sql
echo "-- Secure MariaDB"
mysql -u root < scripts/secure_mariadb.sql
echo "-- Prepare CM database 'scm'"
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm cloudera
## PostgreSQL see: https://www.postgresql.org/download/linux/redhat/
yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
#yum install -y postgresql
sudo yum install -y postgresql11-server
pip install psycopg2==2.7.5 --ignore-installed
#pip install psycopg2==2.9.5 --ignore-installed - got some erros to fix yet on this version install needed for later CDP Base versions
echo 'LC_ALL="en_US.UTF-8"' >> /etc/locale.conf
/usr/pgsql-11/bin/postgresql-11-setup initdb
cat conf/pg_hba.conf > /var/lib/pgsql/11/data/pg_hba.conf
cat conf/postgresql.conf > /var/lib/pgsql/11/data/postgresql.conf
echo "--Enable and start pgsql"
#systemctl enable postgresql-9.6
#systemctl start postgresql-9.6
sudo systemctl enable postgresql-11
sudo systemctl start postgresql-11
echo "-- Create DBs required by CM"
sudo -u postgres psql <<EOF
CREATE DATABASE ranger;
CREATE USER ranger WITH PASSWORD 'cloudera';
GRANT ALL PRIVILEGES ON DATABASE ranger TO ranger;
CREATE DATABASE das;
CREATE USER das WITH PASSWORD 'cloudera';
GRANT ALL PRIVILEGES ON DATABASE das TO das;
EOF
echo "-- Install CSDs"
# install local CSDs
mv ~/*.jar /opt/cloudera/csd/
mv /home/centos/*.jar /opt/cloudera/csd/
chown cloudera-scm:cloudera-scm /opt/cloudera/csd/*
chmod 644 /opt/cloudera/csd/*
echo "-- Install local parcels"
mv ~/*.parcel ~/*.parcel.sha /opt/cloudera/parcel-repo/
mv /home/centos/*.parcel /home/centos/*.parcel.sha /opt/cloudera/parcel-repo/
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
echo "-- Enable passwordless root login via rsa key"
ssh-keygen -f ~/myRSAkey -t rsa -N ""
mkdir ~/.ssh
cat ~/myRSAkey.pub >> ~/.ssh/authorized_keys
chmod 400 ~/.ssh/authorized_keys
ssh-keyscan -H `hostname` >> ~/.ssh/known_hosts
sed -i 's/.*PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config
systemctl restart sshd
echo "-- Start CM, it takes about 2 minutes to be ready"
systemctl start cloudera-scm-server
while [ `curl -s -X GET -u "admin:admin" http://localhost:7180/api/version` -z ] ;
do
echo "waiting 10s for CM to come up..";
sleep 10;
done
echo "-- Now CM is started and the next step is to automate using the CM API"
wget https://bootstrap.pypa.io/pip/2.7/get-pip.py
python ./get-pip.py
pip install --upgrade cm_client
#pip install --upgrade pip cm_client
echo "-- Kicking off install from ${start_dir}"
cd ${start_dir}
sed -i "s/YourHostname/`hostname -f`/g" $TEMPLATE
#sed -i "s/YourCDSWDomain/cdsw.$PUBLIC_IP.nip.io/g" $TEMPLATE
sed -i "s/YourPrivateIP/`hostname -I | tr -d '[:space:]'`/g" $TEMPLATE
#sed -i "s#YourDockerDevice#$DOCKERDEVICE#g" $TEMPLATE
sed -i "s/YourHostname/`hostname -f`/g" scripts/create_cluster.py
python scripts/create_cluster.py $TEMPLATE