物联网架构之CDH

1、常规初始化操作

三个节点都需要执行

hostnamectl set-hostname cdhmaster

yum -y install vim lrzsz net-tools

cat >>/etc/hosts<<EOF

192.168.180.210 cdhmaster

192.168.180.200 cdhslave01

192.168.180.190 cdhslave02

EOF

systemctl stop firewalld.service && systemctl disable firewalld.service

sed -i '/^SELINUX=/s/enforcing/disabled/' /etc/selinux/config && setenforce 0

2、配置时间同步

cdhmaster:

yum install chrony -y

vim /etc/chrony.conf

server ntp6.aliyun.com iburst

allow 192.168.0.0/16

systemctl enable chronyd.service && systemctl restart chronyd.service

chronyc sources && chronyc -a makestep

cdhslave01/cdhslave02:

yum install chrony -y

vim /etc/chrony.conf

server cdhmaster iburst

systemctl enable chronyd.service && systemctl restart chronyd.service && chronyc sources

3、配置本机SSH免密码登录

ssh-keygen -t rsa

ssh-copy-id cdhmaster

ssh-copy-id cdhslave01

ssh-copy-id cdhslave02

4、安装jdk

三个节点都要执行

tar zxf jdk-8u171-linux-x64.tar.gz -C /opt/

mv /opt/jdk1.8.0_171/ /opt/jdk

cat >>/etc/profile<<EOF

export JAVA_HOME=/opt/jdk

export PATH= J A V A H O M E / b i n : JAVA_HOME/bin: JAVAHOME/bin:PATH

EOF

source /etc/profile && java -version

5、禁用交换分区和透明大页

三个节点都要操作

sysctl -w vm.swappiness=0

echo "vm.swappiness=0" >> /etc/sysctl.conf

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local

echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local

chmod +x /etc/rc.d/rc.local

6、安装数据库

只在主节点cdhmastar上操作

rpm -qa | grep mariadb

rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64

yum install -y libaio

useradd mysql -s /sbin/nologin

tar zvxf mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz

mv mysql-5.7.22-linux-glibc2.12-x86_64 /usr/local/mysql

cd /usr/local/mysql/bin

./mysqld --initialize --user=mysql --datadir=/usr/local/mysql/data

自动生成数据库密码 IrszF>8vd(gy

cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld

vim /etc/my.cnf

[client]

port = 3306

socket = /usr/local/mysql/data/mysql.sock

[mysqld]

port = 3306

socket = /usr/local/mysql/data/mysql.sock

pid_file = /usr/local/mysql/data/mysql.pid

datadir = /usr/local/mysql/data

default_storage_engine = InnoDB

max_allowed_packet = 512M

max_connections = 2048

open_files_limit = 65535

skip-name-resolve

lower_case_table_names=1

character-set-server = utf8mb4

collation-server = utf8mb4_unicode_ci

init_connect='SET NAMES utf8mb4'

innodb_buffer_pool_size = 1024M

innodb_log_file_size = 2048M

innodb_file_per_table = 1

innodb_flush_log_at_trx_commit = 0

key_buffer_size = 64M

log-error = /usr/local/mysql/data/mysql_error.log

log-bin = /usr/local/mysql/data/mysql-bin

slow_query_log = 1

slow_query_log_file = /usr/local/mysql/data/mysql_slow_query.log

long_query_time = 5

tmp_table_size = 32M

max_heap_table_size = 32M

query_cache_type = 0

query_cache_size = 0

server-id=1

/etc/init.d/mysqld start

ln -s /usr/local/mysql/bin/* /usr/sbin/

mysqladmin -uroot -p password '123456'

mysql -u root -p123456

alter user 'root'@'localhost';

alter user 'root'@'localhost' identified by '123456';

create database hive default charset utf8 collate utf8_general_ci;

create database oozie default charset utf8 collate utf8_general_ci;

grant all privileges on . to 'cdh'@'localhost' identified by '123456' with grant option;

grant all privileges on . to 'cdh'@'%' identified by '123456' with grant option;

grant all privileges on . to 'root'@'127.0.0.1' identified by '123456' with grant option;

grant all privileges on . to 'scm'@'127.0.0.1' identified by '123456' with grant option;

flush privileges;

7、安装CDH

所有节点上安装依赖包

yum install -y psmisc libxslt libxslt-python perl

上传cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz、mysql-connector-java-5.1.46.tar.gz到服务器

tar zxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/

mv /opt/cm-5.14.0/ /opt/cm

tar zxf mysql-connector-java-5.1.46.tar.gz

cd mysql-connector-java-5.1.46

chmod +x mysql-connector-java-5.1.46-bin.jar

cp mysql-connector-java-5.1.46-bin.jar /opt/cm/share/cmf/lib/mysql-connector-java.jar

mkdir /usr/share/java

cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

在三台机器上创建cloudera-scm用户

useradd --system --home=/opt/cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm