CentOS7安装CDH6.3.2

1、CDH安装包地址

通过网盘分享的文件：CDH6.3.2.zip
链接: https://pan.baidu.com/s/1QVcPclxU4dB1TfnlkRoDxw?pwd=3env

2、安装脚本

#!/bin/bash
# CDH 6.3.2 自动安装脚本 - 单机伪分布式
# 安装组件: HDFS + YARN + Hive + ZooKeeper

set -e

HOST_IP="192.168.2.113"
HOSTNAME="root"
MYSQL_ROOT_PASSWORD="xxx"

echo "=========================================="
echo "CDH 6.3.2 自动化安装"
echo "=========================================="

# 1. 配置主机名和hosts
echo "[1/10] 配置网络..."
hostnamectl set-hostname $HOSTNAME
cat > /etc/hosts << HOSTEOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
$HOST_IP    $HOSTNAME
HOSTEOF

# 2. 关闭防火墙和SELinux
echo "[2/10] 关闭防火墙和SELinux..."
systemctl stop firewalld 2>/dev/null || true
systemctl disable firewalld 2>/dev/null || true
setenforce 0 2>/dev/null || true
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
sed -i 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config

# 3. 配置系统参数
echo "[3/10] 配置系统参数..."
echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true
echo never > /sys/kernel/mm/transparent_hugepage/defrag 2>/dev/null || true
echo "vm.swappiness=10" >> /etc/sysctl.conf
sysctl -p
echo '* soft nproc 65535' >> /etc/security/limits.conf
echo '* hard nproc 65535' >> /etc/security/limits.conf
echo '* soft nofile 65535' >> /etc/security/limits.conf
echo '* hard nofile 65535' >> /etc/security/limits.conf

# 4. 安装基础依赖
echo "[4/10] 安装基础依赖..."
yum install -y -q wget curl net-tools vim unzip zip \
    bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi \
    fuse portmap fuse-libs httpd mod_ssl openssl-devel \
    libxslt chrony ntpdate 2>/dev/null || yum install -y wget curl net-tools vim unzip

# 5. 配置时间同步
echo "[5/10] 配置时间同步..."
systemctl enable chronyd 2>/dev/null || true
systemctl start chronyd 2>/dev/null || true
timedatectl set-timezone Asia/Shanghai 2>/dev/null || true

# 6. 安装JDK
echo "[6/10] 安装JDK..."
# 卸载系统自带的OpenJDK
rpm -qa | grep -E "java|jdk" | xargs -I {} rpm -e --nodeps {} 2>/dev/null || true

# 从CDH包中查找JDK
JDK_RPM=$(find /opt/software -name "oracle-j2sdk*.rpm" 2>/dev/null | head -1)
if [[ -n "$JDK_RPM" ]]; then
    rpm -ivh $JDK_RPM 2>/dev/null || true
fi

# 配置环境变量
if [[ -d "/usr/java/jdk1.8.0_181-cloudera" ]]; then
    echo 'export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera' > /etc/profile.d/jdk.sh
    echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile.d/jdk.sh
    source /etc/profile.d/jdk.sh
fi

# 创建CDH数据库
echo "[8/10] 创建CDH数据库..."
# 9. 安装Cloudera Manager
echo "[10/10] 安装Cloudera Manager..."
mkdir -p /opt/cloudera-manager

# 查找并复制RPM包
find /opt/software/CDH6.3.2 -name "*.rpm" -exec cp {} /opt/cloudera-manager/ \; 2>/dev/null || true

cd /opt/cloudera-manager

# 安装RPM包
for rpm in cloudera-manager-daemons-*.rpm; do
    [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true
done

for rpm in cloudera-manager-agent-*.rpm; do
    [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true
done

for rpm in cloudera-manager-server-*.rpm; do
    [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true
done

# 配置MySQL JDBC驱动
mkdir -p /usr/share/java
CONNECTOR=$(find /opt/software -name "mysql-connector-java*.jar" 2>/dev/null | head -1)
[[ -n "$CONNECTOR" ]] && cp "$CONNECTOR" /usr/share/java/mysql-connector-java.jar

# 配置CM
cat > /etc/cloudera-scm-agent/config.ini << CMEOF
[General]
server_host=cdh-node
server_port=7182
CMEOF

cat > /etc/cloudera-scm-server/db.properties << DBEOF
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=192.168.2.113:3306
com.cloudera.cmf.db.name=scm
com.cloudera.cmf.db.user=root
com.cloudera.cmf.db.password=xxx
com.cloudera.cmf.db.setupType=EXTERNAL
DBEOF

# 配置Parcel
mkdir -p /opt/cloudera/parcel-repo
find /opt/software/cdh_extracted -name "*.parcel" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true
find /opt/software/cdh_extracted -name "*.sha*" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true
find /opt/software/cdh_extracted -name "manifest.json" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true

cd /opt/cloudera/parcel-repo
for f in *.sha1; do [[ -f "$f" ]] && mv "$f" "${f%.sha1}.sha" 2>/dev/null || true; done

chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo 2>/dev/null || true

# 初始化数据库
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h192.168.2.113 -uroot -p"$MYSQL_ROOT_PASSWORD" scm scm scm 2>/dev/null || echo "数据库初始化可能已存在"

# 启动服务
echo "启动Cloudera Manager服务..."
systemctl start cloudera-scm-server 2>/dev/null || /opt/cloudera/cm/etc/init.d/cloudera-scm-server start 2>/dev/null || true
systemctl start cloudera-scm-agent 2>/dev/null || /opt/cloudera/cm/etc/init.d/cloudera-scm-agent start 2>/dev/null || true
systemctl enable cloudera-scm-server 2>/dev/null || true
systemctl enable cloudera-scm-agent 2>/dev/null || true

echo ""
echo "=========================================="
echo "CDH 6.3.2 基础安装完成！"
echo "=========================================="
echo ""
echo "请等待2-3分钟后访问: http://$HOST_IP:7180"
echo "用户名: admin"
echo "密码: admin"
echo ""
echo "Web UI配置步骤:"
echo "1. 选择版本: Cloudera Express (免费版)"
echo "2. 集群名称: 自定义"
echo "3. 选择主机: 选择 cdh-node"
echo "4. 选择Parcel: CDH-6.3.2"
echo "5. 检查主机: 修复警告(交换空间、透明大页)"
echo "6. 选择服务: 自定义 - HDFS + YARN + Hive + ZooKeeper"
echo "7. 配置数据库: 使用之前创建的hive用户"
echo "=========================================="