Hadoop开发环境搭建

第一关 配置开发环境 - JavaJDK的配置

按顺序输入运行即可

apt remove -y openjdk-8-jdk
apt autoremove -y
cd /opt
wget https://repo.huaweicloud.com/java/jdk/8u171-b11/jdk-8u171-linux-x64.tar.gz
tar -zxvf jdk-8u171-linux-x64.tar.gz
mkdir -p /app
mv jdk1.8.0_171 /app/
sed -i '/JAVA_HOME/d' /etc/profile
echo "export JAVA_HOME=/app/jdk1.8.0_171" >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
source /etc/profile
java -version

第二关 配置开发环境 - Hadoop安装与伪分布式集群搭建

注意：#后的为注释，请不要复制到命令行，按顺序输入代码部分即可。

# ==================== 1. 解压安装 ====================
cd /opt
tar -zxvf hadoop-3.1.0.tar.gz
mkdir -p /app
mv hadoop-3.1.0 /app/hadoop

# ==================== 2. SSH 免密登录 ====================
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

# ==================== 3. 配置 Hadoop 环境变量 ====================
cd /app/hadoop/etc/hadoop
echo 'export JAVA_HOME=/app/jdk1.8.0_171' >> hadoop-env.sh
echo 'export JAVA_HOME=/app/jdk1.8.0_171' >> yarn-env.sh

# ==================== 4. 配置 core-site.xml ====================
cat > core-site.xml << 'EOF'
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/hadoop/tmp</value>
    </property>
</configuration>
EOF

# ==================== 5. 配置 hdfs-site.xml ====================
cat > hdfs-site.xml << 'EOF'
<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/usr/hadoop/hdfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/usr/hadoop/hdfs/data</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>
EOF

# ==================== 6. 配置 mapred-site.xml ====================
cp mapred-site.xml.template mapred-site.xml
cat > mapred-site.xml << 'EOF'
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>
EOF

# ==================== 7. 配置 yarn-site.xml ====================
cat > yarn-site.xml << 'EOF'
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>
EOF

# ==================== 8. 创建数据目录 ====================
mkdir -p /usr/hadoop/tmp
mkdir -p /usr/hadoop/hdfs/data
mkdir -p /usr/hadoop/hdfs/name

# ==================== 9. 配置系统环境变量 ====================
echo 'export HADOOP_HOME=/app/hadoop' >> /etc/profile
echo 'export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH' >> /etc/profile
source /etc/profile

# ==================== 10. 修改启动脚本（root用户） ====================
cd /app/hadoop/sbin
sed -i '1iHDFS_DATANODE_USER=root\nHDFS_NAMENODE_USER=root\nHDFS_SECONDARYNAMENODE_USER=root' start-dfs.sh
sed -i '1iHDFS_DATANODE_USER=root\nHDFS_NAMENODE_USER=root\nHDFS_SECONDARYNAMENODE_USER=root' stop-dfs.sh
sed -i '1iYARN_RESOURCEMANAGER_USER=root\nYARN_NODEMANAGER_USER=root' start-yarn.sh
sed -i '1iYARN_RESOURCEMANAGER_USER=root\nYARN_NODEMANAGER_USER=root' stop-yarn.sh

# ==================== 11. 格式化 HDFS ====================
hadoop namenode -format
# 如果提示输入，输入 Y 确认

# ==================== 12. 启动 Hadoop ====================
start-dfs.sh
start-yarn.sh

# ==================== 13. 验证安装 ====================
jps