Kubernetes运行大数据组件-运行hive

小森饭2024-10-30 13:46

部署组件

● mysql

● hive-metastore

● hive-server2

配置文件

xml 复制代码 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hive
data:
  hive-site.xml: |-
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>javax.jdo.option.ConnectionUserName</name>
            <value>root</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionPassword</name>
            <value>xxxxx</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionURL</name>
            <value>jdbc:mysql://192.168.199.58:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>javax.jdo.option.ConnectionDriverName</name>
            <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
            <name>hive.cli.print.current.db</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.cli.print.header</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.exec.post.hooks</name>
            <value>org.apache.atlas.hive.hook.HiveHook</value>
        </property>
        <property>
            <name>hive.metastore.uris</name>
            <value>thrift://192.168.199.57:9083</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>hive.exec.scratchdir</name>
            <value>/user/hive/tmp</value>
        </property>
        <property>
            <name>hive.metastore.warehouse.dir</name>
            <value>/user/hive/warehouse</value>
        </property>
        <property>
            <name>hive.querylog.location</name>
            <value>/tmp/hive/querylog</value>
        </property>
        <property>
            <name>hive.server2.webui.host</name>
            <value>0.0.0.0</value>
        </property>
        <property>
            <name>hive.server2.webui.port</name>
            <value>10002</value>
        </property>
        <property>
            <name>hive.server2.thrift.port</name>
            <value>10000</value>
        </property>
    </configuration>

mysql部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-mysql
  labels:
    app: hive-mysql
spec:
  selector:
    matchLabels:
      app: hive-mysql
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-mysql
    spec:
      containers:
        - name: hive-mysql
          image: mysql:5.7.32
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 200m
              memory: 500Mi
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: xxxxxx
          args:
            - --character-set-server=utf8mb4
            - --collation-server=utf8mb4_unicode_ci
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: data
              mountPath: /var/lib/mysql
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: data
          hostPath:
            path: /var/lib/mysql
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-mysql: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

hive metastore部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-metastore
  labels:
    app: hive-metastore
spec:
  selector:
    matchLabels:
      app: hive-metastore
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-metastore
    spec:
      initContainers:
        - name: mysql-init
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          command:	# 通过schematool校验数据库，决定是否初始化mysql
            - "sh"
            - "-c"
            - "if schematool -dbType mysql -validate; then schematool -dbType mysql -initSchema; fi"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      containers:
        - name: hive-metastore
          image: harbor.gistack.cn/library/hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:																					# 根据规划修改
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service metastore -v"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-metastore: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

mysql和hive-metastore部署

shell 复制代码 
> kubectl.exe create -f .\hive-config.yaml -n bigdata
configmap/hive created
> kubectl.exe create -f .\hive-mysql.yaml -n bigdata 
deployment.apps/hive-mysql created
> kubectl.exe create -f .\hive-metastore.yaml -n bigdata
deployment.apps/hive-metastore created
>

hive-metastore初始化数据库和运行情况：

hvie-server2部署

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hiveserver2
  labels:
    app: hiveserver2
spec:
  selector:
    matchLabels:
      app: hiveserver2
  replicas: 1
  template:
    metadata:
      labels:
        app: hiveserver2
    spec:
      containers:
        - name: hiveserver2
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service hiveserver2"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hiveserver2: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"
shell 复制代码 
> kubectl.exe create -f .\hiveserver2.yaml -n bigdata
deployment.apps/hiveserver2 created
>

访问hive web服务：

通过SQL客户端连接：

建表测试：

插入数据，查看任务：

相关推荐
星幻元宇VR9 小时前
VR环保学习机|科技助力绿色教育新模式
大数据·科技·学习·安全·vr·虚拟现实
CryptoPP10 小时前
开发者指南：构建实时期货黄金数据监控系统
大数据·数据结构·笔记·金融·区块链
阿里云云原生10 小时前
SLS 智能问答助手：秒解游戏运营客服难题
云原生
ZGi.ai11 小时前
生产级 Agent 编排 从单一 LLM 调用到多智能体工作流的工程设计
大数据·数据库·人工智能
天远数科11 小时前
分布式系统实战：基于天远二手车估值API构建高可用车辆估值微服务
大数据·微服务·云原生·架构
中国胖子风清扬12 小时前
Camunda 8 概念详解：梳理新一代工作流引擎的核心概念与组件
java·spring boot·后端·spring cloud·ai·云原生·spring webflux
殷紫川12 小时前
吃透云原生可观测：Metrics、Logging、Tracing 架构底层逻辑与实战全指南
云原生·架构
阿里云云原生12 小时前
AI Agent 越来越难迭代，你缺少的不是功能
云原生
馨谙12 小时前
万字详解 MCP 协议：AI 智能体连接外部世界的 “通用神经接口”
人工智能·云原生
码农小白AI13 小时前
AI审核加持的IACheck：塔吊与施工电梯安全监测系统检测报告如何实现高效合规与风险可控
大数据·人工智能·安全
热门推荐
01GitHub 镜像站点02Qwen3.5 开源全解析：从 0.8B 到 397B，代际升级 + 全场景选型指南03围棋-html版本04小黑课堂计算机二级WPSoffice题库软件下载安装教程（2026年3月最新版）05班级宠物园部署指南06OpenClaw 使用和管理 MCP 完全指南07UV安装并设置国内源08“wsl --install -d Ubuntu-22.04”下载慢，中国地区离线安装 Ubuntu 22.04 WSL方法（亲测2025年5月6日）09让 Trae IDE 智能体 “读懂”文档 Excel+PDF+DOCX ：mcp-documents-reader 工具使用指南10【计算机一级WPSoffice】小黑课堂题库软件下载安装教程（2026年3月最新版）