Kubernetes运行大数据组件-运行hive

小森饭2024-10-30 13:46

部署组件

● mysql

● hive-metastore

● hive-server2

配置文件

xml 复制代码 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hive
data:
  hive-site.xml: |-
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>javax.jdo.option.ConnectionUserName</name>
            <value>root</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionPassword</name>
            <value>xxxxx</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionURL</name>
            <value>jdbc:mysql://192.168.199.58:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>javax.jdo.option.ConnectionDriverName</name>
            <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
            <name>hive.cli.print.current.db</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.cli.print.header</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.exec.post.hooks</name>
            <value>org.apache.atlas.hive.hook.HiveHook</value>
        </property>
        <property>
            <name>hive.metastore.uris</name>
            <value>thrift://192.168.199.57:9083</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>hive.exec.scratchdir</name>
            <value>/user/hive/tmp</value>
        </property>
        <property>
            <name>hive.metastore.warehouse.dir</name>
            <value>/user/hive/warehouse</value>
        </property>
        <property>
            <name>hive.querylog.location</name>
            <value>/tmp/hive/querylog</value>
        </property>
        <property>
            <name>hive.server2.webui.host</name>
            <value>0.0.0.0</value>
        </property>
        <property>
            <name>hive.server2.webui.port</name>
            <value>10002</value>
        </property>
        <property>
            <name>hive.server2.thrift.port</name>
            <value>10000</value>
        </property>
    </configuration>

mysql部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-mysql
  labels:
    app: hive-mysql
spec:
  selector:
    matchLabels:
      app: hive-mysql
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-mysql
    spec:
      containers:
        - name: hive-mysql
          image: mysql:5.7.32
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 200m
              memory: 500Mi
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: xxxxxx
          args:
            - --character-set-server=utf8mb4
            - --collation-server=utf8mb4_unicode_ci
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: data
              mountPath: /var/lib/mysql
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: data
          hostPath:
            path: /var/lib/mysql
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-mysql: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

hive metastore部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-metastore
  labels:
    app: hive-metastore
spec:
  selector:
    matchLabels:
      app: hive-metastore
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-metastore
    spec:
      initContainers:
        - name: mysql-init
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          command:	# 通过schematool校验数据库，决定是否初始化mysql
            - "sh"
            - "-c"
            - "if schematool -dbType mysql -validate; then schematool -dbType mysql -initSchema; fi"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      containers:
        - name: hive-metastore
          image: harbor.gistack.cn/library/hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:																					# 根据规划修改
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service metastore -v"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-metastore: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

mysql和hive-metastore部署

shell 复制代码 
> kubectl.exe create -f .\hive-config.yaml -n bigdata
configmap/hive created
> kubectl.exe create -f .\hive-mysql.yaml -n bigdata 
deployment.apps/hive-mysql created
> kubectl.exe create -f .\hive-metastore.yaml -n bigdata
deployment.apps/hive-metastore created
>

hive-metastore初始化数据库和运行情况：

hvie-server2部署

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hiveserver2
  labels:
    app: hiveserver2
spec:
  selector:
    matchLabels:
      app: hiveserver2
  replicas: 1
  template:
    metadata:
      labels:
        app: hiveserver2
    spec:
      containers:
        - name: hiveserver2
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service hiveserver2"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hiveserver2: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"
shell 复制代码 
> kubectl.exe create -f .\hiveserver2.yaml -n bigdata
deployment.apps/hiveserver2 created
>

访问hive web服务：

通过SQL客户端连接：

建表测试：

插入数据，查看任务：

相关推荐
jwolf21 小时前
基于K8S的微服务：一、服务发现，负载均衡测试（附calico网络问题解决）
微服务·kubernetes·服务发现
nangonghen2 小时前
在华为云通过operator部署Doris v2.1集群
kubernetes·华为云·doris·operator
lovelin+v175030409662 小时前
智能电商：API接口如何驱动自动化与智能化转型
大数据·人工智能·爬虫·python
rpa_top2 小时前
RPA 助力电商：自动化商品信息上传，节省人力资源 —— 以影刀 RPA 为例【rpa.top】
大数据·前端·人工智能·自动化·rpa
神秘打工猴3 小时前
hive常用函数有哪些
hive
会飞的土拨鼠呀3 小时前
chart文件结构
运维·云原生·kubernetes
ssxueyi5 小时前
Paimon 是什么？Apache Paimon简介
大数据·流数据·stream·数据湖·流批一体·apache paimon
Asiram_6 小时前
大数据机器学习与计算机视觉应用08：反向传播
大数据·机器学习·计算机视觉
自在的LEE6 小时前
当 Go 遇上 Windows：15.625ms 的时间更新困局
后端·kubernetes·go
Json_181790144807 小时前
淘系商品评论json数据示例参考，API接口系列
大数据·json·api
热门推荐
01〔AI 绘画〕Stable Diffusion 之 解决绘制多人或面部很小的人物时面部崩坏问题 篇02YOLOv8训练好模型后，追加轮数继续训练、或者提前终止训练，缩减训练轮数03玄机平台应急响应—webshell查杀04SQL 主从数据库实时备份05HCIA-datacom数通题库和录播视频资料06Docker 夺命连环 15 问07VSCode插件 —— Cody AI （免费AI助手！）08校验 GPT-4 真实性的三个经典问题：快速区分 GPT-3.5 与 GPT-4，并提供免费测试网站09(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明10组基轨迹建模 GBTM的介绍与实现（Stata 或 R）