Kubernetes运行大数据组件-运行hive

小森饭2024-10-30 13:46

部署组件

● mysql

● hive-metastore

● hive-server2

配置文件

xml 复制代码 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hive
data:
  hive-site.xml: |-
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>javax.jdo.option.ConnectionUserName</name>
            <value>root</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionPassword</name>
            <value>xxxxx</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionURL</name>
            <value>jdbc:mysql://192.168.199.58:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>javax.jdo.option.ConnectionDriverName</name>
            <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
            <name>hive.cli.print.current.db</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.cli.print.header</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.exec.post.hooks</name>
            <value>org.apache.atlas.hive.hook.HiveHook</value>
        </property>
        <property>
            <name>hive.metastore.uris</name>
            <value>thrift://192.168.199.57:9083</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>hive.exec.scratchdir</name>
            <value>/user/hive/tmp</value>
        </property>
        <property>
            <name>hive.metastore.warehouse.dir</name>
            <value>/user/hive/warehouse</value>
        </property>
        <property>
            <name>hive.querylog.location</name>
            <value>/tmp/hive/querylog</value>
        </property>
        <property>
            <name>hive.server2.webui.host</name>
            <value>0.0.0.0</value>
        </property>
        <property>
            <name>hive.server2.webui.port</name>
            <value>10002</value>
        </property>
        <property>
            <name>hive.server2.thrift.port</name>
            <value>10000</value>
        </property>
    </configuration>

mysql部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-mysql
  labels:
    app: hive-mysql
spec:
  selector:
    matchLabels:
      app: hive-mysql
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-mysql
    spec:
      containers:
        - name: hive-mysql
          image: mysql:5.7.32
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 200m
              memory: 500Mi
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: xxxxxx
          args:
            - --character-set-server=utf8mb4
            - --collation-server=utf8mb4_unicode_ci
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: data
              mountPath: /var/lib/mysql
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: data
          hostPath:
            path: /var/lib/mysql
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-mysql: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

hive metastore部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-metastore
  labels:
    app: hive-metastore
spec:
  selector:
    matchLabels:
      app: hive-metastore
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-metastore
    spec:
      initContainers:
        - name: mysql-init
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          command:	# 通过schematool校验数据库，决定是否初始化mysql
            - "sh"
            - "-c"
            - "if schematool -dbType mysql -validate; then schematool -dbType mysql -initSchema; fi"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      containers:
        - name: hive-metastore
          image: harbor.gistack.cn/library/hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:																					# 根据规划修改
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service metastore -v"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-metastore: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

mysql和hive-metastore部署

shell 复制代码 
> kubectl.exe create -f .\hive-config.yaml -n bigdata
configmap/hive created
> kubectl.exe create -f .\hive-mysql.yaml -n bigdata 
deployment.apps/hive-mysql created
> kubectl.exe create -f .\hive-metastore.yaml -n bigdata
deployment.apps/hive-metastore created
>

hive-metastore初始化数据库和运行情况：

hvie-server2部署

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hiveserver2
  labels:
    app: hiveserver2
spec:
  selector:
    matchLabels:
      app: hiveserver2
  replicas: 1
  template:
    metadata:
      labels:
        app: hiveserver2
    spec:
      containers:
        - name: hiveserver2
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service hiveserver2"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hiveserver2: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"
shell 复制代码 
> kubectl.exe create -f .\hiveserver2.yaml -n bigdata
deployment.apps/hiveserver2 created
>

访问hive web服务：

通过SQL客户端连接：

建表测试：

插入数据，查看任务：

相关推荐
二进制_博客1 小时前
Flink学习连载第二篇-使用flink编写WordCount(多种情况演示)
大数据
hong1616881 小时前
大数据技术Kafka详解：消息队列（Messages Queue）
大数据·分布式·kafka
我要用代码向我喜欢的女孩表白2 小时前
k8s入门（不教部署，部署跟着文档来就行了）
云原生·容器·kubernetes
隔着天花板看星星11 小时前
Kafka-创建topic源码
大数据·分布式·中间件·kafka
晴子呀11 小时前
微服务系列概览
微服务·云原生·架构
goTsHgo11 小时前
在Spark Streaming中简单实现实时用户画像系统
大数据·分布式·spark
努力的小陈^O^11 小时前
docker学习笔记跟常用命令总结
java·笔记·docker·云原生
老周聊架构11 小时前
聊聊Flink：Flink中的时间语义和Watermark详解
大数据·flink
别这么骄傲11 小时前
Flink Lookup Join（维表 Join）
大数据·flink·linq
长囧鹿11 小时前
k8s资源对象管理
云原生·容器·kubernetes
热门推荐
01(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明02PyTorch机器学习实现液态神经网络03【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总04玄机平台应急响应—webshell查杀05Coze扣子平台完整体验和实践（附国内和国际版对比）06RAG 实践- Ollama+RagFlow 部署本地知识库07【目标跟踪】相机运动补偿08Unity中PICO实现 隔空取物 和 接触抓取物体09Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO1004 - matlab m_map地学绘图工具基础函数 - 设置网格