Kubernetes运行大数据组件-运行hive

小森饭2024-10-30 13:46

部署组件

● mysql

● hive-metastore

● hive-server2

配置文件

xml 复制代码 
apiVersion: v1
kind: ConfigMap
metadata:
  name: hive
data:
  hive-site.xml: |-
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>javax.jdo.option.ConnectionUserName</name>
            <value>root</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionPassword</name>
            <value>xxxxx</value>
        </property>
        <property>
            <name>javax.jdo.option.ConnectionURL</name>
            <value>jdbc:mysql://192.168.199.58:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>javax.jdo.option.ConnectionDriverName</name>
            <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
            <name>hive.cli.print.current.db</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.cli.print.header</name>
            <value>true</value>
        </property>
        <property>
            <name>hive.exec.post.hooks</name>
            <value>org.apache.atlas.hive.hook.HiveHook</value>
        </property>
        <property>
            <name>hive.metastore.uris</name>
            <value>thrift://192.168.199.57:9083</value>
            <!-- 根据角色修改IP地址 -->
        </property>
        <property>
            <name>hive.exec.scratchdir</name>
            <value>/user/hive/tmp</value>
        </property>
        <property>
            <name>hive.metastore.warehouse.dir</name>
            <value>/user/hive/warehouse</value>
        </property>
        <property>
            <name>hive.querylog.location</name>
            <value>/tmp/hive/querylog</value>
        </property>
        <property>
            <name>hive.server2.webui.host</name>
            <value>0.0.0.0</value>
        </property>
        <property>
            <name>hive.server2.webui.port</name>
            <value>10002</value>
        </property>
        <property>
            <name>hive.server2.thrift.port</name>
            <value>10000</value>
        </property>
    </configuration>

mysql部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-mysql
  labels:
    app: hive-mysql
spec:
  selector:
    matchLabels:
      app: hive-mysql
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-mysql
    spec:
      containers:
        - name: hive-mysql
          image: mysql:5.7.32
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 200m
              memory: 500Mi
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: xxxxxx
          args:
            - --character-set-server=utf8mb4
            - --collation-server=utf8mb4_unicode_ci
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: data
              mountPath: /var/lib/mysql
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: data
          hostPath:
            path: /var/lib/mysql
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-mysql: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

hive metastore部署文件

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hive-metastore
  labels:
    app: hive-metastore
spec:
  selector:
    matchLabels:
      app: hive-metastore
  replicas: 1
  template:
    metadata:
      labels:
        app: hive-metastore
    spec:
      initContainers:
        - name: mysql-init
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          command:	# 通过schematool校验数据库，决定是否初始化mysql
            - "sh"
            - "-c"
            - "if schematool -dbType mysql -validate; then schematool -dbType mysql -initSchema; fi"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      containers:
        - name: hive-metastore
          image: harbor.gistack.cn/library/hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:																					# 根据规划修改
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service metastore -v"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hive-metastore: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"

mysql和hive-metastore部署

shell 复制代码 
> kubectl.exe create -f .\hive-config.yaml -n bigdata
configmap/hive created
> kubectl.exe create -f .\hive-mysql.yaml -n bigdata 
deployment.apps/hive-mysql created
> kubectl.exe create -f .\hive-metastore.yaml -n bigdata
deployment.apps/hive-metastore created
>

hive-metastore初始化数据库和运行情况：

hvie-server2部署

yaml 复制代码 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hiveserver2
  labels:
    app: hiveserver2
spec:
  selector:
    matchLabels:
      app: hiveserver2
  replicas: 1
  template:
    metadata:
      labels:
        app: hiveserver2
    spec:
      containers:
        - name: hiveserver2
          image: hive:2.3.8
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 2000m
              memory: 4Gi
          command:
            - "sh"
            - "-c"
            - "hive --service hiveserver2"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/core-site.xml
              subPath: core-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/hdfs-site.xml
              subPath: hdfs-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/yarn-site.xml
              subPath: yarn-site.xml
            - name: hadoop-config
              mountPath: /opt/hadoop/etc/hadoop/mapred-site.xml
              subPath: mapred-site.xml
            - name: hive-config
              mountPath: /opt/hive/conf/hive-site.xml
              subPath: hive-site.xml
      volumes:
        - name: localtime
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - name: hadoop-config
          configMap:
            name: hadoop
        - name: hive-config
          configMap:
            name: hive
      restartPolicy: Always
      hostNetwork: true
      hostAliases:
        - ip: "192.168.199.56"
          hostnames:
            - "bigdata199056"
        - ip: "192.168.199.57"
          hostnames:
            - "bigdata199057"
        - ip: "192.168.199.58"
          hostnames:
            - "bigdata199058"
      nodeSelector:
        hiveserver2: "true"
      tolerations:
        - key: "bigdata"
          value: "true"
          operator: "Equal"
          effect: "NoSchedule"
shell 复制代码 
> kubectl.exe create -f .\hiveserver2.yaml -n bigdata
deployment.apps/hiveserver2 created
>

访问hive web服务：

通过SQL客户端连接：

建表测试：

插入数据，查看任务：

相关推荐
知初～4 小时前
出行项目案例
hive·hadoop·redis·sql·mysql·spark·database
狮歌~资深攻城狮8 小时前
HBase性能优化秘籍：让数据处理飞起来
大数据·hbase
Elastic 中国社区官方博客8 小时前
Elasticsearch Open Inference API 增加了对 Jina AI 嵌入和 Rerank 模型的支持
大数据·人工智能·elasticsearch·搜索引擎·ai·全文检索·jina
workflower9 小时前
Prompt Engineering的重要性
大数据·人工智能·设计模式·prompt·软件工程·需求分析·ai编程
东风微鸣10 小时前
TTRSS 迁移实战
docker·云原生·kubernetes·可观察性
API_technology11 小时前
电商搜索API的Elasticsearch优化策略
大数据·elasticsearch·搜索引擎
黄雪超11 小时前
大数据SQL调优专题——引擎优化
大数据·数据库·sql
The god of big data11 小时前
MapReduce 第二部：深入分析与实践
大数据·mapreduce
Smile_Gently12 小时前
Docker
云原生·eureka
G***技12 小时前
杰和科技GAM-AI视觉识别管理系统，让AI走进零售营销
大数据·人工智能·系统架构
热门推荐
01太炸裂了！清华大学deepseek从入门到精通使用手册又出第三版了，《普通人如何抓住DeepSeek红利》（无套路，直接下载）02DeepSeek各版本说明与优缺点分析03如何在WPS和Word/Excel中直接使用DeepSeek功能04本地部署DeepSeek教程（Mac版本）05本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程06DeepSeek本地部署详细指南07DeepSeek R1本地化部署 Ollama + Chatbox 打造最强 AI 工具08Page Assist - 本地Deepseek模型 Web UI 的安装和使用09DeepSeek r1本地安装全指南10本地部署DeepSeek后的调用与删除全攻略