flink/spark/hive 使用paimon

Flink使用paimon

/usr/dif/7.1.0-0/flink/bin/sql-client.sh

CREATE CATALOG paimon_hive_catalog

WITH ( 'type'='paimon',

'metastore' = 'hive',

'hive-conf-dir'='/etc/hive/conf/',

'hadoop-conf-dir' = '/etc/hive/conf');

use catalog paimon_hive_catalog;

use ${DB};

XXXXXXXXXXXXXXXXXXXXXXXXXX

Spark on Hive（kyuubi）使用paimon

/usr/dif/7.1.0-0/kyuubi/bin/beeline -u 'jdbc:hive2://XXX:2181,XXX:2181,XXX:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=kyuubi;principal=ocdp/_HOST@XXX.COM;#spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog;spark.sql.catalog.spark_catalog=org.apache.paimon.spark.SparkGenericCatalog;spark.sql.catalog.paimon.warehouse=hdfs://XXX/apps/hive/warehouse;spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions;spark.yarn.queue=XXX;spark.executor.cores=4;spark.driver.cores=2;spark.executor.instances=10;spark.executor.memory=8g'

XXXXXXXXXXXXXXXXXXXXXXXXXX

Spark Jar使用paimon

/usr/dif/7.1.0-0/spark/bin/spark-submit

--master yarn

--deploy-mode client

--driver-memory 1G

--num-executors 2

--executor-cores 2

--executor-memory 4G

--name "SparkODSJob"

--class com.goertek.it.main.gmes.OdsGMesHiveMain

--conf spark.sql.catalog.spark_catalog=org.apache.paimon.spark.SparkGenericCatalog

--conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog

--conf spark.sql.catalog.paimon.warehouse=hdfs://goertekwf/apps/hive/warehouse

--conf spark.sql.catalog.paimon.metastore=hive

--conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions

--conf spark.yarn.principal=wf_bpit_mes@XXX.COM

--conf spark.yarn.keytab=/data01/dataos/apps/keytabs/wf_bpit_mes.keytab

--conf 'spark.driver.extraJavaOptions=-Djava.security.krb5.conf=/etc/krb5.conf'

--conf 'spark.driverEnv.KRB5_CONFIG=/etc/krb5.conf'

hdfs:///spark_jar/spark3-1.0-SNAPSHOT.jar

"test" "152" "165" "${batchNo}"