flink 同步oracle11g数据表到pg库

我是琦琦琦琦2024-11-15 21:19

1. 关闭防火墙和selinux

bash 复制代码 
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
 
vi /etc/selinux/config
修改为disabled

2.安装java8

bash 复制代码 
yum list java-1.8* 
 
yum install java-1.8.0-openjdk* -y
 
java -version

3.下载和部署postgresql

看需求安装pg库版本

bash 复制代码 
下载地址：http://www.postgresql.org/ftp/source/

上传到服务器解压

bash 复制代码 
tar -zxvf postgresql-16.2.tar.gz
bash 复制代码 
./configure

如果出现以下报错解决方法：

bash 复制代码 
yum -y install gcc

如果出现以下错误解决方法：

bash 复制代码 
yum -y install libicu libicu-devel libunwind readline-devel zlib-devel
bash 复制代码 
make
make install

groupadd postgres
useradd -g postgres postgres
id postgres
 
cd /usr/local/pgsql/
mkdir data
chown postgres:postgres data
 
cd /home/postgres/
ll -al
bash 复制代码 
vi .bash_profile
添加
export PGHOME=/usr/local/pgsql/
export PGDATA=/usr/local/pgsql/data
PATH=$PATH:$HOME/bin:$PGHOME/bin
bash 复制代码 
source .bash_profile
su -- postgres
initdb
bash 复制代码 
cd /usr/local/pgsql/data/
vi postgresql.conf
修改为listen_addresses = '*'

启动pg库

bash 复制代码 
service postgresql start
su -- postgres
psql

4.下载和部署oracle

Oracle下载地址：

bash 复制代码 
下载好上传两个文件到系统
mkdir /oracle
cd /oracle
bash 复制代码 
yum install unzip -y
unzip linux.x64_11gR2_database_1of2.zip
unzip linux.x64_11gR2_database_2of2.zip
yum -y install wget
yum install yum install gcc make binutils gcc-c++ compat-libstdc++-33 elfutils-libelf-devel elfutils-libelf-devel-static ksh libaio libaio-devel numactl-devel sysstat unixODBC unixODBC-devel pcre-devel -y 
添加安装用户和用户组
groupadd oinstall
groupadd dba
useradd -g oinstall -G dba oracle
passwd oracle
 
创建安装目录和设置文件权限
mkdir -p /u01/app/oracle/product/11.2.0
mkdir /u01/app/oracle/oradata
mkdir /u01/app/oracle/inventory
mkdir /u01/app/oracle/fast_recovery_area
chown -R oracle:oinstall /u01/
chmod -R 775 /u01/app/oracle
chown -R oracle:oinstall /oracle/

设置oracle用户环境变量

bash 复制代码 
su -- oracle	
vi .bash_profile

export PATH
ORACLE_BASE=/u01/app/oracle
ORACLE_HOME=$ORACLE_BASE/product/11.2.0
ORACLE_SID=orcl
PATH=$PATH:$ORACLE_HOME/bin
export ORACLE_BASE ORACLE_HOME ORACLE_SID PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
 
source .bash_profile

cp -R /oracle/database/response/ .
cd response/

vi db_install.rsp

bash 复制代码 
oracle.install.responseFileVersion=oracleinstallrspfmt_dbinstall_response_schema_v11_2_0
oracle.install.option=INSTALL_DB_SWONLY
ORACLE_HOSTNAME=flink
UNIX_GROUP_NAME=oinstall
INVENTORY_LOCATION=/u01/app/oracle/inventory
SELECTED_LANGUAGES=
ORACLE_HOME=u01/app/oracle/product/11.2.0
ORACLE_BASE=/u01/app/oracle
oracle.install.db.InstallEdition=
oracle.install.db.isCustomInstall=false
oracle.install.db.customComponents=oracle.server11.2.0.1.0,oracle.sysman.ccr10.2.7.0.0,oracle.xdk11.2.0.1.0,oracle.rdbms.oci11.2.0.1.0,oracle.network11.2.0.1.0,oracle.network.listener11.2.0.1.0,oracle.rdbms11.2.0.1.0,oracle.options11.2.0.1.0,oracle.rdbms.partitioning11.2.0.1.0,oracle.oraolap11.2.0.1.0,oracle.rdbms.dm11.2.0.1.0,oracle.rdbms.dv11.2.0.1.0,orcle.rdbms.lbac11.2.0.1.0,oracle.rdbms.rat11.2.0.1.0
oracle.install.db.DBA_GROUP=dba
oracle.install.db.OPER_GROUP=dba
oracle.install.db.CLUSTER_NODES=
oracle.install.db.config.starterdb.type=
oracle.install.db.config.starterdb.globalDBName=oracle.sunyard
oracle.install.db.config.starterdb.SID=orcl
oracle.install.db.config.starterdb.characterSet=AL32UTF8
oracle.install.db.config.starterdb.memoryOption=true
oracle.install.db.config.starterdb.memoryLimit=512
oracle.install.db.config.starterdb.installExampleSchemas=false
oracle.install.db.config.starterdb.enableSecuritySettings=true
oracle.install.db.config.starterdb.password.ALL=
oracle.install.db.config.starterdb.password.SYS=
oracle.install.db.config.starterdb.password.SYSTEM=
oracle.install.db.config.starterdb.password.SYSMAN=
oracle.install.db.config.starterdb.password.DBSNMP=
oracle.install.db.config.starterdb.control=DB_CONTROL
oracle.install.db.config.starterdb.gridcontrol.gridControlServiceURL=
oracle.install.db.config.starterdb.dbcontrol.enableEmailNotification=false
oracle.install.db.config.starterdb.dbcontrol.emailAddress=
oracle.install.db.config.starterdb.dbcontrol.SMTPServer=
oracle.install.db.config.starterdb.automatedBackup.enable=false
oracle.install.db.config.starterdb.automatedBackup.osuid=
oracle.install.db.config.starterdb.automatedBackup.ospwd=
oracle.install.db.config.starterdb.storageType=
oracle.install.db.config.starterdb.fileSystemStorage.dataLocation=
oracle.install.db.config.starterdb.fileSystemStorage.recoveryLocation=
oracle.install.db.config.asm.diskGroup=
oracle.install.db.config.asm.ASMSNMPPassword=LED_ASM_SYSTEM
MYORACLESUPPORT_USERNAME=
MYORACLESUPPORT_PASSWORD=
SECURITY_UPDATES_VIA_MYORACLESUPPORT=
DECLINE_SECURITY_UPDATES=true
PROXY_HOST=
PROXY_PORT=
PROXY_USER=
PROXY_PWD=
bash 复制代码 
cd /oracle/database/
./runInstaller -silent -responseFile /home/oracle/response/db_install.rsp -ignorePrereq

报错原因：swap分区关闭了 打开即可

bash 复制代码 
/u01/app/oracle/inventory/orainstRoot.sh
/u01/app/oracle/product/11.2.0/root.sh
yum -y install net-tools

vi /home/oracle/response/dbca.rsp

bash 复制代码 
修改参数
GDBNAME = "orcl"
SID = "orcl"
SYSPASSWORD = "oracle"
SYSTEMPASSWORD = "oracle"
SYSMANPASSWORD = "oracle"
DBSNMPPASSWORD = "oracle"
DATAFILEDESTINATION =/u01/app/oracle/oradata
RECOVERYAREADESTINATION=/u01/app/oracle/fast_recovery_area
CHARACTERSET = "ZHS16GBK"
TOTALMEMORY = "1638"
bash 复制代码 
dbca -silent -responseFile /home/oracle/response/dbca.rsp

实例检查

ps -ef | grep ora_ | grep -v grep

启动监听

lsnrctl start

查看监听状态

bash 复制代码 
sqlplus / as sysdba

startup报错

解决方法

bash 复制代码 
cp /u01/app/oracle/admin/orcl/pfile/init.ora.21520241602 /u01/app/oracle/product/11.2.0/dbs/
cd /u01/app/oracle/product/11.2.0/dbs/
mv init.ora.21520241602 initoral.ora

5.下载部署flink

bash 复制代码 
flink下载地址：https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz
下载flink sql所需驱动
1.	flink-connector-jdbc-3.1.1-1.17.jar 下载地址：https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/  
2.	flink-sql-connector-oracle-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-oracle-cdc/3.0.1
3.	flink-sql-connector-postgres-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-postgres-cdc/3.0.1

下载完之后上传flink压缩包并解压

bash 复制代码 
tar -zxvf flink-1.18.1-bin-scala_2.12.tgz
进入flink的lib目录上传三个依赖
cd flink-1.18.1/lib/

需要修改一下flink配置文件

不然会出现以下情况

bash 复制代码 
vi flink-1.18.1/conf/flink-conf.yaml

原本是localhost修改为ip

bash 复制代码 
./flink-1.18.1/bin/start-cluster.sh

访问 192.168.207.193:8081 （默认是8081端口 可在配置文件里修改）

6.实时同步oracle数据到postgresql

数据库先创建一个库,在库里创建表再添加数据

bash 复制代码 
sqlplus / as sysdba
create user ljq identified by linux123;
grant create session,resource to ljq;
sqlplus ljq/linux123@192.168.207.195/orcl

create table players (
     player_id INT NOT NULL,
     team_id INT,
     player_name VARCHAR(255),
     height FLOAT(53),
PRIMARY KEY (player_id)
);
bash 复制代码 
insert into players (player_id,team_id,player_name,height) values (1001,1001,'韦德','1.93');
insert into players (player_id,team_id,player_name,height) values (1002,1002,'雷吉','1.91');
insert into players (player_id,team_id,player_name,height) values (1003,1003,'安德烈','2.11');
insert into players (player_id,team_id,player_name,height) values (1004,1004,'索恩','2.16');
insert into players (player_id,team_id,player_name,height) values (1005,1005,'兰斯顿','1.88');
insert into players (player_id,team_id,player_name,height) values (1006,1006,'格伦','1.98');
insert into players (player_id,team_id,player_name,height) values (1007,1007,'伊斯梅尔','1.83');
insert into players (player_id,team_id,player_name,height) values (1008,1008,'扎扎','2.11');
insert into players (player_id,team_id,player_name,height) values (1009,1009,'乔恩','2.08');
bash 复制代码 
select * from players;

格式很乱

bash 复制代码 
解决方法：
set line 320
col player_name for a20

在pg库创建一个库,在库里创建一个表不插入数据

bash 复制代码 
create database ljq;
\c ljq

CREATE TABLE players3 (
player_id INT NOT NULL,
team_id INT,
player_name VARCHAR(255),
height FLOAT(53),
PRIMARY KEY (player_id)
);

启动flink-sql

bash 复制代码 
./flink-1.18.1/bin/sql-client.sh embedded
根据需要同步的数据创建源表
CREATE TABLE nbaplayers (
PLAYER_ID INT,
TEAM_ID INT,
PLAYER_NAME VARCHAR,
HEIGHT FLOAT,
PRIMARY KEY (PLAYER_ID) NOT ENFORCED
) WITH (
'connector' = 'oracle-cdc',
'hostname' = '192.168.207.192',
'port' = '1521',
'username' = 'ljq',
'password' = 'ljq',
'database-name' = 'test',
'schema-name' = 'LJQ',
'table-name' = 'PLAYERS'
);
bash 复制代码 
select * from nbaplayers;

创建结果表

bash 复制代码 
CREATE TABLE nba (
player_id INT,
team_id INT,
player_name VARCHAR,
height NUMERIC(3,2),
PRIMARY KEY (player_id) NOT ENFORCED
) WITH (
'connector' = 'jdbc',
'url' = 'jdbc:postgresql://192.168.207.193:5432/ljq??currentSchema=public&reWriteBatchedInserts=true',
'username' = 'postgres',
'password' = 'linux123',
'table-name' = 'players3'
);

执行从源表插入结果表操作，生成同步作业

bash 复制代码 
INSERT INTO nba
SELECT
player_id,
team_id,
player_name,
height
FROM nbaplayers;

Web端查看

查看是否同步数据到pg库的players3表

相关推荐
szxinmai主板定制专家7 分钟前
【国产NI替代】基于FPGA的32通道（24bits）高精度终端采集核心板卡
大数据·人工智能·fpga开发
TGB-Earnest2 小时前
【py脚本+logstash+es实现自动化检测工具】
大数据·elasticsearch·自动化
大圣数据星球4 小时前
Fluss 写入数据湖实战
大数据·设计模式·flink
suweijie7684 小时前
SpringCloudAlibaba | Sentinel从基础到进阶
java·大数据·sentinel
Data跳动9 小时前
Spark内存都消耗在哪里了？
大数据·分布式·spark
woshiabc11110 小时前
windows安装Elasticsearch及增删改查操作
大数据·elasticsearch·搜索引擎
百度智能云技术站10 小时前
广告投放系统成本降低 70%+，基于 Redis 容量型数据库 PegaDB 的方案设计和业务实践
数据库·redis·oracle
lucky_syq11 小时前
Saprk和Flink的区别
大数据·flink
lucky_syq11 小时前
流式处理，为什么Flink比Spark Streaming好？
大数据·flink·spark
袋鼠云数栈11 小时前
深入浅出Flink CEP丨如何通过Flink SQL作业动态更新Flink CEP作业
大数据
热门推荐
01〔AI 绘画〕Stable Diffusion 之 解决绘制多人或面部很小的人物时面部崩坏问题 篇02ARM学习（31）编译器对overlay方式的支持03玄机平台应急响应—webshell查杀04DisplayPort AltMode On Type-c协议_Ver2.0(学习笔记)05校验 GPT-4 真实性的三个经典问题：快速区分 GPT-3.5 与 GPT-4，并提供免费测试网站06Docker 夺命连环 15 问076个主流的工业3D管道设计软件08基于大语言模型（LLM）的合成数据生成、策展和评估的综述09(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明10树莓派 5 AI 套件（Hailo-8L）使用教程