flink 同步oracle11g数据表到pg库

我是琦琦琦琦2024-11-15 21:19

1. 关闭防火墙和selinux

bash 复制代码 
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
 
vi /etc/selinux/config
修改为disabled

2.安装java8

bash 复制代码 
yum list java-1.8* 
 
yum install java-1.8.0-openjdk* -y
 
java -version

3.下载和部署postgresql

看需求安装pg库版本

bash 复制代码 
下载地址：http://www.postgresql.org/ftp/source/

上传到服务器解压

bash 复制代码 
tar -zxvf postgresql-16.2.tar.gz
bash 复制代码 
./configure

如果出现以下报错解决方法：

bash 复制代码 
yum -y install gcc

如果出现以下错误解决方法：

bash 复制代码 
yum -y install libicu libicu-devel libunwind readline-devel zlib-devel
bash 复制代码 
make
make install

groupadd postgres
useradd -g postgres postgres
id postgres
 
cd /usr/local/pgsql/
mkdir data
chown postgres:postgres data
 
cd /home/postgres/
ll -al
bash 复制代码 
vi .bash_profile
添加
export PGHOME=/usr/local/pgsql/
export PGDATA=/usr/local/pgsql/data
PATH=$PATH:$HOME/bin:$PGHOME/bin
bash 复制代码 
source .bash_profile
su -- postgres
initdb
bash 复制代码 
cd /usr/local/pgsql/data/
vi postgresql.conf
修改为listen_addresses = '*'

启动pg库

bash 复制代码 
service postgresql start
su -- postgres
psql

4.下载和部署oracle

Oracle下载地址：

bash 复制代码 
下载好上传两个文件到系统
mkdir /oracle
cd /oracle
bash 复制代码 
yum install unzip -y
unzip linux.x64_11gR2_database_1of2.zip
unzip linux.x64_11gR2_database_2of2.zip
yum -y install wget
yum install yum install gcc make binutils gcc-c++ compat-libstdc++-33 elfutils-libelf-devel elfutils-libelf-devel-static ksh libaio libaio-devel numactl-devel sysstat unixODBC unixODBC-devel pcre-devel -y 
添加安装用户和用户组
groupadd oinstall
groupadd dba
useradd -g oinstall -G dba oracle
passwd oracle
 
创建安装目录和设置文件权限
mkdir -p /u01/app/oracle/product/11.2.0
mkdir /u01/app/oracle/oradata
mkdir /u01/app/oracle/inventory
mkdir /u01/app/oracle/fast_recovery_area
chown -R oracle:oinstall /u01/
chmod -R 775 /u01/app/oracle
chown -R oracle:oinstall /oracle/

设置oracle用户环境变量

bash 复制代码 
su -- oracle	
vi .bash_profile

export PATH
ORACLE_BASE=/u01/app/oracle
ORACLE_HOME=$ORACLE_BASE/product/11.2.0
ORACLE_SID=orcl
PATH=$PATH:$ORACLE_HOME/bin
export ORACLE_BASE ORACLE_HOME ORACLE_SID PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
 
source .bash_profile

cp -R /oracle/database/response/ .
cd response/

vi db_install.rsp

bash 复制代码 
oracle.install.responseFileVersion=oracleinstallrspfmt_dbinstall_response_schema_v11_2_0
oracle.install.option=INSTALL_DB_SWONLY
ORACLE_HOSTNAME=flink
UNIX_GROUP_NAME=oinstall
INVENTORY_LOCATION=/u01/app/oracle/inventory
SELECTED_LANGUAGES=
ORACLE_HOME=u01/app/oracle/product/11.2.0
ORACLE_BASE=/u01/app/oracle
oracle.install.db.InstallEdition=
oracle.install.db.isCustomInstall=false
oracle.install.db.customComponents=oracle.server11.2.0.1.0,oracle.sysman.ccr10.2.7.0.0,oracle.xdk11.2.0.1.0,oracle.rdbms.oci11.2.0.1.0,oracle.network11.2.0.1.0,oracle.network.listener11.2.0.1.0,oracle.rdbms11.2.0.1.0,oracle.options11.2.0.1.0,oracle.rdbms.partitioning11.2.0.1.0,oracle.oraolap11.2.0.1.0,oracle.rdbms.dm11.2.0.1.0,oracle.rdbms.dv11.2.0.1.0,orcle.rdbms.lbac11.2.0.1.0,oracle.rdbms.rat11.2.0.1.0
oracle.install.db.DBA_GROUP=dba
oracle.install.db.OPER_GROUP=dba
oracle.install.db.CLUSTER_NODES=
oracle.install.db.config.starterdb.type=
oracle.install.db.config.starterdb.globalDBName=oracle.sunyard
oracle.install.db.config.starterdb.SID=orcl
oracle.install.db.config.starterdb.characterSet=AL32UTF8
oracle.install.db.config.starterdb.memoryOption=true
oracle.install.db.config.starterdb.memoryLimit=512
oracle.install.db.config.starterdb.installExampleSchemas=false
oracle.install.db.config.starterdb.enableSecuritySettings=true
oracle.install.db.config.starterdb.password.ALL=
oracle.install.db.config.starterdb.password.SYS=
oracle.install.db.config.starterdb.password.SYSTEM=
oracle.install.db.config.starterdb.password.SYSMAN=
oracle.install.db.config.starterdb.password.DBSNMP=
oracle.install.db.config.starterdb.control=DB_CONTROL
oracle.install.db.config.starterdb.gridcontrol.gridControlServiceURL=
oracle.install.db.config.starterdb.dbcontrol.enableEmailNotification=false
oracle.install.db.config.starterdb.dbcontrol.emailAddress=
oracle.install.db.config.starterdb.dbcontrol.SMTPServer=
oracle.install.db.config.starterdb.automatedBackup.enable=false
oracle.install.db.config.starterdb.automatedBackup.osuid=
oracle.install.db.config.starterdb.automatedBackup.ospwd=
oracle.install.db.config.starterdb.storageType=
oracle.install.db.config.starterdb.fileSystemStorage.dataLocation=
oracle.install.db.config.starterdb.fileSystemStorage.recoveryLocation=
oracle.install.db.config.asm.diskGroup=
oracle.install.db.config.asm.ASMSNMPPassword=LED_ASM_SYSTEM
MYORACLESUPPORT_USERNAME=
MYORACLESUPPORT_PASSWORD=
SECURITY_UPDATES_VIA_MYORACLESUPPORT=
DECLINE_SECURITY_UPDATES=true
PROXY_HOST=
PROXY_PORT=
PROXY_USER=
PROXY_PWD=
bash 复制代码 
cd /oracle/database/
./runInstaller -silent -responseFile /home/oracle/response/db_install.rsp -ignorePrereq

报错原因：swap分区关闭了 打开即可

bash 复制代码 
/u01/app/oracle/inventory/orainstRoot.sh
/u01/app/oracle/product/11.2.0/root.sh
yum -y install net-tools

vi /home/oracle/response/dbca.rsp

bash 复制代码 
修改参数
GDBNAME = "orcl"
SID = "orcl"
SYSPASSWORD = "oracle"
SYSTEMPASSWORD = "oracle"
SYSMANPASSWORD = "oracle"
DBSNMPPASSWORD = "oracle"
DATAFILEDESTINATION =/u01/app/oracle/oradata
RECOVERYAREADESTINATION=/u01/app/oracle/fast_recovery_area
CHARACTERSET = "ZHS16GBK"
TOTALMEMORY = "1638"
bash 复制代码 
dbca -silent -responseFile /home/oracle/response/dbca.rsp

实例检查

ps -ef | grep ora_ | grep -v grep

启动监听

lsnrctl start

查看监听状态

bash 复制代码 
sqlplus / as sysdba

startup报错

解决方法

bash 复制代码 
cp /u01/app/oracle/admin/orcl/pfile/init.ora.21520241602 /u01/app/oracle/product/11.2.0/dbs/
cd /u01/app/oracle/product/11.2.0/dbs/
mv init.ora.21520241602 initoral.ora

5.下载部署flink

bash 复制代码 
flink下载地址：https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz
下载flink sql所需驱动
1.	flink-connector-jdbc-3.1.1-1.17.jar 下载地址：https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/  
2.	flink-sql-connector-oracle-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-oracle-cdc/3.0.1
3.	flink-sql-connector-postgres-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-postgres-cdc/3.0.1

下载完之后上传flink压缩包并解压

bash 复制代码 
tar -zxvf flink-1.18.1-bin-scala_2.12.tgz
进入flink的lib目录上传三个依赖
cd flink-1.18.1/lib/

需要修改一下flink配置文件

不然会出现以下情况

bash 复制代码 
vi flink-1.18.1/conf/flink-conf.yaml

原本是localhost修改为ip

bash 复制代码 
./flink-1.18.1/bin/start-cluster.sh

访问 192.168.207.193:8081 （默认是8081端口 可在配置文件里修改）

6.实时同步oracle数据到postgresql

数据库先创建一个库,在库里创建表再添加数据

bash 复制代码 
sqlplus / as sysdba
create user ljq identified by linux123;
grant create session,resource to ljq;
sqlplus ljq/linux123@192.168.207.195/orcl

create table players (
     player_id INT NOT NULL,
     team_id INT,
     player_name VARCHAR(255),
     height FLOAT(53),
PRIMARY KEY (player_id)
);
bash 复制代码 
insert into players (player_id,team_id,player_name,height) values (1001,1001,'韦德','1.93');
insert into players (player_id,team_id,player_name,height) values (1002,1002,'雷吉','1.91');
insert into players (player_id,team_id,player_name,height) values (1003,1003,'安德烈','2.11');
insert into players (player_id,team_id,player_name,height) values (1004,1004,'索恩','2.16');
insert into players (player_id,team_id,player_name,height) values (1005,1005,'兰斯顿','1.88');
insert into players (player_id,team_id,player_name,height) values (1006,1006,'格伦','1.98');
insert into players (player_id,team_id,player_name,height) values (1007,1007,'伊斯梅尔','1.83');
insert into players (player_id,team_id,player_name,height) values (1008,1008,'扎扎','2.11');
insert into players (player_id,team_id,player_name,height) values (1009,1009,'乔恩','2.08');
bash 复制代码 
select * from players;

格式很乱

bash 复制代码 
解决方法：
set line 320
col player_name for a20

在pg库创建一个库,在库里创建一个表不插入数据

bash 复制代码 
create database ljq;
\c ljq

CREATE TABLE players3 (
player_id INT NOT NULL,
team_id INT,
player_name VARCHAR(255),
height FLOAT(53),
PRIMARY KEY (player_id)
);

启动flink-sql

bash 复制代码 
./flink-1.18.1/bin/sql-client.sh embedded
根据需要同步的数据创建源表
CREATE TABLE nbaplayers (
PLAYER_ID INT,
TEAM_ID INT,
PLAYER_NAME VARCHAR,
HEIGHT FLOAT,
PRIMARY KEY (PLAYER_ID) NOT ENFORCED
) WITH (
'connector' = 'oracle-cdc',
'hostname' = '192.168.207.192',
'port' = '1521',
'username' = 'ljq',
'password' = 'ljq',
'database-name' = 'test',
'schema-name' = 'LJQ',
'table-name' = 'PLAYERS'
);
bash 复制代码 
select * from nbaplayers;

创建结果表

bash 复制代码 
CREATE TABLE nba (
player_id INT,
team_id INT,
player_name VARCHAR,
height NUMERIC(3,2),
PRIMARY KEY (player_id) NOT ENFORCED
) WITH (
'connector' = 'jdbc',
'url' = 'jdbc:postgresql://192.168.207.193:5432/ljq??currentSchema=public&reWriteBatchedInserts=true',
'username' = 'postgres',
'password' = 'linux123',
'table-name' = 'players3'
);

执行从源表插入结果表操作，生成同步作业

bash 复制代码 
INSERT INTO nba
SELECT
player_id,
team_id,
player_name,
height
FROM nbaplayers;

Web端查看

查看是否同步数据到pg库的players3表

相关推荐
火火PM打怪中2 小时前
产品经理如何绘制服务蓝图（Service Blueprint）
大数据·产品经理
Elastic 中国社区官方博客10 小时前
在 Windows 上使用 Docker 运行 Elastic Open Crawler
大数据·windows·爬虫·elasticsearch·搜索引擎·docker·容器
一切顺势而行12 小时前
Flink cdc 使用总结
大数据·flink
云资源服务商12 小时前
探索阿里云DMS：解锁高效数据管理新姿势
数据库·阿里云·oracle·云计算
淦暴尼14 小时前
基于spark的二手房数据分析可视化系统
大数据·分布式·数据分析·spark
expect7g15 小时前
Flink-反压-1.基本概念
后端·flink
Ashlee_code15 小时前
裂变时刻：全球关税重构下的券商交易系统跃迁路线图（2025-2027）
java·大数据·数据结构·python·云原生·区块链·perl
Flink_China15 小时前
淘天AB实验分析平台Fluss落地实践：更适合实时OLAP的消息队列
大数据·flink
马立杰15 小时前
Oracle 19.28 RU 升级最佳实践指南
数据库·oracle
阿里云大数据AI技术16 小时前
云上AI推理平台全掌握 (4)：大模型分发加速
大数据·人工智能·llm
热门推荐
01全球最强模型Grok4，国内已可免费使用！（附教程）02Cursor Claude 模型无法使用的解决方法03KGG转MP3工具|非KGM文件|解密音频04【2025.7.18】更新vscode后所有.vue文件template标签后报红的临时解决办法，Vue - Official 插件3.0.2导致05【无标题】06集群聊天服务器---MySQL数据库的建立07突破限制：使用 Claude Code Proxy 让 Claude Code 自由连接任意模型08绿色建筑新态势：楼宇自控助力能效提升，推动成本优化新路径09使用Ruby接入实时行情API教程10Claude Code 最新版已经支持 Windows 安装使用！