flink 同步oracle11g数据表到pg库

我是琦琦琦琦2024-11-15 21:19

1. 关闭防火墙和selinux

bash 复制代码 
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
 
vi /etc/selinux/config
修改为disabled

2.安装java8

bash 复制代码 
yum list java-1.8* 
 
yum install java-1.8.0-openjdk* -y
 
java -version

3.下载和部署postgresql

看需求安装pg库版本

bash 复制代码 
下载地址：http://www.postgresql.org/ftp/source/

上传到服务器解压

bash 复制代码 
tar -zxvf postgresql-16.2.tar.gz
bash 复制代码 
./configure

如果出现以下报错解决方法：

bash 复制代码 
yum -y install gcc

如果出现以下错误解决方法：

bash 复制代码 
yum -y install libicu libicu-devel libunwind readline-devel zlib-devel
bash 复制代码 
make
make install

groupadd postgres
useradd -g postgres postgres
id postgres
 
cd /usr/local/pgsql/
mkdir data
chown postgres:postgres data
 
cd /home/postgres/
ll -al
bash 复制代码 
vi .bash_profile
添加
export PGHOME=/usr/local/pgsql/
export PGDATA=/usr/local/pgsql/data
PATH=$PATH:$HOME/bin:$PGHOME/bin
bash 复制代码 
source .bash_profile
su -- postgres
initdb
bash 复制代码 
cd /usr/local/pgsql/data/
vi postgresql.conf
修改为listen_addresses = '*'

启动pg库

bash 复制代码 
service postgresql start
su -- postgres
psql

4.下载和部署oracle

Oracle下载地址：

bash 复制代码 
下载好上传两个文件到系统
mkdir /oracle
cd /oracle
bash 复制代码 
yum install unzip -y
unzip linux.x64_11gR2_database_1of2.zip
unzip linux.x64_11gR2_database_2of2.zip
yum -y install wget
yum install yum install gcc make binutils gcc-c++ compat-libstdc++-33 elfutils-libelf-devel elfutils-libelf-devel-static ksh libaio libaio-devel numactl-devel sysstat unixODBC unixODBC-devel pcre-devel -y 
添加安装用户和用户组
groupadd oinstall
groupadd dba
useradd -g oinstall -G dba oracle
passwd oracle
 
创建安装目录和设置文件权限
mkdir -p /u01/app/oracle/product/11.2.0
mkdir /u01/app/oracle/oradata
mkdir /u01/app/oracle/inventory
mkdir /u01/app/oracle/fast_recovery_area
chown -R oracle:oinstall /u01/
chmod -R 775 /u01/app/oracle
chown -R oracle:oinstall /oracle/

设置oracle用户环境变量

bash 复制代码 
su -- oracle	
vi .bash_profile

export PATH
ORACLE_BASE=/u01/app/oracle
ORACLE_HOME=$ORACLE_BASE/product/11.2.0
ORACLE_SID=orcl
PATH=$PATH:$ORACLE_HOME/bin
export ORACLE_BASE ORACLE_HOME ORACLE_SID PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
 
source .bash_profile

cp -R /oracle/database/response/ .
cd response/

vi db_install.rsp

bash 复制代码 
oracle.install.responseFileVersion=oracleinstallrspfmt_dbinstall_response_schema_v11_2_0
oracle.install.option=INSTALL_DB_SWONLY
ORACLE_HOSTNAME=flink
UNIX_GROUP_NAME=oinstall
INVENTORY_LOCATION=/u01/app/oracle/inventory
SELECTED_LANGUAGES=
ORACLE_HOME=u01/app/oracle/product/11.2.0
ORACLE_BASE=/u01/app/oracle
oracle.install.db.InstallEdition=
oracle.install.db.isCustomInstall=false
oracle.install.db.customComponents=oracle.server11.2.0.1.0,oracle.sysman.ccr10.2.7.0.0,oracle.xdk11.2.0.1.0,oracle.rdbms.oci11.2.0.1.0,oracle.network11.2.0.1.0,oracle.network.listener11.2.0.1.0,oracle.rdbms11.2.0.1.0,oracle.options11.2.0.1.0,oracle.rdbms.partitioning11.2.0.1.0,oracle.oraolap11.2.0.1.0,oracle.rdbms.dm11.2.0.1.0,oracle.rdbms.dv11.2.0.1.0,orcle.rdbms.lbac11.2.0.1.0,oracle.rdbms.rat11.2.0.1.0
oracle.install.db.DBA_GROUP=dba
oracle.install.db.OPER_GROUP=dba
oracle.install.db.CLUSTER_NODES=
oracle.install.db.config.starterdb.type=
oracle.install.db.config.starterdb.globalDBName=oracle.sunyard
oracle.install.db.config.starterdb.SID=orcl
oracle.install.db.config.starterdb.characterSet=AL32UTF8
oracle.install.db.config.starterdb.memoryOption=true
oracle.install.db.config.starterdb.memoryLimit=512
oracle.install.db.config.starterdb.installExampleSchemas=false
oracle.install.db.config.starterdb.enableSecuritySettings=true
oracle.install.db.config.starterdb.password.ALL=
oracle.install.db.config.starterdb.password.SYS=
oracle.install.db.config.starterdb.password.SYSTEM=
oracle.install.db.config.starterdb.password.SYSMAN=
oracle.install.db.config.starterdb.password.DBSNMP=
oracle.install.db.config.starterdb.control=DB_CONTROL
oracle.install.db.config.starterdb.gridcontrol.gridControlServiceURL=
oracle.install.db.config.starterdb.dbcontrol.enableEmailNotification=false
oracle.install.db.config.starterdb.dbcontrol.emailAddress=
oracle.install.db.config.starterdb.dbcontrol.SMTPServer=
oracle.install.db.config.starterdb.automatedBackup.enable=false
oracle.install.db.config.starterdb.automatedBackup.osuid=
oracle.install.db.config.starterdb.automatedBackup.ospwd=
oracle.install.db.config.starterdb.storageType=
oracle.install.db.config.starterdb.fileSystemStorage.dataLocation=
oracle.install.db.config.starterdb.fileSystemStorage.recoveryLocation=
oracle.install.db.config.asm.diskGroup=
oracle.install.db.config.asm.ASMSNMPPassword=LED_ASM_SYSTEM
MYORACLESUPPORT_USERNAME=
MYORACLESUPPORT_PASSWORD=
SECURITY_UPDATES_VIA_MYORACLESUPPORT=
DECLINE_SECURITY_UPDATES=true
PROXY_HOST=
PROXY_PORT=
PROXY_USER=
PROXY_PWD=
bash 复制代码 
cd /oracle/database/
./runInstaller -silent -responseFile /home/oracle/response/db_install.rsp -ignorePrereq

报错原因：swap分区关闭了 打开即可

bash 复制代码 
/u01/app/oracle/inventory/orainstRoot.sh
/u01/app/oracle/product/11.2.0/root.sh
yum -y install net-tools

vi /home/oracle/response/dbca.rsp

bash 复制代码 
修改参数
GDBNAME = "orcl"
SID = "orcl"
SYSPASSWORD = "oracle"
SYSTEMPASSWORD = "oracle"
SYSMANPASSWORD = "oracle"
DBSNMPPASSWORD = "oracle"
DATAFILEDESTINATION =/u01/app/oracle/oradata
RECOVERYAREADESTINATION=/u01/app/oracle/fast_recovery_area
CHARACTERSET = "ZHS16GBK"
TOTALMEMORY = "1638"
bash 复制代码 
dbca -silent -responseFile /home/oracle/response/dbca.rsp

实例检查

ps -ef | grep ora_ | grep -v grep

启动监听

lsnrctl start

查看监听状态

bash 复制代码 
sqlplus / as sysdba

startup报错

解决方法

bash 复制代码 
cp /u01/app/oracle/admin/orcl/pfile/init.ora.21520241602 /u01/app/oracle/product/11.2.0/dbs/
cd /u01/app/oracle/product/11.2.0/dbs/
mv init.ora.21520241602 initoral.ora

5.下载部署flink

bash 复制代码 
flink下载地址：https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz
下载flink sql所需驱动
1.	flink-connector-jdbc-3.1.1-1.17.jar 下载地址：https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/  
2.	flink-sql-connector-oracle-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-oracle-cdc/3.0.1
3.	flink-sql-connector-postgres-cdc-3.0.1.jar 下载地址：https://mvnrepository.com/artifact/com.ververica/flink-sql-connector-postgres-cdc/3.0.1

下载完之后上传flink压缩包并解压

bash 复制代码 
tar -zxvf flink-1.18.1-bin-scala_2.12.tgz
进入flink的lib目录上传三个依赖
cd flink-1.18.1/lib/

需要修改一下flink配置文件

不然会出现以下情况

bash 复制代码 
vi flink-1.18.1/conf/flink-conf.yaml

原本是localhost修改为ip

bash 复制代码 
./flink-1.18.1/bin/start-cluster.sh

访问 192.168.207.193:8081 （默认是8081端口 可在配置文件里修改）

6.实时同步oracle数据到postgresql

数据库先创建一个库,在库里创建表再添加数据

bash 复制代码 
sqlplus / as sysdba
create user ljq identified by linux123;
grant create session,resource to ljq;
sqlplus ljq/linux123@192.168.207.195/orcl

create table players (
     player_id INT NOT NULL,
     team_id INT,
     player_name VARCHAR(255),
     height FLOAT(53),
PRIMARY KEY (player_id)
);
bash 复制代码 
insert into players (player_id,team_id,player_name,height) values (1001,1001,'韦德','1.93');
insert into players (player_id,team_id,player_name,height) values (1002,1002,'雷吉','1.91');
insert into players (player_id,team_id,player_name,height) values (1003,1003,'安德烈','2.11');
insert into players (player_id,team_id,player_name,height) values (1004,1004,'索恩','2.16');
insert into players (player_id,team_id,player_name,height) values (1005,1005,'兰斯顿','1.88');
insert into players (player_id,team_id,player_name,height) values (1006,1006,'格伦','1.98');
insert into players (player_id,team_id,player_name,height) values (1007,1007,'伊斯梅尔','1.83');
insert into players (player_id,team_id,player_name,height) values (1008,1008,'扎扎','2.11');
insert into players (player_id,team_id,player_name,height) values (1009,1009,'乔恩','2.08');
bash 复制代码 
select * from players;

格式很乱

bash 复制代码 
解决方法：
set line 320
col player_name for a20

在pg库创建一个库,在库里创建一个表不插入数据

bash 复制代码 
create database ljq;
\c ljq

CREATE TABLE players3 (
player_id INT NOT NULL,
team_id INT,
player_name VARCHAR(255),
height FLOAT(53),
PRIMARY KEY (player_id)
);

启动flink-sql

bash 复制代码 
./flink-1.18.1/bin/sql-client.sh embedded
根据需要同步的数据创建源表
CREATE TABLE nbaplayers (
PLAYER_ID INT,
TEAM_ID INT,
PLAYER_NAME VARCHAR,
HEIGHT FLOAT,
PRIMARY KEY (PLAYER_ID) NOT ENFORCED
) WITH (
'connector' = 'oracle-cdc',
'hostname' = '192.168.207.192',
'port' = '1521',
'username' = 'ljq',
'password' = 'ljq',
'database-name' = 'test',
'schema-name' = 'LJQ',
'table-name' = 'PLAYERS'
);
bash 复制代码 
select * from nbaplayers;

创建结果表

bash 复制代码 
CREATE TABLE nba (
player_id INT,
team_id INT,
player_name VARCHAR,
height NUMERIC(3,2),
PRIMARY KEY (player_id) NOT ENFORCED
) WITH (
'connector' = 'jdbc',
'url' = 'jdbc:postgresql://192.168.207.193:5432/ljq??currentSchema=public&reWriteBatchedInserts=true',
'username' = 'postgres',
'password' = 'linux123',
'table-name' = 'players3'
);

执行从源表插入结果表操作，生成同步作业

bash 复制代码 
INSERT INTO nba
SELECT
player_id,
team_id,
player_name,
height
FROM nbaplayers;

Web端查看

查看是否同步数据到pg库的players3表

相关推荐
夏小花花6 分钟前
postgresql 创建序列
数据库·postgresql
AORO_BEIDOU6 分钟前
抢抓5G机遇，AORO A23防爆手机如何直击园区巡检挑战？
大数据·5g·智能手机·信息与通信
Shaidou_Data7 分钟前
信息技术引领未来：大数据治理的实践与挑战
大数据·人工智能·数据清洗·信息技术·数据治理技术
Elastic 中国社区官方博客8 分钟前
开始使用 Elastic AI Assistant 进行可观察性和 Microsoft Azure OpenAI
大数据·人工智能·elasticsearch·microsoft·搜索引擎·全文检索·azure
engchina13 分钟前
Oracle ADB 导入 BANK_GRAPH 的学习数据
数据库·学习·oracle·graph
青云交20 分钟前
大数据新视界 -- 大数据大厂之 Impala 性能优化：新技术融合的无限可能（下）（12/30）
大数据·性能优化·impala·技术创新·新技术融合·电商案例·跨行业应用
CCSBRIDGE21 分钟前
sql文件
数据库·oracle
weixin_4426434222 分钟前
FileLink跨网文件安全摆渡系统——企业数据流转的安全桥梁
大数据·网络·安全·filelink文件摆渡系统
糖豆大将军1 小时前
Mysql个人八股总结
数据库·oracle
OBOO鸥柏1 小时前
OBOO鸥柏“触摸屏广告一体机交互”亮相2024中国珠海航展
大数据·人工智能·科技·交互
热门推荐
01玄机平台应急响应—webshell查杀02Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO03【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总04新版微信小程序获取用户手机号05文件或文件夹名称中有空格如何批量去除06组基轨迹建模 GBTM的介绍与实现（Stata 或 R）07【TC3xx芯片】TC3xx芯片电源管理系统PMS详解08优化手机性能，解决卡顿问题：关闭这3个微信开关，释放内存空间09基于YOLOv10深度学习的CT扫描图像肾结石智能检测系统【python源码+Pyqt5界面+数据集+训练代码】深度学习实战、目标检测10智能车联网安全发展形势、挑战