Oracle之自动化部署etl程序

自动化配置 v1.0 user权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from user_all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme');

自动化配置 v2.0 Dba权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme')
   and t.OWNER = UPPER('&owner');

稽核1-数据量核对 自动化 v1.0

sql 复制代码
select (select to_char(nvl(max(to_number(a.sk_measure)), 0))
          from dc_ctl.ctl_audit_param a) + row_number() over(partition by 1 order by 1) as sk_measure,
       lower(t.src_tablename) as measure_name,
       '数量总量校对' as measure_desc,
       'STG_' || Upper(t.src_db_name) || '_RH_CNT' as data_set,
       Upper(t.src_db_name) as srcsys,
       Upper(t.src_db_name) as src_db,
       'STG' as trg_db,
       'SELECT COUNT(*) FROM $${schema_dstcs}.' || Upper(t.src_tablename) as src_sql,
       'REFRESH $${schema_stg}.' || Upper(t.trg_table) || ';' || CHR(10) ||
       'SELECT COUNT(*) FROM $${schema_stg}.' || Upper(t.trg_table) as trg_sql,
       'RH' as rule,
       0 as err_val,
       0 as err_type,
       'ERROR' as treatment_type,
       1 as dk_active,
       3 as exec_type
  from dc_ctl.ctl_load_config t
 where t.src_db_name = 'DSTCS'
 order by t.src_tablename;
相关推荐
七夜zippoe8 小时前
破解 VMware 迁移难题:跨平台迁移常见问题及自动化解决方案
运维·自动化·vmware
白仑色9 小时前
Oracle 高可用性与安全性
数据库·oracle·数据安全·goldengate·高可用架构
Listennnn14 小时前
Agent自动化与代码智能
人工智能·自动化
叶甯14 小时前
【oracle】oracle 常用日期操作
oracle
暮色驶过苍茫17 小时前
H2 与高斯数据库兼容性解决方案:虚拟表与类型处理
数据库·oracle
爱分享的飘哥18 小时前
第十九篇 自动化报表生成:Python一键生成可视化Excel图表与专业PDF报告,老板看了都点赞!
自动化·办公自动化·数据可视化·excel自动化·python报表·pdf报告
特种加菲猫1 天前
构建完整工具链:GCC/G++ + Makefile + Git 自动化开发流程
linux·笔记·git·自动化
李元豪1 天前
grpo nl2sql qwen3 模型强化学习训练有效果的成立条件有哪些
数据库·oracle
王者鳜錸1 天前
使用Selenium自动化获取抖音创作者平台视频数据
selenium·自动化·音视频
chao_7891 天前
frame 与新窗口切换操作【selenium 】
前端·javascript·css·selenium·测试工具·自动化·html