Oracle之自动化部署etl程序

自动化配置 v1.0 user权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from user_all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme');

自动化配置 v2.0 Dba权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme')
   and t.OWNER = UPPER('&owner');

稽核1-数据量核对 自动化 v1.0

sql 复制代码
select (select to_char(nvl(max(to_number(a.sk_measure)), 0))
          from dc_ctl.ctl_audit_param a) + row_number() over(partition by 1 order by 1) as sk_measure,
       lower(t.src_tablename) as measure_name,
       '数量总量校对' as measure_desc,
       'STG_' || Upper(t.src_db_name) || '_RH_CNT' as data_set,
       Upper(t.src_db_name) as srcsys,
       Upper(t.src_db_name) as src_db,
       'STG' as trg_db,
       'SELECT COUNT(*) FROM $${schema_dstcs}.' || Upper(t.src_tablename) as src_sql,
       'REFRESH $${schema_stg}.' || Upper(t.trg_table) || ';' || CHR(10) ||
       'SELECT COUNT(*) FROM $${schema_stg}.' || Upper(t.trg_table) as trg_sql,
       'RH' as rule,
       0 as err_val,
       0 as err_type,
       'ERROR' as treatment_type,
       1 as dk_active,
       3 as exec_type
  from dc_ctl.ctl_load_config t
 where t.src_db_name = 'DSTCS'
 order by t.src_tablename;
相关推荐
听闻风很好吃1 小时前
DAY9:Oracle数据库安全管理深度解析
oracle
工头阿乐4 小时前
Appium自动化开发环境搭建
运维·appium·自动化
m0_748232924 小时前
你还在手动画ER图吗?让SQL自动生成ER图,轻松解决作业难题!
数据库·sql·oracle
代码小侦探5 小时前
Java中以Maven方式引入Oracle JDBC Driver依赖的详解
java·oracle·maven
DarkAthena6 小时前
【ORACLE】记录一些ORACLE的merge into语句的BUG
数据库·oracle·bug
江沉晚呤时6 小时前
深入了解C# List集合及两种常见排序算法:插入排序与堆排序
windows·sql·算法·oracle·c#·排序算法·mybatis
_extraordinary_10 小时前
MySQL 库的操作 -- 增删改查,备份和恢复,系统编码
android·mysql·oracle
施嘉伟15 小时前
Oracle 11g RAC ASM磁盘组剔盘、加盘实施过程
数据库·oracle
夸克App17 小时前
实现营销投放全流程自动化 超级汇川推出信息流智能投放产品“AI智投“
运维·人工智能·自动化
Rainbond云原生17 小时前
83k Star!n8n 让 AI 驱动的工作流自动化触手可及
运维·人工智能·自动化