Oracle之自动化部署etl程序

自动化配置 v1.0 user权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from user_all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme');

自动化配置 v2.0 Dba权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme')
   and t.OWNER = UPPER('&owner');

稽核1-数据量核对 自动化 v1.0

sql 复制代码
select (select to_char(nvl(max(to_number(a.sk_measure)), 0))
          from dc_ctl.ctl_audit_param a) + row_number() over(partition by 1 order by 1) as sk_measure,
       lower(t.src_tablename) as measure_name,
       '数量总量校对' as measure_desc,
       'STG_' || Upper(t.src_db_name) || '_RH_CNT' as data_set,
       Upper(t.src_db_name) as srcsys,
       Upper(t.src_db_name) as src_db,
       'STG' as trg_db,
       'SELECT COUNT(*) FROM $${schema_dstcs}.' || Upper(t.src_tablename) as src_sql,
       'REFRESH $${schema_stg}.' || Upper(t.trg_table) || ';' || CHR(10) ||
       'SELECT COUNT(*) FROM $${schema_stg}.' || Upper(t.trg_table) as trg_sql,
       'RH' as rule,
       0 as err_val,
       0 as err_type,
       'ERROR' as treatment_type,
       1 as dk_active,
       3 as exec_type
  from dc_ctl.ctl_load_config t
 where t.src_db_name = 'DSTCS'
 order by t.src_tablename;
相关推荐
工业通讯探索者8 小时前
ProfiNet转CANopen协议转换网关驱动新能源汽车生产线多轴同步控制
自动化·工业物联网·协议转换网关·网关模块·总线协议
小马哥编程10 小时前
【软件测试】自动化测试结合 CI/CD有哪些方案
自动化·集成测试·测试覆盖率
Qi妙代码10 小时前
MYSQL基础
数据库·mysql·oracle
Arbori_2621510 小时前
Oracle中的UNION原理
数据库·oracle
weixin_4578858211 小时前
JavaScript智能对话机器人——企业知识库自动化
开发语言·javascript·自动化
东方佑11 小时前
利用Python自动化处理PPT样式与结构:从提取到生成
python·自动化·powerpoint
WannaRunning13 小时前
浅谈Tomcat数据源连接池
java·oracle·tomcat
niuniu_66614 小时前
简单的自动化场景(以 Chrome 浏览器 为例)
运维·chrome·python·selenium·测试工具·自动化·安全性测试
嘴对嘴编程14 小时前
oracle数据泵操作
数据库·oracle
吾日三省吾码1 天前
Python 脚本:自动化你的日常任务
数据库·python·自动化