Oracle之自动化部署etl程序

自动化配置 v1.0 user权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from user_all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ,')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from USER_TAB_COLUMNS t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme');

自动化配置 v2.0 Dba权限

sql 复制代码
select upper('DSTCS_' || t.TABLE_NAME) as filename,
       'DSTCS' as src_db_name,
       'TCS' as src_schema,
       upper(t.TABLE_NAME) as src_tablename,
       null as src_where,
       t1.src_query,
       null as src_splitpk,
       'STG' as trg_schema,
       Upper('DSTCS_' || t.TABLE_NAME) as trg_table,
       'F' as trg_writemode,
       null as post_sql,
       '\0x001' as filed_delimter,
       null as sqoop_param,
       null as hive_param,
       upper('DSTCS_' || t.TABLE_NAME || '_EXT') as ext_tablename,
       t2.post_sql_impala,
       5 as map_num,
       '3' as load_type
  from all_tables t
  left join ( --src_query
             select t.TABLE_NAME,
                     'select ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(lower(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as inserttime' ||
                     CHR(10) ||
                     '      ,to_char(sysdate, ''yyyy-mm-dd hh24:mi:ss'') as updatetime' ||
                     CHR(10) || '      ,${batchno} as batchno' || CHR(10) ||
                     '      ,${srcsys} as srcsys' || CHR(10) ||
                     ' from $${schema_dstcs}.' || lower(t.TABLE_NAME) ||
                     CHR(10) || 'where 1 = 1' as src_query
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t1
    on t.TABLE_NAME = t1.TABLE_NAME
  left join ( --post_sql_impala
             select t.TABLE_NAME,
                     'REFRESH $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) ||
                     '_EXT;' || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' || CHR(10) || CHR(10) ||
                     'INSERT OVERWRITE TABLE $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || CHR(10) || 'SELECT ' ||
                     RTRIM(XMLAGG(XMLPARSE(CONTENT(upper(COLUMN_NAME) ||
                                                   CHR(10) || '      ' || ',')
                                           WELLFORMED) ORDER BY COLUMN_ID).GETCLOBVAL()) ||
                     'INSERTTIME' || CHR(10) || '      ,UPDATETIME' || CHR(10) ||
                     '      ,BATCHNO' || CHR(10) || '      ,SRCSYS' || CHR(10) ||
                     ' FROM $${schema_stg}.DSTCS_' || upper(t.TABLE_NAME) || '_EXT;' ||
                     CHR(10) || CHR(10) || 'REFRESH $${schema_stg}.DSTCS_' ||
                     upper(t.TABLE_NAME) || ';' as post_sql_impala
               from all_tab_columns t
              where t.TABLE_NAME = upper(t.TABLE_NAME)
              group by t.TABLE_NAME) t2
    on t.TABLE_NAME = t2.TABLE_NAME
 where t.TABLE_NAME = upper('&tanme')
   and t.OWNER = UPPER('&owner');

稽核1-数据量核对 自动化 v1.0

sql 复制代码
select (select to_char(nvl(max(to_number(a.sk_measure)), 0))
          from dc_ctl.ctl_audit_param a) + row_number() over(partition by 1 order by 1) as sk_measure,
       lower(t.src_tablename) as measure_name,
       '数量总量校对' as measure_desc,
       'STG_' || Upper(t.src_db_name) || '_RH_CNT' as data_set,
       Upper(t.src_db_name) as srcsys,
       Upper(t.src_db_name) as src_db,
       'STG' as trg_db,
       'SELECT COUNT(*) FROM $${schema_dstcs}.' || Upper(t.src_tablename) as src_sql,
       'REFRESH $${schema_stg}.' || Upper(t.trg_table) || ';' || CHR(10) ||
       'SELECT COUNT(*) FROM $${schema_stg}.' || Upper(t.trg_table) as trg_sql,
       'RH' as rule,
       0 as err_val,
       0 as err_type,
       'ERROR' as treatment_type,
       1 as dk_active,
       3 as exec_type
  from dc_ctl.ctl_load_config t
 where t.src_db_name = 'DSTCS'
 order by t.src_tablename;
相关推荐
TDengine (老段)5 小时前
从 ETL 到 Agentic AI:工业数据管理变革与 TDengine IDMP 的治理之道
数据库·数据仓库·人工智能·物联网·时序数据库·etl·tdengine
wei_shuo10 小时前
使用 Auto-Keras 进行自动化机器学习
机器学习·自动化·keras
HCIE考证研究所14 小时前
OCA、OCP、OCM傻傻分不清?Oracle认证就看这篇
oracle
--运维实习生--14 小时前
自动化运维之ansible
运维·自动化·ansible
曹牧17 小时前
Oracle:select top 5
数据库·sql·oracle
Source.Liu19 小时前
【Python自动化】 21.2 Pandas 读取 Excel 时的 dtype 参数完全指南
python·自动化·pandas
Source.Liu1 天前
【Python自动化】 21.1 Pandas 读取 Excel 文件的完整指南
python·自动化·pandas
张人玉1 天前
SQLSERVER数据备份
数据库·oracle·sqlserver
幂简集成1 天前
基于 Gemini 的 CI/CD 自动化测评 API 集成实战教程
运维·ci/cd·自动化
W-GEO1 天前
Spring Boot 源码深度解析:揭秘自动化配置的魔法
spring boot·后端·自动化