使用datax通过HbaseShell封装_同步hbase数据到瀚高数据库highgo_踩坑_细节总结---大数据之DataX工作笔记009

脑瓜凉2025-04-21 9:12

复制代码

{
    "job": {
        "setting": {
            "speed": {
                "channel": 3, 
                "byte": 1048576
            }, 
            "errorLimit": {
                "record": 0, 
                "percentage": 0.02
            }
        }, 
        "content": [
            {
                "reader": {
                    "name": "hbase11xreader", 
                    "parameter": {
                        "hbaseConfig": {
                            "hbase.zookeeper.quorum": "172.19.126.120,172.19.126.121,172.19.126.122:2181"
                        }, 
                        "table": "userinfoshell", 
                        "mode": "normal", 
                        "column": [
 	         	{
                               "name": "rowkey", 
                                "type": "string"
                            }, 
                            {
                                "name": "info:addr", 
                                "type": "string"
                            }, 
                            {
                                "name": "info:son_json", 
                                "type": "string"
                            }, 
                            {
                                "name": "info:mobile", 
                                "type": "string"
                            }, 
                            {
                                "name": "info:name", 
                                "type": "string"
                            }, 
                            {
                                "name": "info:email", 
                                "type": "string"
                            }
                        ]
                    }
                }, 
                "writer": {
                    "name": "mysqlwriter", 
                    "parameter": {
                        "username": "0uADFAaASDASDFASDmOA==", 
                        "password": "niVbuSDSESEFSDSdA==", 
                        "column": [
                            "id", 
		    "addr",
                            "son_json", 
                            "mobile", 
                            "name", 
                            "email"
                        ], 
                        "connection": [
                            {
                                "table": [
                                    "dzrb_bigdata_huiju.user_info"
                                ], 
                                "jdbcUrl": "jdbc:highgo://172.19.126.125:5866/datS_bigdata_cloud"
                            }
                        ]
                    }
                }
            }
        ]
    }, 
    "core": {
        "transport": {
            "channel": {
                "speed": {
                    "byte": 1231412
                }
            }
        }
    }
}

给出一个可以运行的json 用来把hbase同步到瀚高的,有了这个json就好多了.

可以看到,对应的,字段对应关系是可以直接对应起来的,不用字段名字一样,就是说,

hbase中写了比如info:name,info:age ,其实对应的瀚高中可以写成column $id,addr$

这样就相当于把name传入到id中,把ageaddr

然后就可以去使用datax-web去跑一下试试了.