hive HQL

set hive.cli.print.header=true;

CREATE TABLE page_view(viewTime INT, userid BIGINT,

page_url STRING, referrer_url STRING,

ip STRING COMMENT 'IP Address of the User')

COMMENT 'This is the page view table'

PARTITIONED BY(dt STRING, country STRING)

ROW FORMAT DELIMITED

FIELDS TERMINATED BY '\001'

STORED AS SEQUENCEFILE; TEXTFILE

//sequencefile

create table tab_ip_seq(id int,name string,ip string,country string)

row format delimited

fields terminated by ','

stored as sequencefile;

insert overwrite table tab_ip_seq select * from tab_ext;

//create & load

create table tab_ip(id int,name string,ip string,country string)

row format delimited

fields terminated by ','

stored as textfile;

load data local inpath '/home/hadoop/ip.txt' into table tab_ext;

//external

CREATE EXTERNAL TABLE tab_ip_ext(id int, name string,

ip STRING,

country STRING)

ROW FORMAT DELIMITED FIELDS TERMINATED BY ','

STORED AS TEXTFILE

LOCATION '/external/hive';

// CTAS 用于创建一些临时表存储中间结果

CREATE TABLE tab_ip_ctas

AS

SELECT id new_id, name new_name, ip new_ip,country new_country

FROM tab_ip_ext

SORT BY new_id;

//insert from select 用于向临时表中追加中间结果数据

create table tab_ip_like like tab_ip;

insert overwrite table tab_ip_like

select * from tab_ip;

//CLUSTER <--相对高级一点,你可以放在有精力的时候才去学习>

create table tab_ip_cluster(id int,name string,ip string,country string)

clustered by(id) into 3 buckets;

load data local inpath '/home/hadoop/ip.txt' overwrite into table tab_ip_cluster;

set hive.enforce.bucketing=true;

insert into table tab_ip_cluster select * from tab_ip;

select * from tab_ip_cluster tablesample(bucket 2 out of 3 on id);

//PARTITION

create table tab_ip_part(id int,name string,ip string,country string)

partitioned by (part_flag string)

row format delimited fields terminated by ',';

load data local inpath '/home/hadoop/ip.txt' overwrite into table tab_ip_part

partition(part_flag='part1');

load data local inpath '/home/hadoop/ip_part2.txt' overwrite into table tab_ip_part

partition(part_flag='part2');

select * from tab_ip_part;

select * from tab_ip_part where part_flag='part2';

select count(*) from tab_ip_part where part_flag='part2';

alter table tab_ip change id id_alter string;

ALTER TABLE tab_cts ADD PARTITION (partCol = 'dt') location '/external/hive/dt';

show partitions tab_ip_part;

//write to hdfs

insert overwrite local directory '/home/hadoop/hivetemp/test.txt' select * from tab_ip_part where part_flag='part1';

insert overwrite directory '/hiveout.txt' select * from tab_ip_part where part_flag='part1';

//array

create table tab_array(a array<int>,b array<string>)

row format delimited

fields terminated by '\t'

collection items terminated by ',';

示例数据

tobenbrone,laihama,woshishui 13866987898,13287654321

abc,iloveyou,itcast 13866987898,13287654321

select a[0] from tab_array;

select * from tab_array where array_contains(b,'word');

insert into table tab_array select array(0),array(name,ip) from tab_ext t;

//map

create table tab_map(name string,info map<string,string>)

row format delimited

fields terminated by '\t'

collection items terminated by ';'

map keys terminated by ':';

示例数据:

fengjie age:18;size:36A;addr:usa

furong age:28;size:39C;addr:beijing;weight:180KG

load data local inpath '/home/hadoop/hivetemp/tab_map.txt' overwrite into table tab_map;

insert into table tab_map select name,map('name',name,'ip',ip) from tab_ext;

//struct

create table tab_struct(name string,info struct<age:int,tel:string,addr:string>)

row format delimited

fields terminated by '\t'

collection items terminated by ','

load data local inpath '/home/hadoop/hivetemp/tab_st.txt' overwrite into table tab_struct;

insert into table tab_struct select name,named_struct('age',id,'tel',name,'addr',country) from tab_ext;

//cli shell

hive -S -e 'select country,count(*) from tab_ext' > /home/hadoop/hivetemp/e.txt

有了这种执行机制,就使得我们可以利用脚本语言(bash shell,python)进行hql语句的批量执行

select * from tab_ext sort by id desc limit 5;

select a.ip,b.book from tab_ext a join tab_ip_book b on(a.name=b.name);

//UDF

0.要继承org.apache.hadoop.hive.ql.exec.UDF类实现evaluate

自定义函数调用过程:

1.添加jar包(在hive命令行里面执行)

hive> add jar /root/NUDF.jar;

2.创建临时函数

hive> create temporary function getNation as 'cn.itcast.hive.udf.NationUDF';

3.调用

hive> select id, name, getNation(nation) from beauty;

4.将查询结果保存到HDFS中

hive> create table result row format delimited fields terminated by '\t' as select * from beauty order by id desc;

hive> select id, getAreaName(id) as name from tel_rec;

hive>create table result row format delimited fields terminated by '\t' as select id, getNation(nation) from beauties;

hive>select if(id=1,first,no-first),name from tab_ext;

hive>add jar /home/hadoop/myudf.jar;

hive>CREATE TEMPORARY FUNCTION my_lower AS 'org.dht.Lower';

hive>select my_upper(name) from tab_ext;

相关推荐
2501_938782094 小时前
《大数据框架选型指南:Hadoop 与 Spark 的性能、成本与扩展性对比》
大数据·hadoop·spark
梦里不知身是客118 小时前
hive的SQL语句练习2
hive·hadoop·sql
梦里不知身是客118 小时前
hive的SQL练习3
hive·hadoop·sql
Q264336502311 小时前
【有源码】基于Hadoop与Spark的时尚精品店数据分析与可视化系统-基于多维度分析的零售时尚销售数据挖掘与可视化研究
大数据·hadoop·机器学习·数据挖掘·数据分析·spark·毕业设计
顧棟11 小时前
【Flink实战】升级HDFS,对Flink SQL(On Yarn模式)的功能兼容性验证
hadoop·flink·1024程序员节
笨蛋少年派21 小时前
使用hdfs命令报错:Unknown command:dfs(环境变量正确)
大数据·hadoop·hdfs
yumgpkpm1 天前
CMP(类ClouderaCDP7.3(404次编译) )完全支持华为鲲鹏Aarch64(ARM)POC报告
大数据·hive·hadoop·python·elasticsearch·hbase·cloudera
B站_计算机毕业设计之家2 天前
基于大数据的短视频数据分析系统 Spark哔哩哔哩视频数据分析可视化系统 Hadoop大数据技术 情感分析 舆情分析 爬虫 推荐系统 协同过滤推荐算法 ✅
大数据·hadoop·爬虫·spark·音视频·短视频·1024程序员节
洛克大航海2 天前
集群环境安装与部署 Hadoop
大数据·hadoop·ubuntu·集群部署 hadoop
笨蛋少年派2 天前
Hadoop High Availability 简介
大数据·hadoop·分布式