Hudi 集成 | Apache Hudi 集成 Spark SQL 抢先体验

spark-sql --jars $PATH_TO_SPARK_BUNDLE_JAR  --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension'

set hoodie.upsert.shuffle.parallelism = 1;set hoodie.insert.shuffle.parallelism = 1;set hoodie.delete.shuffle.parallelism = 1;

set hoodie.datasource.meta.sync.enable=false;

create table test_hudi_table (  id int,  name string,  price double,  ts long,  dt string) using hudi partitioned by (dt) options (  primaryKey = 'id',  type = 'mor' ) location 'file:///tmp/test_hudi_table'

show create table test_hudi_table

 insert into test_hudi_table select 1 as id, 'hudi' as name, 10 as price, 1000 as ts, '2021-05-05' as dt

select * from test_hudi_table

update test_hudi_table set price = 20.0 where id = 1

select * from test_hudi_table

delete from test_hudi_table where id = 1

select * from test_hudi_table;

 merge into test_hudi_table as t0 using (  select 1 as id, 'a1' as name, 10 as price, 1000 as ts, '2021-03-21' as dt ) as s0 on t0.id = s0.id when not matched and s0.id % 2 = 1 then insert *

select * from test_hudi_table

 merge into test_hudi_table as t0 using (  select 1 as id, 'a1' as name, 12 as price, 1001 as ts, '2021-03-21' as dt ) as s0 on t0.id = s0.id when matched and s0.id % 2 = 1 then update set *

select * from test_hudi_table

merge into test_hudi_table t0 using (  select 1 as s_id, 'a2' as s_name, 15 as s_price, 1001 as s_ts, '2021-03-21' as dt ) s0 on t0.id = s0.s_id when matched and s_ts = 1001 then delete

drop table test_hudi_table;

show tables;