用Spark Streaming实时计算海量用户UV
浪尖聊大数据
共 2030字,需浏览 5分钟
·
2021-07-17 10:31
// 插入
setbit key offset value
//获取
getbit key offset
//计数
BITCOUNT key [start] [end]
redis 127.0.0.1:6379> PFADD runoobkey "redis"
1) (integer) 1
redis 127.0.0.1:6379> PFADD runoobkey "mongodb"
1) (integer) 1
redis 127.0.0.1:6379> PFADD runoobkey "mysql"
1) (integer) 1
redis 127.0.0.1:6379> PFCOUNT runoobkey
(integer) 3
stream.foreachRDD { rdd =>
//统计人数
rdd.foreachPartition { partition =>
//从分区所属executor的redis线程池获取一个连接.
val redis = RedisUtil.getRedis
partition.foreach { case (date, userId) =>
//统计当前userId
redis.pfadd(s"uv:$date", userId)
}
redis.close()
}
}
评论