1，因業務需求，需要監控yarn的某些常駐程序，如果挂了的話，打電話報警或者把相關報警發到報警群裡。
package com.crgt;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
/**
* @Author: wpp
* @Date: 2020/3/29 13:36
*
* https://blog.csdn.net/zhangshenghang/article/details/104447587#%E4%BB%A3%E7%A0%81
*
*/
public class YarnMonitor {
public static void main(String[] args) throws IOException, InterruptedException {
// while(true) {
// TimeUnit.SECONDS.sleep(10);
// Map<String,String> hashMaps = RedisUtil.getInstance().hash().hgetAll("yarn_monitor_hash");
// for(String key : hashMaps.keySet()) {
// Boolean yarnIsContains = yarnIsContains(key);
// String msg = "yarn的排程名稱：" + key +",yarn的任務："+ hashMaps.getOrDefault(key,"")+",任務是否存在"+yarnIsContains;
// System.out.println(msg);
// if(!yarnIsContains){
// sendMsgToWeiXin(msg);
// }
// }
// }
while(true) {
String threadSleep = RedisUtil.getInstance().strings().get("yarn_scheduler_time");
System.out.println(Integer.valueOf(threadSleep));
TimeUnit.SECONDS.sleep(Integer.valueOf(threadSleep));
Map<String,String> hashMaps = RedisUtil.getInstance().hash().hgetAll("yarn_monitor_hash");
StringBuilder sb = new StringBuilder();
Map<String ,Boolean> yarnContainsMap = yarnIsRuning(hashMaps);
for(String key : yarnContainsMap.keySet() ){
if(!yarnContainsMap.get(key)){
sb.append("yarn的排程名稱：" + key +",yarn的任務："+ hashMaps.getOrDefault(key,"")+",任務是否存在:"+yarnContainsMap.get(key) + "\n");
}else{
System.out.println(key + "資料存在");
}
}
if(sb.toString().length() != 0){
sendMsgToWeiXin(sb.toString());
System.out.println("ressss======="+sb.toString());
}
}
}
/**
* 擷取任務的applicationId
* @return String
* @param jobName
* @return
*/
public static String getAppId(String jobName) {
YarnClient client = YarnClient.createYarnClient();
Configuration conf = new Configuration();
client.init(conf);
client.start();
EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
if (appStates.isEmpty()) {
appStates.add(YarnApplicationState.RUNNING);
appStates.add(YarnApplicationState.ACCEPTED);
appStates.add(YarnApplicationState.SUBMITTED);
}
List<ApplicationReport> appsReport = null;
try {
//傳回EnumSet<YarnApplicationState>中個人任務狀态的所有任務
appsReport = client.getApplications(appStates);
} catch (YarnException | IOException e) {
e.printStackTrace();
}
assert appsReport != null;
for (ApplicationReport appReport : appsReport) {
//擷取任務名
String jn = appReport.getName();
String applicationType = appReport.getApplicationType();
if (jn.equals(jobName) && "Apache Flink".equals(applicationType)) {
try {
client.close();
} catch (IOException e) {
e.printStackTrace();
}
return appReport.getApplicationId().toString();
}
}
try {
client.close();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
/**
* 根據任務的applicationId去擷取任務的狀态
* @return YarnApplicationState
* @param appId
* @return
*/
public static YarnApplicationState getState(String appId) {
YarnClient client = YarnClient.createYarnClient();
Configuration conf = new Configuration();
client.init(conf);
client.start();
ApplicationId applicationId = ApplicationId.fromString(appId);
// ApplicationId appId = ConverterUtils.toApplicationId(appId);
YarnApplicationState yarnApplicationState = null;
try {
ApplicationReport applicationReport = client.getApplicationReport(applicationId);
yarnApplicationState = applicationReport.getYarnApplicationState();
} catch (YarnException | IOException e) {
e.printStackTrace();
}
try {
client.close();
} catch (IOException e) {
e.printStackTrace();
}
return yarnApplicationState;
}
public static void sendMsgToWeiXin(String msg){
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("https://screen-bigdata.ccrgt.com/sms/sendReport");
httpPost.addHeader("Content-Type", "application/x-www-form-urlencoded");
List<NameValuePair> valuePairs = new ArrayList<>();
NameValuePair valuePair = new BasicNameValuePair("msg", msg);
valuePairs.add(valuePair);
HttpEntity entity = null;
try {
entity = new UrlEncodedFormEntity(valuePairs,"UTF-8");
httpPost.setEntity(entity);
httpclient.execute(httpPost);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 判斷任務名為appName的任務，是否在yarn中運作，狀态為RUNNING
* @return Boolean
* @param appName
* @return
*/
public static Boolean yarnIsContains(String appName) {
Configuration conf = new YarnConfiguration();
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
Boolean isContains = false;
List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
try {
//applications = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FINISHED));
applications = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
for(ApplicationReport application : applications) {
String name = application.getName();
if(name.equals(appName)) {
System.out.println("ApplicationId ============> "+application.getApplicationId());
System.out.println("name ============> "+application.getName());
System.out.println("queue ============> "+application.getQueue());
System.out.println("user ============> "+application.getUser());
System.out.println("application=========" + application);
isContains = true;
}
}
/*
* if(applications.contains(appName)) {
* System.out.println("ApplicationId ============> "+applications.get(0).
* getApplicationId());
* System.out.println("name ============> "+applications.get(0).getName());
* System.out.println("queue ============> "+applications.get(0).getQueue());
* System.out.println("queue ============> "+applications.get(0).getUser());
* System.out.println(applications); }
*/
} catch (YarnException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
yarnClient.stop();
}
return isContains;
}
/**
* 判斷任務名為appName的任務，是否在yarn中運作，狀态為RUNNING
* @return Boolean
* @param appName
* @return
*/
public static Map<String,Boolean> yarnIsRuning(Map<String,String> map) {
Configuration conf = new YarnConfiguration();
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
Map<String,Boolean> resMap = new HashMap<String,Boolean>();
try {
//applications = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FINISHED));
applications = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
for(String appName : map.keySet()){
resMap.put(appName,false);
for(ApplicationReport application : applications) {
String name = application.getName();
if(name.equals(appName)) {
System.out.println("ApplicationId ============> "+application.getApplicationId());
System.out.println("name ============> "+application.getName());
System.out.println("queue ============> "+application.getQueue());
System.out.println("user ============> "+application.getUser());
System.out.println("application=========" + application);
resMap.put(appName,true);
}
}
}
} catch (YarnException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
yarnClient.stop();
}
return resMap;
}
}
2，使用python，把要添加的任務添加redis，進行相關處理
import redis
r = redis.StrictRedis(host="10.2.12.76", port=6379, password="eD5JbS9y9ZDB5umf", db=0)
key = 'yarn_monitor_hash'
r.set('yarn_scheduler_time', 3600)
#添加相關監控名額
#r.hset(key, 'ComputeGpsInOutStationDev', '進出站計算')
r.hset(key, 'dws_first_mac_dri2', '首次開網人數')
#r.hset(key, 'dws_first_mac_dri2', '首次開網人數2')
#删除相關資料
# r.hdel(key, 'TracingDataSparkStreamingToESTest1')
r.hdel(key, 'dws_first_mac_dri')
#r.delete(key)
list2 = r.hgetall(key)
for (key, val) in list2.items():
print(key.decode('utf-8'), val.decode('utf-8'))
3.連結redis的util
package com.crgt;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.SortingParams;
import redis.clients.util.SafeEncoder;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
/**
* Redis工具類
* Created by caicf on 2017/1/1.
*/
public class RedisUtil {
private static final Logger log = LoggerFactory.getLogger(RedisUtil.class);
//預設緩存時間
private static final int EXPIRE = 60000;
private static Properties properties;
private static RedisUtil instance;
private static JedisPool jedisPool;
private static ReentrantLock lock = new ReentrantLock();
private RedisUtil() {
}
public static RedisUtil getInstance() {
if (instance == null) {
lock.lock();
if (instance == null) {
instance = new RedisUtil();
}
lock.unlock();
}
return instance;
}
/**
* 初始化JedisPool
*/
private void initJedisPool() {
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxIdle(100);
config.setTestOnBorrow(Boolean.getBoolean("true"));
config.setTestOnReturn(Boolean.getBoolean("true"));
Properties properties = new Properties();
InputStream fileInputStream = RedisUtil.class.getClassLoader().getResourceAsStream("app-dev.properties");
if ( !";".equals(File.pathSeparator)) {//非window環境
fileInputStream = RedisUtil.class.getClassLoader().getResourceAsStream("app-prod.properties");
}
try {
properties.load(fileInputStream);
} catch (IOException e) {
e.printStackTrace();
}
/**擷取屬性檔案中的值**/
String REDIS_HOST_CLS = properties.getProperty("REDIS_HOST_CLS");
String REDIS_PASSWORD_CLS = properties.getProperty("REDIS_PASSWORD_CLS");
jedisPool = new JedisPool(
config,
// "10.3.1.9", 6379,60000,"iINEivdRA8Kpin13"
// "10.3.1.6", 6379,60000,"pt1Os)Qhpch1jmft"
REDIS_HOST_CLS, 6379,60000,REDIS_PASSWORD_CLS
);
}
/**
* 通用方法：從JedisPool中擷取Jedis
*
* @return
*/
private Jedis getJedis() {
if (jedisPool == null) {
lock.lock(); //防止吃初始化時多線程競争問題
initJedisPool();
lock.unlock();
log.info("JedisPool init success！");
}
return jedisPool.getResource();
}
/**
* 通用方法：釋放Jedis
*
* @param jedis
*/
private void closeJedis(Jedis jedis) {
jedis.close();
}
//===========================================================
/**
* 對Keys,以及存儲結構為String、List、Set、HashMap類型的操作
*/
private final Keys keys = new Keys();
private final Strings strings = new Strings();
private final Lists lists = new Lists();
private final Sets sets = new Sets();
private final Hash hash = new Hash();
private final SortSet sortset = new SortSet();
public Keys keys() {
return keys;
}
public Strings strings() {
return strings;
}
public Lists lists() {
return lists;
}
public Sets sets() {
return sets;
}
public Hash hash() {
return hash;
}
public SortSet sortSet() {
return sortset;
}
//===========================================================
//*******************************************Keys*******************************************//
public class Keys {
/**
* 設定過期時間
*
* @param key
* @param seconds
* @return 傳回影響的記錄數
*/
public long expire(String key, int seconds) {
if (seconds <= 0) {
return -1L;
}
Jedis jedis = getJedis();
long result = jedis.expire(key, seconds);
closeJedis(jedis);
return result;
}
/**
* 設定過期時間，預設值為60000seconds
*
* @param key
*/
public long expire(String key) {
return expire(key, EXPIRE);
}
/**
* 設定key的過期時間,它是距曆元（即格林威治标準時間 1970 年 1 月 1 日的 00:00:00，格裡高利曆）的偏移量。
*
* @param key
* @param timestamp 秒
* @return 影響的記錄數
*/
public long expireAt(String key, long timestamp) {
Jedis jedis = getJedis();
long count = jedis.expireAt(key, timestamp);
closeJedis(jedis);
return count;
}
/**
* 查詢key的過期時間
*
* @param key
* @return 以秒為機關的時間表示
*/
public long ttl(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long len = sjedis.ttl(key);
closeJedis(sjedis);
return len;
}
/**
* 取消對key過期時間的設定
*
* @param key
* @return 影響的記錄數
*/
public long persist(String key) {
Jedis jedis = getJedis();
long count = jedis.persist(key);
closeJedis(jedis);
return count;
}
/**
* 清空所有key
*
* @return
*/
public String flushAll() {
Jedis jedis = getJedis();
String stata = jedis.flushAll();
closeJedis(jedis);
return stata;
}
/**
* 判斷key是否存在
*
* @param key
* @return boolean
*/
public boolean exists(String key) {
Jedis sjedis = getJedis();
boolean exis = sjedis.exists(key);
closeJedis(sjedis);
return exis;
}
/**
* 更改key
*/
public String rename(String oldKey, String newKey) {
return rename(SafeEncoder.encode(oldKey),
SafeEncoder.encode(newKey));
}
/**
* 更改key,僅當新key不存在時才執行
*
* @param oldKey
* @param newKey
* @return 狀态碼
*/
public long renamenx(String oldKey, String newKey) {
Jedis jedis = getJedis();
long status = jedis.renamenx(oldKey, newKey);
closeJedis(jedis);
return status;
}
/**
* 更改key
*/
public String rename(byte[] oldKey, byte[] newKey) {
Jedis jedis = getJedis();
String status = jedis.rename(oldKey, newKey);
closeJedis(jedis);
return status;
}
/**
* 删除keys對應的記錄,可以是多個key
*
* @param keys
* @return 删除的記錄數
*/
public long del(String... keys) {
Jedis jedis = getJedis();
long count = jedis.del(keys);
closeJedis(jedis);
return count;
}
/**
* 删除keys對應的記錄,可以是多個key
*
* @param keys
* @return 删除的記錄數
*/
public long del(byte[]... keys) {
Jedis jedis = getJedis();
long count = jedis.del(keys);
closeJedis(jedis);
return count;
}
/**
* 對List,Set,SortSet進行排序,如果集合資料較大應避免使用這個方法
*
* @param key
* @return List<String> 集合的全部記錄
**/
public List<String> sort(String key) {
Jedis sjedis = getJedis();
List<String> list = sjedis.sort(key);
closeJedis(sjedis);
return list;
}
/**
* 對List,Set,SortSet進行排序或limit
*
* @param key
* @param parame 定義排序類型或limit的起止位置.
* @return List<String> 全部或部分記錄
**/
public List<String> sort(String key, SortingParams parame) {
Jedis jedis = getJedis();
List<String> list = jedis.sort(key, parame);
closeJedis(jedis);
return list;
}
/**
* 傳回指定key存儲的類型
*
* @param key
* @return String string|list|set|zset|hash
**/
public String type(String key) {
Jedis sjedis = getJedis();
String type = sjedis.type(key);
closeJedis(sjedis);
return type;
}
/**
* 查找所有比對給定的模式的鍵
*
* @param pattern 的表達式,*表示多個，？表示一個
*/
public Set<String> keys(String pattern) {
Jedis jedis = getJedis();
Set<String> set = jedis.keys(pattern);
closeJedis(jedis);
return set;
}
}
//*******************************************Sets*******************************************//
public class Sets {
/**
* 向Set添加一條記錄，如果member已存在傳回0,否則傳回1
*
* @param key
* @param member
* @return 操作碼, 0或1
*/
public long sadd(String key, String member) {
Jedis jedis = getJedis();
long s = jedis.sadd(key, member);
closeJedis(jedis);
return s;
}
public long sadd(byte[] key, byte[] member) {
Jedis jedis = getJedis();
long s = jedis.sadd(key, member);
closeJedis(jedis);
return s;
}
/**
* 擷取給定key中元素個數
*
* @param key
* @return 元素個數
*/
public long scard(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long len = sjedis.scard(key);
closeJedis(sjedis);
return len;
}
/**
* 傳回從第一組和所有的給定集合之間的差異的成員
*
* @param keys
* @return 差異的成員集合
*/
public Set<String> sdiff(String... keys) {
Jedis jedis = getJedis();
Set<String> set = jedis.sdiff(keys);
closeJedis(jedis);
return set;
}
/**
* 這個指令等于sdiff,但傳回的不是結果集,而是将結果集存儲在新的集合中，如果目标已存在，則覆寫。
*
* @param newKey 新結果集的key
* @param keys 比較的集合
* @return 新集合中的記錄數
**/
public long sdiffstore(String newKey, String... keys) {
Jedis jedis = getJedis();
long s = jedis.sdiffstore(newKey, keys);
closeJedis(jedis);
return s;
}
/**
* 傳回給定集合交集的成員,如果其中一個集合為不存在或為空，則傳回空Set
*
* @param keys
* @return 交內建員的集合
**/
public Set<String> sinter(String... keys) {
Jedis jedis = getJedis();
Set<String> set = jedis.sinter(keys);
closeJedis(jedis);
return set;
}
/**
* 這個指令等于sinter,但傳回的不是結果集,而是将結果集存儲在新的集合中，如果目标已存在，則覆寫。
*
* @param newKey 新結果集的key
* @param keys 比較的集合
* @return 新集合中的記錄數
**/
public long sinterstore(String newKey, String... keys) {
Jedis jedis = getJedis();
long s = jedis.sinterstore(newKey, keys);
closeJedis(jedis);
return s;
}
/**
* 确定一個給定的值是否存在
*
* @param key
* @param member 要判斷的值
* @return 存在傳回1，不存在傳回0
**/
public boolean sismember(String key, String member) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
boolean s = sjedis.sismember(key, member);
closeJedis(sjedis);
return s;
}
/**
* 傳回集合中的所有成員
*
* @param key
* @return 成員集合
*/
public Set<String> smembers(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<String> set = sjedis.smembers(key);
closeJedis(sjedis);
return set;
}
public Set<byte[]> smembers(byte[] key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<byte[]> set = sjedis.smembers(key);
closeJedis(sjedis);
return set;
}
/**
* 将成員從源集合移出放入目标集合 <br/>
* 如果源集合不存在或不包哈指定成員，不進行任何操作，傳回0<br/>
* 否則該成員從源集合上删除，并添加到目标集合，如果目标集合中成員已存在，則隻在源集合進行删除
*
* @param srckey 源集合
* @param dstkey 目标集合
* @param member 源集合中的成員
* @return 狀态碼，1成功，0失敗
*/
public long smove(String srckey, String dstkey, String member) {
Jedis jedis = getJedis();
long s = jedis.smove(srckey, dstkey, member);
closeJedis(jedis);
return s;
}
/**
* 從集合中删除成員
*
* @param key
* @return 被删除的成員
*/
public String spop(String key) {
Jedis jedis = getJedis();
String s = jedis.spop(key);
closeJedis(jedis);
return s;
}
/**
* 從集合中删除指定成員
*
* @param key
* @param member 要删除的成員
* @return 狀态碼，成功傳回1，成員不存在傳回0
*/
public long srem(String key, String member) {
Jedis jedis = getJedis();
long s = jedis.srem(key, member);
closeJedis(jedis);
return s;
}
/**
* 合并多個集合并傳回合并後的結果，合并後的結果集合并不儲存<br/>
*
* @param keys
* @return 合并後的結果集合
*/
public Set<String> sunion(String... keys) {
Jedis jedis = getJedis();
Set<String> set = jedis.sunion(keys);
closeJedis(jedis);
return set;
}
/**
* 合并多個集合并将合并後的結果集儲存在指定的新集合中，如果新集合已經存在則覆寫
*
* @param newKey 新集合的key
* @param keys 要合并的集合
**/
public long sunionstore(String newKey, String... keys) {
Jedis jedis = getJedis();
long s = jedis.sunionstore(newKey, keys);
closeJedis(jedis);
return s;
}
}
//*******************************************SortSet*******************************************//
public class SortSet {
/**
* 向集合中增加一條記錄,如果這個值已存在，這個值對應的權重将被置為新的權重
*
* @param key
* @param score 權重
* @param member 要加入的值，
* @return 狀态碼 1成功，0已存在member的值
*/
public long zadd(String key, double score, String member) {
Jedis jedis = getJedis();
long s = jedis.zadd(key, score, member);
closeJedis(jedis);
return s;
}
/**
* 擷取集合中元素的數量
*
* @param key
* @return 如果傳回0則集合不存在
*/
public long zcard(String key) {
Jedis sjedis = getJedis();
long len = sjedis.zcard(key);
closeJedis(sjedis);
return len;
}
/**
* 擷取指定權重區間内集合的數量
*
* @param key
* @param min 最小排序位置
* @param max 最大排序位置
*/
public long zcount(String key, double min, double max) {
Jedis sjedis = getJedis();
long len = sjedis.zcount(key, min, max);
closeJedis(sjedis);
return len;
}
/**
* 獲得set的長度
*
* @param key
* @return
*/
public long zlength(String key) {
long len = 0;
Set<String> set = zrange(key, 0, -1);
len = set.size();
return len;
}
/**
* 權重增加給定值，如果給定的member已存在
*
* @param key
* @param score 要增的權重
* @param member 要插入的值
* @return 增後的權重
*/
public double zincrby(String key, double score, String member) {
Jedis jedis = getJedis();
double s = jedis.zincrby(key, score, member);
closeJedis(jedis);
return s;
}
/**
* 傳回指定位置的集合元素,0為第一個元素，-1為最後一個元素
*
* @param key
* @param start 開始位置(包含)
* @param end 結束位置(包含)
* @return Set<String>
*/
public Set<String> zrange(String key, int start, int end) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<String> set = sjedis.zrange(key, start, end);
closeJedis(sjedis);
return set;
}
/**
* 傳回指定權重區間的元素集合
*
* @param key
* @param min 上限權重
* @param max 下限權重
* @return Set<String>
*/
public Set<String> zrangeByScore(String key, double min, double max) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<String> set = sjedis.zrangeByScore(key, min, max);
closeJedis(sjedis);
return set;
}
/**
* 擷取指定值在集合中的位置，集合排序從低到高
*
* @param key
* @param member
* @return long 位置
*/
public long zrank(String key, String member) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long index = sjedis.zrank(key, member);
closeJedis(sjedis);
return index;
}
/**
* 擷取指定值在集合中的位置，集合排序從高到低
*
* @param key
* @param member
* @return long 位置
*/
public long zrevrank(String key, String member) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long index = sjedis.zrevrank(key, member);
closeJedis(sjedis);
return index;
}
/**
* 從集合中删除成員
*
* @param key
* @param member
* @return 傳回1成功
*/
public long zrem(String key, String member) {
Jedis jedis = getJedis();
long s = jedis.zrem(key, member);
closeJedis(jedis);
return s;
}
/**
* 删除
*
* @param key
* @return
*/
public long zrem(String key) {
Jedis jedis = getJedis();
long s = jedis.del(key);
closeJedis(jedis);
return s;
}
/**
* 删除給定位置區間的元素
*
* @param key
* @param start 開始區間，從0開始(包含)
* @param end 結束區間,-1為最後一個元素(包含)
* @return 删除的數量
*/
public long zremrangeByRank(String key, int start, int end) {
Jedis jedis = getJedis();
long s = jedis.zremrangeByRank(key, start, end);
closeJedis(jedis);
return s;
}
/**
* 删除給定權重區間的元素
*
* @param key
* @param min 下限權重(包含)
* @param max 上限權重(包含)
* @return 删除的數量
*/
public long zremrangeByScore(String key, double min, double max) {
Jedis jedis = getJedis();
long s = jedis.zremrangeByScore(key, min, max);
closeJedis(jedis);
return s;
}
/**
* 擷取給定區間的元素，原始按照權重由高到低排序
*
* @param key
* @param start
* @param end
* @return Set<String>
*/
public Set<String> zrevrange(String key, int start, int end) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<String> set = sjedis.zrevrange(key, start, end);
closeJedis(sjedis);
return set;
}
/**
* 擷取給定值在集合中的權重
*
* @param key
* @param memebr
* @return double 權重
*/
public double zscore(String key, String memebr) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Double score = sjedis.zscore(key, memebr);
closeJedis(sjedis);
if (score != null)
return score;
return 0;
}
}
//*******************************************Hash*******************************************//
public class Hash {
/**
* 從hash中删除指定的存儲
*
* @param key
* @param fieid 存儲的名字
* @return 狀态碼，1成功，0失敗
*/
public long hdel(String key, String fieid) {
Jedis jedis = getJedis();
long s = jedis.hdel(key, fieid);
closeJedis(jedis);
return s;
}
public long hdel(String key) {
Jedis jedis = getJedis();
long s = jedis.del(key);
closeJedis(jedis);
return s;
}
/**
* 測試hash中指定的存儲是否存在
*
* @param key
* @param fieid 存儲的名字
* @return 1存在，0不存在
*/
public boolean hexists(String key, String fieid) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
boolean s = sjedis.hexists(key, fieid);
closeJedis(sjedis);
return s;
}
/**
* 傳回hash中指定存儲位置的值
*
* @param key
* @param fieid 存儲的名字
* @return 存儲對應的值
*/
public String hget(String key, String fieid) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
String s = sjedis.hget(key, fieid);
closeJedis(sjedis);
return s;
}
public byte[] hget(byte[] key, byte[] fieid) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
byte[] s = sjedis.hget(key, fieid);
closeJedis(sjedis);
return s;
}
/**
* 以Map的形式傳回hash中的存儲和值
*
* @param key
* @return Map<Strinig,String>
*/
public Map<String, String> hgetAll(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Map<String, String> map = sjedis.hgetAll(key);
closeJedis(sjedis);
return map;
}
/**
* 添加一個對應關系
*
* @param key
* @param fieid
* @param value
* @return 狀态碼 1成功，0失敗，fieid已存在将更新，也傳回0
**/
public long hset(String key, String fieid, String value) {
Jedis jedis = getJedis();
long s = jedis.hset(key, fieid, value);
closeJedis(jedis);
return s;
}
public long hset(String key, String fieid, byte[] value) {
Jedis jedis = getJedis();
long s = jedis.hset(key.getBytes(), fieid.getBytes(), value);
closeJedis(jedis);
return s;
}
/**
* 添加對應關系，隻有在fieid不存在時才執行
*
* @param key
* @param fieid
* @param value
* @return 狀态碼 1成功，0失敗fieid已存
**/
public long hsetnx(String key, String fieid, String value) {
Jedis jedis = getJedis();
long s = jedis.hsetnx(key, fieid, value);
closeJedis(jedis);
return s;
}
/**
* 擷取hash中value的集合
*
* @param key
* @return List<String>
*/
public List<String> hvals(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
List<String> list = sjedis.hvals(key);
closeJedis(sjedis);
return list;
}
/**
* 在指定的存儲位置加上指定的數字，存儲位置的值必須可轉為數字類型
*
* @param key
* @param fieid 存儲位置
* @param value 要增加的值,可以是負數
* @return 增加指定數字後，存儲位置的值
*/
public long hincrby(String key, String fieid, long value) {
Jedis jedis = getJedis();
long s = jedis.hincrBy(key, fieid, value);
closeJedis(jedis);
return s;
}
/**
* 傳回指定hash中的所有存儲名字,類似Map中的keySet方法
*
* @param key
* @return Set<String> 存儲名稱的集合
*/
public Set<String> hkeys(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
Set<String> set = sjedis.hkeys(key);
closeJedis(sjedis);
return set;
}
/**
* 擷取hash中存儲的個數，類似Map中size方法
*
* @param key
* @return long 存儲的個數
*/
public long hlen(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long len = sjedis.hlen(key);
closeJedis(sjedis);
return len;
}
/**
* 根據多個key，擷取對應的value，傳回List,如果指定的key不存在,List對應位置為null
*
* @param key
* @param fieids 存儲位置
* @return List<String>
*/
public List<String> hmget(String key, String... fieids) {
Jedis sjedis = getJedis();
List<String> list = sjedis.hmget(key, fieids);
closeJedis(sjedis);
return list;
}
public List<byte[]> hmget(byte[] key, byte[]... fieids) {
Jedis sjedis = getJedis();
List<byte[]> list = sjedis.hmget(key, fieids);
closeJedis(sjedis);
return list;
}
/**
* 添加對應關系，如果對應關系已存在，則覆寫
*
* @param key
* @param map 對應關系
* @return 狀态，成功傳回OK
*/
public String hmset(String key, Map<String, String> map) {
Jedis jedis = getJedis();
String s = jedis.hmset(key, map);
closeJedis(jedis);
return s;
}
/**
* 添加對應關系，如果對應關系已存在，則覆寫
*
* @param key
* @param map 對應關系
* @return 狀态，成功傳回OK
*/
public String hmset(byte[] key, Map<byte[], byte[]> map) {
Jedis jedis = getJedis();
String s = jedis.hmset(key, map);
closeJedis(jedis);
return s;
}
}
//*******************************************Strings*******************************************//
public class Strings {
/**
* 根據key擷取記錄
*
* @param key
* @return 值
*/
public String get(String key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
String value = sjedis.get(key);
closeJedis(sjedis);
return value;
}
/**
* 根據key擷取記錄
*
* @param key
* @return 值
*/
public byte[] get(byte[] key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
byte[] value = sjedis.get(key);
closeJedis(sjedis);
return value;
}
/**
* 添加有過期時間的記錄
*
* @param key
* @param seconds 過期時間，以秒為機關
* @param value
* @return String 操作狀态
*/
public String setEx(String key, int seconds, String value) {
Jedis jedis = getJedis();
String str = jedis.setex(key, seconds, value);
closeJedis(jedis);
return str;
}
/**
* 添加有過期時間的記錄
*
* @param key
* @param seconds 過期時間，以秒為機關
* @param value
* @return String 操作狀态
*/
public String setEx(byte[] key, int seconds, byte[] value) {
Jedis jedis = getJedis();
String str = jedis.setex(key, seconds, value);
closeJedis(jedis);
return str;
}
/**
* 添加一條記錄，僅當給定的key不存在時才插入
*
* @param key
* @param value
* @return long 狀态碼，1插入成功且key不存在，0未插入，key存在
*/
public long setnx(String key, String value) {
Jedis jedis = getJedis();
long str = jedis.setnx(key, value);
closeJedis(jedis);
return str;
}
/**
* 添加記錄,如果記錄已存在将覆寫原有的value
*
* @param key
* @param value
* @return 狀态碼
*/
public String set(String key, String value) {
return set(SafeEncoder.encode(key), SafeEncoder.encode(value));
}
/**
* 添加記錄,如果記錄已存在将覆寫原有的value
*
* @param key
* @param value
* @return 狀态碼
*/
public String set(String key, byte[] value) {
return set(SafeEncoder.encode(key), value);
}
/**
* 添加記錄,如果記錄已存在将覆寫原有的value
*
* @param key
* @param value
* @return 狀态碼
*/
public String set(byte[] key, byte[] value) {
Jedis jedis = getJedis();
String status = jedis.set(key, value);
closeJedis(jedis);
return status;
}
/**
* 從指定位置開始插入資料，插入的資料會覆寫指定位置以後的資料<br/>
* 例:String str1="123456789";<br/>
* 對str1操作後setRange(key,4,0000)，str1="123400009";
*
* @param key
* @param offset
* @param value
* @return long value的長度
*/
public long setRange(String key, long offset, String value) {
Jedis jedis = getJedis();
long len = jedis.setrange(key, offset, value);
closeJedis(jedis);
return len;
}
/**
* 在指定的key中追加value
*
* @param key
* @param value
* @return long 追加後value的長度
**/
public long append(String key, String value) {
Jedis jedis = getJedis();
long len = jedis.append(key, value);
closeJedis(jedis);
return len;
}
/**
* 将key對應的value減去指定的值，隻有value可以轉為數字時該方法才可用
*
* @param key
* @param number 要減去的值
* @return long 減指定值後的值
*/
public long decrBy(String key, long number) {
Jedis jedis = getJedis();
long len = jedis.decrBy(key, number);
closeJedis(jedis);
return len;
}
/**
* <b>可以作為擷取唯一id的方法</b><br/>
* 将key對應的value加上指定的值，隻有value可以轉為數字時該方法才可用
*
* @param key
* @param number 要減去的值
* @return long 相加後的值
*/
public long incrBy(String key, long number) {
Jedis jedis = getJedis();
long len = jedis.incrBy(key, number);
closeJedis(jedis);
return len;
}
/**
* 對指定key對應的value進行截取
*
* @param key
* @param startOffset 開始位置(包含)
* @param endOffset 結束位置(包含)
* @return String 截取的值
*/
public String getrange(String key, long startOffset, long endOffset) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
String value = sjedis.getrange(key, startOffset, endOffset);
closeJedis(sjedis);
return value;
}
/**
* 擷取并設定指定key對應的value<br/>
* 如果key存在傳回之前的value,否則傳回null
*
* @param key
* @param value
* @return String 原始value或null
*/
public String getSet(String key, String value) {
Jedis jedis = getJedis();
String str = jedis.getSet(key, value);
closeJedis(jedis);
return str;
}
/**
* 批量擷取記錄,如果指定的key不存在傳回List的對應位置将是null
*
* @param keys
* @return List<String> 值得集合
*/
public List<String> mget(String... keys) {
Jedis jedis = getJedis();
List<String> str = jedis.mget(keys);
closeJedis(jedis);
return str;
}
/**
* 批量存儲記錄
*
* @param keysvalues 例:keysvalues="key1","value1","key2","value2";
* @return String 狀态碼
*/
public String mset(String... keysvalues) {
Jedis jedis = getJedis();
String str = jedis.mset(keysvalues);
closeJedis(jedis);
return str;
}
/**
* 擷取key對應的值的長度
*
* @param key
* @return value值得長度
*/
public long strlen(String key) {
Jedis jedis = getJedis();
long len = jedis.strlen(key);
closeJedis(jedis);
return len;
}
}
//*******************************************Lists*******************************************//
public class Lists {
/**
* List長度
*
* @param key
* @return 長度
*/
public long llen(String key) {
return llen(SafeEncoder.encode(key));
}
/**
* List長度
*
* @param key
* @return 長度
*/
public long llen(byte[] key) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
long count = sjedis.llen(key);
closeJedis(sjedis);
return count;
}
/**
* 覆寫操作,将覆寫List中指定位置的值
*
* @param key
* @param index 位置
* @param value 值
* @return 狀态碼
*/
public String lset(byte[] key, int index, byte[] value) {
Jedis jedis = getJedis();
String status = jedis.lset(key, index, value);
closeJedis(jedis);
return status;
}
/**
* 覆寫操作,将覆寫List中指定位置的值
*
* @param
* @param index 位置
* @param value 值
* @return 狀态碼
*/
public String lset(String key, int index, String value) {
return lset(SafeEncoder.encode(key), index,
SafeEncoder.encode(value));
}
/**
* 擷取List中指定位置的值
*
* @param key
* @param index 位置
* @return 值
**/
public String lindex(String key, int index) {
return SafeEncoder.encode(lindex(SafeEncoder.encode(key), index));
}
/**
* 擷取List中指定位置的值
*
* @param key
* @param index 位置
* @return 值
**/
public byte[] lindex(byte[] key, int index) {
Jedis sjedis = getJedis();
byte[] value = sjedis.lindex(key, index);
closeJedis(sjedis);
return value;
}
/**
* 将List中的第一條記錄移出List
*
* @param key
* @return 移出的記錄
*/
public String lpop(String key) {
return SafeEncoder.encode(lpop(SafeEncoder.encode(key)));
}
/**
* 将List中的第一條記錄移出List
*
* @param key
* @return 移出的記錄
*/
public byte[] lpop(byte[] key) {
Jedis jedis = getJedis();
byte[] value = jedis.lpop(key);
closeJedis(jedis);
return value;
}
/**
* 将List中最後第一條記錄移出List
*
* @param key
* @return 移出的記錄
*/
public String rpop(String key) {
Jedis jedis = getJedis();
String value = jedis.rpop(key);
closeJedis(jedis);
return value;
}
/**
* 向List尾部追加記錄
*
* @param key
* @param value
* @return 記錄總數
*/
public long lpush(String key, String value) {
return lpush(SafeEncoder.encode(key), SafeEncoder.encode(value));
}
/**
* 向List頭部追加記錄
*
* @param key
* @param value
* @return 記錄總數
*/
public long rpush(String key, String value) {
Jedis jedis = getJedis();
long count = jedis.rpush(key, value);
closeJedis(jedis);
return count;
}
/**
* 向List頭部追加記錄
*
* @param key
* @param value
* @return 記錄總數
*/
public long rpush(byte[] key, byte[] value) {
Jedis jedis = getJedis();
long count = jedis.rpush(key, value);
closeJedis(jedis);
return count;
}
/**
* 向List中追加記錄
*
* @param key
* @param value
* @return 記錄總數
*/
public long lpush(byte[] key, byte[] value) {
Jedis jedis = getJedis();
long count = jedis.lpush(key, value);
closeJedis(jedis);
return count;
}
/**
* 擷取指定範圍的記錄，可以做為分頁使用
*
* @param key
* @param start
* @param end
* @return List
*/
public List<String> lrange(String key, long start, long end) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
List<String> list = sjedis.lrange(key, start, end);
closeJedis(sjedis);
return list;
}
/**
* 擷取指定範圍的記錄，可以做為分頁使用
*
* @param key
* @param start
* @param end 如果為負數，則尾部開始計算
* @return List
*/
public List<byte[]> lrange(byte[] key, int start, int end) {
//ShardedJedis sjedis = getShardedJedis();
Jedis sjedis = getJedis();
List<byte[]> list = sjedis.lrange(key, start, end);
closeJedis(sjedis);
return list;
}
/**
* 删除List中c條記錄，被删除的記錄值為value
*
* @param key
* @param c 要删除的數量，如果為負數則從List的尾部檢查并删除符合的記錄
* @param value 要比對的值
* @return 删除後的List中的記錄數
*/
public long lrem(byte[] key, int c, byte[] value) {
Jedis jedis = getJedis();
long count = jedis.lrem(key, c, value);
closeJedis(jedis);
return count;
}
/**
* 删除List中c條記錄，被删除的記錄值為value
*
* @param key
* @param c 要删除的數量，如果為負數則從List的尾部檢查并删除符合的記錄
* @param value 要比對的值
* @return 删除後的List中的記錄數
*/
public long lrem(String key, int c, String value) {
return lrem(SafeEncoder.encode(key), c, SafeEncoder.encode(value));
}
/**
* 算是删除吧，隻保留start與end之間的記錄
*
* @param key
* @param start 記錄的開始位置(0表示第一條記錄)
* @param end 記錄的結束位置（如果為-1則表示最後一個，-2，-3以此類推）
* @return 執行狀态碼
*/
public String ltrim(byte[] key, int start, int end) {
Jedis jedis = getJedis();
String str = jedis.ltrim(key, start, end);
closeJedis(jedis);
return str;
}
/**
* 算是删除吧，隻保留start與end之間的記錄
*
* @param key
* @param start 記錄的開始位置(0表示第一條記錄)
* @param end 記錄的結束位置（如果為-1則表示最後一個，-2，-3以此類推）
* @return 執行狀态碼
*/
public String ltrim(String key, int start, int end) {
return ltrim(SafeEncoder.encode(key), start, end);
}
}
public static void main(String[] args) {
Map<String, Object> map = Maps.newHashMap();
map.put("1", 1);
map.put("2", "sfasdfa");
map.put("3", "sfawrwere3fa");
String werwe = RedisUtil.getInstance().strings().set("werwe", JSON.toJSONString(map));
// RedisUtil.getInstance().lists().lpush("BIGDATA:TENANT_GROUP:LIST_COMUPUTE_SQL",
// "{\"groupId\":39,\"version\":\"2020_02_19\",\"sqlRule\":\"{\\\"operator\\\":\\\"and\\\",\\\"conditions\\\":[{\\\"operator\\\":\\\"or\\\",\\\"sqls\\\":[\\\"select user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'consumer_attributes' and second_classification = 'place_dimension' and third_classification = 'all_channel' and fourth_classification = 'place_order_cnt' group by user_no having count(*) <= 3 \\\",\\\"select user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'consumer_attributes' and second_classification = 'finish_dimension' and third_classification = 'carsh' and fourth_classification = 'avg_order_amount' group by user_no having count(*) <= 4 \\\"]}]}\"}"
// );
// RedisUtil.getInstance().lists().lpush("BIGDATA:TENANT_GROUP:LIST_COMUPUTE_SQL",
// "{\"groupId\":33,\"version\":\"2020_02_19\",\"sqlRule\":\"{\\\"operator\\\":\\\"or\\\",\\\"conditions\\\":[{\\\"operator\\\":\\\"or\\\",\\\"sqls\\\":[\\\"select user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'consumer_attributes' and second_classification = 'place_dimension' and third_classification = 'carsh' and fourth_classification = 'place_order_cnt' and options between '2019-12-20' and '2020-02-18' group by user_no having count(*) >= 2 \\\",\\\"select distinct user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'user_preferences' and second_classification = 'site_takeaway' and third_classification = 'shop_preferences' and detail not in ('三品王餐飲') \\\"]}]}\"}"
// );
//
// RedisUtil.getInstance().lists().lpush("BIGDATA:TENANT_GROUP:LIST_COMUPUTE_SQL",
// "{\"groupId\":36,\"version\":\"2020_02_19\",\"sqlRule\":\"{\\\"operator\\\":\\\"and\\\",\\\"conditions\\\":[{\\\"operator\\\":\\\"or\\\",\\\"sqls\\\":[\\\"select user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'consumer_attributes' and second_classification = 'place_dimension' and third_classification = 'all_channel' and fourth_classification = 'place_order_cnt' group by user_no having count(*) <= 5 \\\",\\\"select distinct user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'user_coupon_attributes' and second_classification = 'use' and third_classification = 'used' and fourth_classification = 'coupon_id' and detail not in ('1122160070776991744') \\\"]}]}\"}"
// );
//
// RedisUtil.getInstance().lists().lpush("BIGDATA:TENANT_GROUP:LIST_COMUPUTE_SQL",
// "{\"groupId\":37,\"version\":\"2020_02_19\",\"sqlRule\":\"{\\\"operator\\\":\\\"and\\\",\\\"conditions\\\":[{\\\"operator\\\":\\\"or\\\",\\\"sqls\\\":[\\\"select distinct user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'user_coupon_attributes' and second_classification = 'receive' and third_classification = 'not_received' and fourth_classification = 'coupon_id' and detail not in ('1123072501900648448','1123072740208050176') \\\",\\\"select distinct user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'user_coupon_attributes' and second_classification = 'use' and third_classification = 'used' and fourth_classification = 'coupon_id' and detail not in ('1123068680860237824') \\\"]}]}\"}"
// );
RedisUtil.getInstance().lists().lpush("BIGDATA:TENANT_GROUP:LIST_COMUPUTE_SQL",
"{\"groupId\":38,\"version\":\"2020_02_19\",\"sqlRule\":\"{\\\"operator\\\":\\\"and\\\",\\\"conditions\\\":[{\\\"operator\\\":\\\"and\\\",\\\"sqls\\\":[\\\"select distinct user_no from crgt.ads_user_portrait_vertical_df_cls where first_classification = 'user_coupon_attributes' and second_classification = 'use' and third_classification = 'not_used' and fourth_classification = 'coupon_id' and detail not in ('1123072501900648448','1123072740208050176') \\\"]}]}\"}"
);
String werwe2 = RedisUtil.getInstance().hash().hget("hash.imei", "333");
String s = RedisUtil.getInstance().strings().get("werwe");
System.out.println(s);
System.out.println(werwe2);
}
}
![開學啦｜赢戰開學季，搶跑新學期[圖]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)