记一次找因redis使用不当导致应用卡死bug的过程

top

top -H -p 12798

jstack 12798 |grep 12799的16进制 31ff

/** * Returns a Jedis instance to be used as a Redis connection. The instance can be newly created or retrieved from a * pool. * * @return Jedis instance ready for wrapping into a {@link RedisConnection}. */protected Jedis fetchJedisConnector() {   try {      if (usePool && pool != null) {         return pool.getResource();      }      Jedis jedis = new Jedis(getShardInfo());      // force initialization (see Jedis issue #82)      jedis.connect();      return jedis;   } catch (Exception ex) {      throw new RedisConnectionFailureException("Cannot get Jedis connection", ex);   }}

public T getResource() {  try {    return internalPool.borrowObject();  } catch (Exception e) {    throw new JedisConnectionException("Could not get a resource from the pool", e);  }}

public T borrowObject(long borrowMaxWaitMillis) throws Exception {    this.assertOpen();    AbandonedConfig ac = this.abandonedConfig;    if (ac != null && ac.getRemoveAbandonedOnBorrow() && this.getNumIdle() < 2 && this.getNumActive() > this.getMaxTotal() - 3) {        this.removeAbandoned(ac);    }
    PooledObject p = null;    boolean blockWhenExhausted = this.getBlockWhenExhausted();    long waitTime = 0L;
    while(p == null) {        boolean create = false;        if (blockWhenExhausted) {            p = (PooledObject)this.idleObjects.pollFirst();            if (p == null) {                create = true;                p = this.create();            }
            if (p == null) {                if (borrowMaxWaitMillis < 0L) {                    p = (PooledObject)this.idleObjects.takeFirst();                } else {                    waitTime = System.currentTimeMillis();                    p = (PooledObject)this.idleObjects.pollFirst(borrowMaxWaitMillis, TimeUnit.MILLISECONDS);                    waitTime = System.currentTimeMillis() - waitTime;                }            }
            if (p == null) {                throw new NoSuchElementException("Timeout waiting for idle object");            }

if (p == null) {    if (borrowMaxWaitMillis < 0L) {        p = (PooledObject)this.idleObjects.takeFirst();    } else {        waitTime = System.currentTimeMillis();        p = (PooledObject)this.idleObjects.pollFirst(borrowMaxWaitMillis, TimeUnit.MILLISECONDS);        waitTime = System.currentTimeMillis() - waitTime;    }}

public E takeFirst() throws InterruptedException {    this.lock.lock();
    Object var2;    try {        Object x;        while((x = this.unlinkFirst()) == null) {            this.notEmpty.await();        }
        var2 = x;    } finally {        this.lock.unlock();    }
    return var2;}

thread

thread -428

JedisConnectionFactory jedisConnectionFactory = new JedisConnectionFactory();.......JedisPoolConfig config = new JedisPoolConfig();config.setMaxWaitMillis(2000);.......jedisConnectionFactory.afterPropertiesSet();

org.springframework.data.redis.RedisConnectionFailureException: Cannot get Jedis connection; nested exception is redis.clients.jedis.exceptions.JedisConnectionException: Could not get a resource from the pool    at org.springframework.data.redis.connection.jedis.JedisConnectionFactory.fetchJedisConnector(JedisConnectionFactory.java:140)    at org.springframework.data.redis.connection.jedis.JedisConnectionFactory.getConnection(JedisConnectionFactory.java:229)    at org.springframework.data.redis.connection.jedis.JedisConnectionFactory.getConnection(JedisConnectionFactory.java:57)    at org.springframework.data.redis.core.RedisConnectionUtils.doGetConnection(RedisConnectionUtils.java:128)    at org.springframework.data.redis.core.RedisConnectionUtils.getConnection(RedisConnectionUtils.java:91)    at org.springframework.data.redis.core.RedisConnectionUtils.getConnection(RedisConnectionUtils.java:78)    at org.springframework.data.redis.core.RedisTemplate.execute(RedisTemplate.java:177)    at org.springframework.data.redis.core.RedisTemplate.execute(RedisTemplate.java:152)    at org.springframework.data.redis.core.AbstractOperations.execute(AbstractOperations.java:85)    at org.springframework.data.redis.core.DefaultHashOperations.get(DefaultHashOperations.java:48)

.......Cursor c = stringRedisTemplate.getConnectionFactory().getConnection().scan(options);while (c.hasNext()) {.....,,}

stringRedisTemplate.execute(new RedisCallback() {
     @Override     public Cursor doInRedis(RedisConnection connection) throws DataAccessException {
       return connection.scan(options);     }
});

RedisConnectionUtils.releaseConnection(conn, factory);