View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.commons.pool2.impl;
18  
19  import java.time.Duration;
20  import java.time.Instant;
21  import java.util.ArrayList;
22  import java.util.Deque;
23  import java.util.HashMap;
24  import java.util.Iterator;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Map.Entry;
28  import java.util.NoSuchElementException;
29  import java.util.Objects;
30  import java.util.TreeMap;
31  import java.util.concurrent.ConcurrentHashMap;
32  import java.util.concurrent.TimeUnit;
33  import java.util.concurrent.atomic.AtomicBoolean;
34  import java.util.concurrent.atomic.AtomicInteger;
35  import java.util.concurrent.atomic.AtomicLong;
36  import java.util.concurrent.locks.Lock;
37  import java.util.concurrent.locks.ReadWriteLock;
38  import java.util.concurrent.locks.ReentrantReadWriteLock;
39  import java.util.stream.Collectors;
40  
41  import org.apache.commons.pool2.DestroyMode;
42  import org.apache.commons.pool2.KeyedObjectPool;
43  import org.apache.commons.pool2.KeyedPooledObjectFactory;
44  import org.apache.commons.pool2.PoolUtils;
45  import org.apache.commons.pool2.PooledObject;
46  import org.apache.commons.pool2.PooledObjectState;
47  import org.apache.commons.pool2.SwallowedExceptionListener;
48  import org.apache.commons.pool2.UsageTracking;
49  
50  /**
51   * A configurable {@code KeyedObjectPool} implementation.
52   * <p>
53   * When coupled with the appropriate {@link KeyedPooledObjectFactory},
54   * {@code GenericKeyedObjectPool} provides robust pooling functionality for
55   * keyed objects. A {@code GenericKeyedObjectPool} can be viewed as a map
56   * of sub-pools, keyed on the (unique) key values provided to the
57   * {@link #preparePool preparePool}, {@link #addObject addObject} or
58   * {@link #borrowObject borrowObject} methods. Each time a new key value is
59   * provided to one of these methods, a sub-new pool is created under the given
60   * key to be managed by the containing {@code GenericKeyedObjectPool.}
61   * </p>
62   * <p>
63   * Note that the current implementation uses a ConcurrentHashMap which uses
64   * equals() to compare keys.
65   * This means that distinct instance keys must be distinguishable using equals.
66   * </p>
67   * <p>
68   * Optionally, one may configure the pool to examine and possibly evict objects
69   * as they sit idle in the pool and to ensure that a minimum number of idle
70   * objects is maintained for each key. This is performed by an "idle object
71   * eviction" thread, which runs asynchronously. Caution should be used when
72   * configuring this optional feature. Eviction runs contend with client threads
73   * for access to objects in the pool, so if they run too frequently performance
74   * issues may result.
75   * </p>
76   * <p>
77   * Implementation note: To prevent possible deadlocks, care has been taken to
78   * ensure that no call to a factory method will occur within a synchronization
79   * block. See POOL-125 and DBCP-44 for more information.
80   * </p>
81   * <p>
82   * This class is intended to be thread-safe.
83   * </p>
84   *
85   * @see GenericObjectPool
86   * @param <K> The type of keys maintained by this pool.
87   * @param <T> Type of element pooled in this pool.
88   * @since 2.0
89   */
90  public class GenericKeyedObjectPool<K, T> extends BaseGenericObjectPool<T>
91          implements KeyedObjectPool<K, T>, GenericKeyedObjectPoolMXBean<K>, UsageTracking<T> {
92  
93      /**
94       * Maintains information on the per key queue for a given key.
95       *
96       * @param <S> type of objects in the pool
97       */
98      private static final class ObjectDeque<S> {
99  
100         private final LinkedBlockingDeque<PooledObject<S>> idleObjects;
101 
102         /*
103          * Number of instances created - number destroyed.
104          * Invariant: createCount <= maxTotalPerKey
105          */
106         private final AtomicInteger createCount = new AtomicInteger();
107 
108         private long makeObjectCount;
109         private final Object makeObjectCountLock = new Object();
110 
111         /*
112          * The map is keyed on pooled instances, wrapped to ensure that
113          * they work properly as keys.
114          */
115         private final Map<IdentityWrapper<S>, PooledObject<S>> allObjects =
116                 new ConcurrentHashMap<>();
117 
118         /*
119          * Number of threads with registered interest in this key.
120          * register(K) increments this counter and deRegister(K) decrements it.
121          * Invariant: empty keyed pool will not be dropped unless numInterested
122          *            is 0.
123          */
124         private final AtomicLong numInterested = new AtomicLong();
125 
126         /**
127          * Constructs a new ObjectDeque with the given fairness policy.
128          * @param fairness true means client threads waiting to borrow / return instances
129          * will be served as if waiting in a FIFO queue.
130          */
131         public ObjectDeque(final boolean fairness) {
132             idleObjects = new LinkedBlockingDeque<>(fairness);
133         }
134 
135         /**
136          * Gets all the objects for the current key.
137          *
138          * @return All the objects
139          */
140         public Map<IdentityWrapper<S>, PooledObject<S>> getAllObjects() {
141             return allObjects;
142         }
143 
144         /**
145          * Gets the number of instances created - number destroyed.
146          * Should always be less than or equal to maxTotalPerKey.
147          *
148          * @return The net instance addition count for this deque
149          */
150         public AtomicInteger getCreateCount() {
151             return createCount;
152         }
153 
154         /**
155          * Gets the idle objects for the current key.
156          *
157          * @return The idle objects
158          */
159         public LinkedBlockingDeque<PooledObject<S>> getIdleObjects() {
160             return idleObjects;
161         }
162 
163         /**
164          * Gets the number of threads with an interest registered in this key.
165          *
166          * @return The number of threads with a registered interest in this key
167          */
168         public AtomicLong getNumInterested() {
169             return numInterested;
170         }
171 
172         @Override
173         public String toString() {
174             final StringBuilder builder = new StringBuilder();
175             builder.append("ObjectDeque [idleObjects=");
176             builder.append(idleObjects);
177             builder.append(", createCount=");
178             builder.append(createCount);
179             builder.append(", allObjects=");
180             builder.append(allObjects);
181             builder.append(", numInterested=");
182             builder.append(numInterested);
183             builder.append("]");
184             return builder.toString();
185         }
186 
187     }
188 
189     private static final Integer ZERO = Integer.valueOf(0);
190 
191     // JMX specific attributes
192     private static final String ONAME_BASE =
193             "org.apache.commons.pool2:type=GenericKeyedObjectPool,name=";
194 
195     private volatile int maxIdlePerKey =
196             GenericKeyedObjectPoolConfig.DEFAULT_MAX_IDLE_PER_KEY;
197 
198     private volatile int minIdlePerKey =
199             GenericKeyedObjectPoolConfig.DEFAULT_MIN_IDLE_PER_KEY;
200 
201     private volatile int maxTotalPerKey =
202             GenericKeyedObjectPoolConfig.DEFAULT_MAX_TOTAL_PER_KEY;
203 
204     private final KeyedPooledObjectFactory<K, T> factory;
205 
206     private final boolean fairness;
207 
208     /*
209      * My hash of sub-pools (ObjectQueue). The list of keys <strong>must</strong> be kept
210      * in step with {@link #poolKeyList} using {@link #keyLock} to ensure any
211      * changes to the list of current keys is made in a thread-safe manner.
212      */
213     private final Map<K, ObjectDeque<T>> poolMap =
214             new ConcurrentHashMap<>(); // @GuardedBy("keyLock") for write access (and some read access)
215 
216     /*
217      * List of pool keys - used to control eviction order. The list of keys
218      * <strong>must</strong> be kept in step with {@link #poolMap} using {@link #keyLock}
219      * to ensure any changes to the list of current keys is made in a
220      * thread-safe manner.
221      */
222     private final ArrayList<K> poolKeyList = new ArrayList<>(); // @GuardedBy("keyLock")
223 
224     private final ReadWriteLock keyLock = new ReentrantReadWriteLock(true);
225 
226     /*
227      * The combined count of the currently active objects for all keys and those
228      * in the process of being created. Under load, it may exceed
229      * {@link #maxTotal} but there will never be more than {@link #maxTotal}
230      * created at any one time.
231      */
232     private final AtomicInteger numTotal = new AtomicInteger();
233 
234     private Iterator<K> evictionKeyIterator; // @GuardedBy("evictionLock")
235 
236     private K evictionKey; // @GuardedBy("evictionLock")
237 
238     /**
239      * Constructs a new {@code GenericKeyedObjectPool} using defaults from
240      * {@link GenericKeyedObjectPoolConfig}.
241      * @param factory the factory to be used to create entries
242      */
243     public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory) {
244         this(factory, new GenericKeyedObjectPoolConfig<>());
245     }
246 
247     /**
248      * Constructs a new {@code GenericKeyedObjectPool} using a specific
249      * configuration.
250      *
251      * @param factory the factory to be used to create entries
252      * @param config    The configuration to use for this pool instance. The
253      *                  configuration is used by value. Subsequent changes to
254      *                  the configuration object will not be reflected in the
255      *                  pool.
256      */
257     public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
258             final GenericKeyedObjectPoolConfig<T> config) {
259 
260         super(config, ONAME_BASE, config.getJmxNamePrefix());
261 
262         if (factory == null) {
263             jmxUnregister(); // tidy up
264             throw new IllegalArgumentException("Factory may not be null");
265         }
266         this.factory = factory;
267         this.fairness = config.getFairness();
268 
269         setConfig(config);
270     }
271 
272     /**
273      * Creates a new {@code GenericKeyedObjectPool} that tracks and destroys
274      * objects that are checked out, but never returned to the pool.
275      *
276      * @param factory   The object factory to be used to create object instances
277      *                  used by this pool
278      * @param config    The base pool configuration to use for this pool instance.
279      *                  The configuration is used by value. Subsequent changes to
280      *                  the configuration object will not be reflected in the
281      *                  pool.
282      * @param abandonedConfig  Configuration for abandoned object identification
283      *                         and removal.  The configuration is used by value.
284      * @since 2.10.0
285      */
286     public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
287             final GenericKeyedObjectPoolConfig<T> config, final AbandonedConfig abandonedConfig) {
288         this(factory, config);
289         setAbandonedConfig(abandonedConfig);
290     }
291 
292     /**
293      * Add an object to the set of idle objects for a given key.
294      * If the object is null this is a no-op.
295      *
296      * @param key The key to associate with the idle object
297      * @param p The wrapped object to add.
298      * @throws Exception If the associated factory fails to passivate the object
299      */
300     private void addIdleObject(final K key, final PooledObject<T> p) throws Exception {
301         if (!PooledObject.isNull(p)) {
302             factory.passivateObject(key, p);
303             final LinkedBlockingDeque<PooledObject<T>> idleObjects = poolMap.get(key).getIdleObjects();
304             if (getLifo()) {
305                 idleObjects.addFirst(p);
306             } else {
307                 idleObjects.addLast(p);
308             }
309         }
310     }
311 
312     /**
313      * Create an object using the {@link KeyedPooledObjectFactory#makeObject
314      * factory}, passivate it, and then place it in the idle object pool.
315      * {@code addObject} is useful for "pre-loading" a pool with idle
316      * objects.
317      * <p>
318      * If there is no capacity available to add to the pool under the given key,
319      * this is a no-op (no exception, no impact to the pool).
320      * </p>
321      * <p>
322      * If the factory returns null when creating an instance,
323      * a {@code NullPointerException} is thrown.
324      * </p>
325      *
326      * @param key the key a new instance should be added to
327      * @throws Exception when {@link KeyedPooledObjectFactory#makeObject}
328      *                   fails.
329      */
330     @Override
331     public void addObject(final K key) throws Exception {
332         assertOpen();
333         register(key);
334         try {
335             addIdleObject(key, create(key));
336         } finally {
337             deregister(key);
338         }
339     }
340 
341     /**
342      * Equivalent to <code>{@link #borrowObject(Object, long) borrowObject}(key,
343      * {@link #getMaxWaitDuration()})</code>.
344      *
345      * {@inheritDoc}
346      */
347     @Override
348     public T borrowObject(final K key) throws Exception {
349         return borrowObject(key, getMaxWaitDuration().toMillis());
350     }
351 
352     /**
353      * Borrows an object from the sub-pool associated with the given key using
354      * the specified waiting time which only applies if
355      * {@link #getBlockWhenExhausted()} is true.
356      * <p>
357      * If there is one or more idle instances available in the sub-pool
358      * associated with the given key, then an idle instance will be selected
359      * based on the value of {@link #getLifo()}, activated and returned.  If
360      * activation fails, or {@link #getTestOnBorrow() testOnBorrow} is set to
361      * {@code true} and validation fails, the instance is destroyed and the
362      * next available instance is examined.  This continues until either a valid
363      * instance is returned or there are no more idle instances available.
364      * </p>
365      * <p>
366      * If there are no idle instances available in the sub-pool associated with
367      * the given key, behavior depends on the {@link #getMaxTotalPerKey()
368      * maxTotalPerKey}, {@link #getMaxTotal() maxTotal}, and (if applicable)
369      * {@link #getBlockWhenExhausted()} and the value passed in to the
370      * {@code borrowMaxWaitMillis} parameter. If the number of instances checked
371      * out from the sub-pool under the given key is less than
372      * {@code maxTotalPerKey} and the total number of instances in
373      * circulation (under all keys) is less than {@code maxTotal}, a new
374      * instance is created, activated and (if applicable) validated and returned
375      * to the caller. If validation fails, a {@code NoSuchElementException}
376      * will be thrown. If the factory returns null when creating an instance,
377      * a {@code NullPointerException} is thrown.
378      * </p>
379      * <p>
380      * If the associated sub-pool is exhausted (no available idle instances and
381      * no capacity to create new ones), this method will either block
382      * ({@link #getBlockWhenExhausted()} is true) or throw a
383      * {@code NoSuchElementException}
384      * ({@link #getBlockWhenExhausted()} is false).
385      * The length of time that this method will block when
386      * {@link #getBlockWhenExhausted()} is true is determined by the value
387      * passed in to the {@code borrowMaxWait} parameter.
388      * </p>
389      * <p>
390      * When {@code maxTotal} is set to a positive value and this method is
391      * invoked when at the limit with no idle instances available under the requested
392      * key, an attempt is made to create room by clearing the oldest 15% of the
393      * elements from the keyed sub-pools.
394      * </p>
395      * <p>
396      * When the pool is exhausted, multiple calling threads may be
397      * simultaneously blocked waiting for instances to become available. A
398      * "fairness" algorithm has been implemented to ensure that threads receive
399      * available instances in request arrival order.
400      * </p>
401      *
402      * @param key pool key
403      * @param borrowMaxWaitMillis The time to wait in milliseconds for an object
404      *                            to become available
405      *
406      * @return object instance from the keyed pool
407      * @throws NoSuchElementException if a keyed object instance cannot be
408      *                                returned because the pool is exhausted.
409      *
410      * @throws Exception if a keyed object instance cannot be returned due to an
411      *                   error
412      */
413     public T borrowObject(final K key, final long borrowMaxWaitMillis) throws Exception {
414         assertOpen();
415 
416         final AbandonedConfig ac = this.abandonedConfig;
417         if (ac != null && ac.getRemoveAbandonedOnBorrow() && getNumIdle() < 2 &&
418                 getNumActive() > getMaxTotal() - 3) {
419             removeAbandoned(ac);
420         }
421 
422         PooledObject<T> p = null;
423 
424         // Get local copy of current config so it is consistent for entire
425         // method execution
426         final boolean blockWhenExhausted = getBlockWhenExhausted();
427 
428         boolean create;
429         final Instant waitTime = Instant.now();
430         final ObjectDeque<T> objectDeque = register(key);
431 
432         try {
433             while (p == null) {
434                 create = false;
435                 p = objectDeque.getIdleObjects().pollFirst();
436                 if (p == null) {
437                     p = create(key);
438                     if (!PooledObject.isNull(p)) {
439                         create = true;
440                     }
441                 }
442                 if (blockWhenExhausted) {
443                     if (PooledObject.isNull(p)) {
444                         p = borrowMaxWaitMillis < 0 ? objectDeque.getIdleObjects().takeFirst():
445                                 objectDeque.getIdleObjects().pollFirst(borrowMaxWaitMillis, TimeUnit.MILLISECONDS);
446                     }
447                     if (PooledObject.isNull(p)) {
448                         throw new NoSuchElementException(appendStats(
449                                 "Timeout waiting for idle object, borrowMaxWaitMillis=" + borrowMaxWaitMillis));
450                     }
451                 } else if (PooledObject.isNull(p)) {
452                     throw new NoSuchElementException(appendStats("Pool exhausted"));
453                 }
454                 if (!p.allocate()) {
455                     p = null;
456                 }
457 
458                 if (!PooledObject.isNull(p)) {
459                     try {
460                         factory.activateObject(key, p);
461                     } catch (final Exception e) {
462                         try {
463                             destroy(key, p, true, DestroyMode.NORMAL);
464                         } catch (final Exception ignored) {
465                             // ignored - activation failure is more important
466                         }
467                         p = null;
468                         if (create) {
469                             final NoSuchElementException nsee = new NoSuchElementException(appendStats("Unable to activate object"));
470                             nsee.initCause(e);
471                             throw nsee;
472                         }
473                     }
474                     if (!PooledObject.isNull(p) && getTestOnBorrow()) {
475                         boolean validate = false;
476                         Throwable validationThrowable = null;
477                         try {
478                             validate = factory.validateObject(key, p);
479                         } catch (final Throwable t) {
480                             PoolUtils.checkRethrow(t);
481                             validationThrowable = t;
482                         }
483                         if (!validate) {
484                             try {
485                                 destroy(key, p, true, DestroyMode.NORMAL);
486                                 destroyedByBorrowValidationCount.incrementAndGet();
487                             } catch (final Exception ignored) {
488                                 // ignored - validation failure is more important
489                             }
490                             p = null;
491                             if (create) {
492                                 final NoSuchElementException nsee = new NoSuchElementException(
493                                         appendStats("Unable to validate object"));
494                                 nsee.initCause(validationThrowable);
495                                 throw nsee;
496                             }
497                         }
498                     }
499                 }
500             }
501         } finally {
502             deregister(key);
503         }
504 
505         updateStatsBorrow(p, Duration.between(waitTime, Instant.now()));
506 
507         return p.getObject();
508     }
509 
510     /**
511      * Calculate the number of objects that need to be created to attempt to
512      * maintain the minimum number of idle objects while not exceeded the limits
513      * on the maximum number of objects either per key or totally.
514      *
515      * @param objectDeque   The set of objects to check
516      * @return The number of new objects to create
517      */
518     private int calculateDeficit(final ObjectDeque<T> objectDeque) {
519 
520         if (objectDeque == null) {
521             return getMinIdlePerKey();
522         }
523 
524         // Used more than once so keep a local copy so the value is consistent
525         final int maxTotal = getMaxTotal();
526         final int maxTotalPerKeySave = getMaxTotalPerKey();
527 
528         // Calculate no of objects needed to be created, in order to have
529         // the number of pooled objects < maxTotalPerKey();
530         int objectDefecit = getMinIdlePerKey() - objectDeque.getIdleObjects().size();
531         if (maxTotalPerKeySave > 0) {
532             final int growLimit = Math.max(0,
533                     maxTotalPerKeySave - objectDeque.getIdleObjects().size());
534             objectDefecit = Math.min(objectDefecit, growLimit);
535         }
536 
537         // Take the maxTotal limit into account
538         if (maxTotal > 0) {
539             final int growLimit = Math.max(0, maxTotal - getNumActive() - getNumIdle());
540             objectDefecit = Math.min(objectDefecit, growLimit);
541         }
542 
543         return objectDefecit;
544     }
545 
546     /**
547      * Clears any objects sitting idle in the pool by removing them from the
548      * idle instance sub-pools and then invoking the configured
549      * PoolableObjectFactory's
550      * {@link KeyedPooledObjectFactory#destroyObject(Object, PooledObject)}
551      * method on each idle instance.
552      * <p>
553      * Implementation notes:
554      * <ul>
555      * <li>This method does not destroy or effect in any way instances that are
556      * checked out when it is invoked.</li>
557      * <li>Invoking this method does not prevent objects being returned to the
558      * idle instance pool, even during its execution. Additional instances may
559      * be returned while removed items are being destroyed.</li>
560      * <li>Exceptions encountered destroying idle instances are swallowed
561      * but notified via a {@link SwallowedExceptionListener}.</li>
562      * </ul>
563      */
564     @Override
565     public void clear() {
566         poolMap.keySet().forEach(key -> clear(key, false));
567     }
568 
569     /**
570      * Clears the specified sub-pool, removing all pooled instances
571      * corresponding to the given {@code key}. Exceptions encountered
572      * destroying idle instances are swallowed but notified via a
573      * {@link SwallowedExceptionListener}.
574      * <p>
575      * If there are clients waiting to borrow objects, this method will
576      * attempt to reuse the capacity freed by this operation, adding
577      * instances to the most loaded keyed pools.  To avoid triggering
578      * possible object creation, use {@link #clear(Object, boolean)}.
579      *
580      * @param key the key to clear
581      */
582     @Override
583     public void clear(final K key) {
584         clear(key, true);
585     }
586 
587     /**
588      * Clears the specified sub-pool, removing all pooled instances
589      * corresponding to the given {@code key}. Exceptions encountered
590      * destroying idle instances are swallowed but notified via a
591      * {@link SwallowedExceptionListener}.
592      * <p>
593      * If reuseCapacity is true and there are clients waiting to
594      * borrow objects, this method will attempt to reuse the capacity freed
595      * by this operation, adding instances to the most loaded keyed pools.
596      * </p>
597      *
598      * @param key the key to clear
599      * @param reuseCapacity whether or not to reuse freed capacity
600      * @since 2.12.0
601      */
602     public void clear(final K key, final boolean reuseCapacity) {
603         // Return immediately if there is no pool under this key.
604         if (!poolMap.containsKey(key)) {
605             return;
606         }
607         final ObjectDeque<T> objectDeque = register(key);
608         int freedCapacity = 0;
609         try {
610             final LinkedBlockingDeque<PooledObject<T>> idleObjects = objectDeque.getIdleObjects();
611             PooledObject<T> p = idleObjects.poll();
612             while (p != null) {
613                 try {
614                     if (destroy(key, p, true, DestroyMode.NORMAL)) {
615                         freedCapacity++;
616                     }
617                 } catch (final Exception e) {
618                     swallowException(e);
619                 }
620                 p = idleObjects.poll();
621             }
622         } finally {
623             deregister(key);
624         }
625         if (reuseCapacity) {
626             reuseCapacity(freedCapacity);
627         }
628     }
629 
630     /**
631      * Clears oldest 15% of objects in pool.  The method sorts the objects into
632      * a TreeMap and then iterates the first 15% for removal.
633      */
634     public void clearOldest() {
635 
636         // build sorted map of idle objects
637         final TreeMap<PooledObject<T>, K> map = new TreeMap<>();
638 
639         // Each item into the map using the PooledObject object as the
640         // key. It then gets sorted based on the idle time
641         poolMap.forEach((key, value) -> value.getIdleObjects().forEach(p -> map.put(p, key)));
642 
643         // Now iterate created map and kill the first 15% plus one to account
644         // for zero
645         int itemsToRemove = (int) (map.size() * 0.15) + 1;
646         final Iterator<Entry<PooledObject<T>, K>> iter = map.entrySet().iterator();
647 
648         while (iter.hasNext() && itemsToRemove > 0) {
649             final Entry<PooledObject<T>, K> entry = iter.next();
650             // kind of backwards on naming. In the map, each key is the
651             // PooledObject because it has the ordering with the timestamp
652             // value. Each value that the key references is the key of the
653             // list it belongs to.
654             final K key = entry.getValue();
655             final PooledObject<T> p = entry.getKey();
656             // Assume the destruction succeeds
657             boolean destroyed = true;
658             try {
659                 destroyed = destroy(key, p, false, DestroyMode.NORMAL);
660             } catch (final Exception e) {
661                 swallowException(e);
662             }
663             if (destroyed) {
664                 itemsToRemove--;
665             }
666         }
667     }
668 
669     /**
670      * Closes the keyed object pool. Once the pool is closed,
671      * {@link #borrowObject(Object)} will fail with IllegalStateException, but
672      * {@link #returnObject(Object, Object)} and
673      * {@link #invalidateObject(Object, Object)} will continue to work, with
674      * returned objects destroyed on return.
675      * <p>
676      * Destroys idle instances in the pool by invoking {@link #clear()}.
677      * </p>
678      */
679     @Override
680     public void close() {
681         if (isClosed()) {
682             return;
683         }
684 
685         synchronized (closeLock) {
686             if (isClosed()) {
687                 return;
688             }
689 
690             // Stop the evictor before the pool is closed since evict() calls
691             // assertOpen()
692             stopEvictor();
693 
694             closed = true;
695             // This clear removes any idle objects
696             clear();
697 
698             jmxUnregister();
699 
700             // Release any threads that were waiting for an object
701             poolMap.values().forEach(e -> e.getIdleObjects().interuptTakeWaiters());
702             // This clear cleans up the keys now any waiting threads have been
703             // interrupted
704             clear();
705         }
706     }
707 
708     /**
709      * Creates a new pooled object or null.
710      *
711      * @param key Key associated with new pooled object.
712      * @return The new, wrapped pooled object. May return null.
713      * @throws Exception If the objection creation fails.
714      */
715     private PooledObject<T> create(final K key) throws Exception {
716         int maxTotalPerKeySave = getMaxTotalPerKey(); // Per key
717         if (maxTotalPerKeySave < 0) {
718             maxTotalPerKeySave = Integer.MAX_VALUE;
719         }
720         final int maxTotal = getMaxTotal();   // All keys
721 
722         final ObjectDeque<T> objectDeque = poolMap.get(key);
723 
724         // Check against the overall limit
725         boolean loop = true;
726 
727         while (loop) {
728             final int newNumTotal = numTotal.incrementAndGet();
729             if (maxTotal > -1 && newNumTotal > maxTotal) {
730                 numTotal.decrementAndGet();
731                 if (getNumIdle() == 0) {
732                     return null;
733                 }
734                 clearOldest();
735             } else {
736                 loop = false;
737             }
738         }
739 
740         // Flag that indicates if create should:
741         // - TRUE:  call the factory to create an object
742         // - FALSE: return null
743         // - null:  loop and re-test the condition that determines whether to
744         //          call the factory
745         Boolean create = null;
746         while (create == null) {
747             synchronized (objectDeque.makeObjectCountLock) {
748                 final long newCreateCount = objectDeque.getCreateCount().incrementAndGet();
749                 // Check against the per key limit
750                 if (newCreateCount > maxTotalPerKeySave) {
751                     // The key is currently at capacity or in the process of
752                     // making enough new objects to take it to capacity.
753                     objectDeque.getCreateCount().decrementAndGet();
754                     if (objectDeque.makeObjectCount == 0) {
755                         // There are no makeObject() calls in progress for this
756                         // key so the key is at capacity. Do not attempt to
757                         // create a new object. Return and wait for an object to
758                         // be returned.
759                         create = Boolean.FALSE;
760                     } else {
761                         // There are makeObject() calls in progress that might
762                         // bring the pool to capacity. Those calls might also
763                         // fail so wait until they complete and then re-test if
764                         // the pool is at capacity or not.
765                         objectDeque.makeObjectCountLock.wait();
766                     }
767                 } else {
768                     // The pool is not at capacity. Create a new object.
769                     objectDeque.makeObjectCount++;
770                     create = Boolean.TRUE;
771                 }
772             }
773         }
774 
775         if (!create.booleanValue()) {
776             numTotal.decrementAndGet();
777             return null;
778         }
779 
780         PooledObject<T> p = null;
781         try {
782             p = factory.makeObject(key);
783             if (PooledObject.isNull(p)) {
784                 numTotal.decrementAndGet();
785                 objectDeque.getCreateCount().decrementAndGet();
786                 throw new NullPointerException(String.format("%s.makeObject() = null", factory.getClass().getSimpleName()));
787             }
788             if (getTestOnCreate() && !factory.validateObject(key, p)) {
789                 numTotal.decrementAndGet();
790                 objectDeque.getCreateCount().decrementAndGet();
791                 return null;
792             }
793         } catch (final Exception e) {
794             numTotal.decrementAndGet();
795             objectDeque.getCreateCount().decrementAndGet();
796             throw e;
797         } finally {
798             synchronized (objectDeque.makeObjectCountLock) {
799                 objectDeque.makeObjectCount--;
800                 objectDeque.makeObjectCountLock.notifyAll();
801             }
802         }
803 
804         final AbandonedConfig ac = this.abandonedConfig;
805         if (ac != null && ac.getLogAbandoned()) {
806             p.setLogAbandoned(true);
807             p.setRequireFullStackTrace(ac.getRequireFullStackTrace());
808         }
809 
810         createdCount.incrementAndGet();
811         objectDeque.getAllObjects().put(new IdentityWrapper<>(p.getObject()), p);
812         return p;
813     }
814 
815     /**
816      * De-register the use of a key by an object.
817      * <p>
818      * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
819      * </p>
820      *
821      * @param k The key to de-register
822      */
823     private void deregister(final K k) {
824         Lock lock = keyLock.readLock();
825         try {
826             lock.lock();
827             ObjectDeque<T> objectDeque = poolMap.get(k);
828             if (objectDeque == null) {
829                 throw new IllegalStateException("Attempt to de-register a key for a non-existent pool");
830             }
831             final long numInterested = objectDeque.getNumInterested().decrementAndGet();
832             if (numInterested < 0) {
833                 throw new IllegalStateException("numInterested count for key " + k + " is less than zero");
834             }
835             if (numInterested == 0 && objectDeque.getCreateCount().get() == 0) {
836                 // Potential to remove key
837                 // Upgrade to write lock
838                 lock.unlock();
839                 lock = keyLock.writeLock();
840                 lock.lock();
841                 // Pool may have changed since we released the read lock
842                 // numInterested decrement could lead to removal while waiting for write lock
843                 objectDeque = poolMap.get(k);
844                 if (null != objectDeque && objectDeque.getNumInterested().get() == 0 && objectDeque.getCreateCount().get() == 0) {
845                     // NOTE: Keys must always be removed from both poolMap and
846                     // poolKeyList at the same time while protected by
847                     // keyLock.writeLock()
848                     poolMap.remove(k);
849                     poolKeyList.remove(k);
850                 }
851             }
852         } finally {
853             lock.unlock();
854         }
855     }
856 
857     /**
858      * Destroy the wrapped, pooled object.
859      *
860      * @param key The key associated with the object to destroy.
861      * @param toDestroy The wrapped object to be destroyed
862      * @param always Should the object be destroyed even if it is not currently
863      *               in the set of idle objects for the given key
864      * @param destroyMode DestroyMode context provided to the factory
865      * @return {@code true} if the object was destroyed, otherwise {@code false}
866      * @throws Exception If the object destruction failed
867      */
868     private boolean destroy(final K key, final PooledObject<T> toDestroy, final boolean always, final DestroyMode destroyMode) throws Exception {
869 
870         final ObjectDeque<T> objectDeque = register(key);
871 
872         try {
873             boolean isIdle;
874             synchronized (toDestroy) {
875                 // Check idle state directly
876                 isIdle = toDestroy.getState().equals(PooledObjectState.IDLE);
877                 // If idle, not under eviction test, or always is true, remove instance,
878                 // updating isIdle if instance is found in idle objects
879                 if (isIdle || always) {
880                     isIdle = objectDeque.getIdleObjects().remove(toDestroy);
881                 }
882             }
883             if (isIdle || always) {
884                 objectDeque.getAllObjects().remove(new IdentityWrapper<>(toDestroy.getObject()));
885                 toDestroy.invalidate();
886 
887                 try {
888                     factory.destroyObject(key, toDestroy, destroyMode);
889                 } finally {
890                     objectDeque.getCreateCount().decrementAndGet();
891                     destroyedCount.incrementAndGet();
892                     numTotal.decrementAndGet();
893                 }
894                 return true;
895             }
896             return false;
897         } finally {
898             deregister(key);
899         }
900     }
901 
902     @Override
903     void ensureMinIdle() throws Exception {
904         final int minIdlePerKeySave = getMinIdlePerKey();
905         if (minIdlePerKeySave < 1) {
906             return;
907         }
908 
909         for (final K k : poolMap.keySet()) {
910             ensureMinIdle(k);
911         }
912     }
913 
914     /**
915      * Try to ensure that the configured number of minimum idle objects is available in
916      * the pool for the given key.
917      * <p>
918      * If there is no capacity available to add to the pool, this is a no-op
919      * (no exception, no impact to the pool).
920      * </p>
921      * <p>
922      * If the factory returns null when creating an object, a {@code NullPointerException}
923      * is thrown.
924      * </p>
925      *
926      * @param key The key to check for idle objects
927      * @throws Exception If a new object is required and cannot be created
928      */
929     private void ensureMinIdle(final K key) throws Exception {
930         // Calculate current pool objects
931         ObjectDeque<T> objectDeque = poolMap.get(key);
932 
933         // objectDeque == null is OK here. It is handled correctly by both
934         // methods called below.
935 
936         // this method isn't synchronized so the
937         // calculateDeficit is done at the beginning
938         // as a loop limit and a second time inside the loop
939         // to stop when another thread already returned the
940         // needed objects
941         final int deficit = calculateDeficit(objectDeque);
942 
943         for (int i = 0; i < deficit && calculateDeficit(objectDeque) > 0; i++) {
944             addObject(key);
945             // If objectDeque was null, it won't be any more. Obtain a reference
946             // to it so the deficit can be correctly calculated. It needs to
947             // take account of objects created in other threads.
948             if (objectDeque == null) {
949                 objectDeque = poolMap.get(key);
950             }
951         }
952     }
953 
954     /**
955      * {@inheritDoc}
956      * <p>
957      * Successive activations of this method examine objects in keyed sub-pools
958      * in sequence, cycling through the keys and examining objects in
959      * oldest-to-youngest order within the keyed sub-pools.
960      * </p>
961      */
962     @Override
963     public void evict() throws Exception {
964         assertOpen();
965 
966         if (getNumIdle() > 0) {
967 
968             PooledObject<T> underTest = null;
969             final EvictionPolicy<T> evictionPolicy = getEvictionPolicy();
970 
971             synchronized (evictionLock) {
972                 final EvictionConfig evictionConfig = new EvictionConfig(
973                         getMinEvictableIdleDuration(),
974                         getSoftMinEvictableIdleDuration(),
975                         getMinIdlePerKey());
976 
977                 final boolean testWhileIdle = getTestWhileIdle();
978 
979                 for (int i = 0, m = getNumTests(); i < m; i++) {
980                     if (evictionIterator == null || !evictionIterator.hasNext()) {
981                         if (evictionKeyIterator == null ||
982                                 !evictionKeyIterator.hasNext()) {
983                             final List<K> keyCopy = new ArrayList<>();
984                             final Lock readLock = keyLock.readLock();
985                             readLock.lock();
986                             try {
987                                 keyCopy.addAll(poolKeyList);
988                             } finally {
989                                 readLock.unlock();
990                             }
991                             evictionKeyIterator = keyCopy.iterator();
992                         }
993                         while (evictionKeyIterator.hasNext()) {
994                             evictionKey = evictionKeyIterator.next();
995                             final ObjectDeque<T> objectDeque = poolMap.get(evictionKey);
996                             if (objectDeque == null) {
997                                 continue;
998                             }
999 
1000                             final Deque<PooledObject<T>> idleObjects = objectDeque.getIdleObjects();
1001                             evictionIterator = new EvictionIterator(idleObjects);
1002                             if (evictionIterator.hasNext()) {
1003                                 break;
1004                             }
1005                             evictionIterator = null;
1006                         }
1007                     }
1008                     if (evictionIterator == null) {
1009                         // Pools exhausted
1010                         return;
1011                     }
1012                     final Deque<PooledObject<T>> idleObjects;
1013                     try {
1014                         underTest = evictionIterator.next();
1015                         idleObjects = evictionIterator.getIdleObjects();
1016                     } catch (final NoSuchElementException nsee) {
1017                         // Object was borrowed in another thread
1018                         // Don't count this as an eviction test so reduce i;
1019                         i--;
1020                         evictionIterator = null;
1021                         continue;
1022                     }
1023 
1024                     if (!underTest.startEvictionTest()) {
1025                         // Object was borrowed in another thread
1026                         // Don't count this as an eviction test so reduce i;
1027                         i--;
1028                         continue;
1029                     }
1030 
1031                     // User provided eviction policy could throw all sorts of
1032                     // crazy exceptions. Protect against such an exception
1033                     // killing the eviction thread.
1034                     boolean evict;
1035                     try {
1036                         evict = evictionPolicy.evict(evictionConfig, underTest,
1037                                 poolMap.get(evictionKey).getIdleObjects().size());
1038                     } catch (final Throwable t) {
1039                         // Slightly convoluted as SwallowedExceptionListener
1040                         // uses Exception rather than Throwable
1041                         PoolUtils.checkRethrow(t);
1042                         swallowException(new Exception(t));
1043                         // Don't evict on error conditions
1044                         evict = false;
1045                     }
1046 
1047                     if (evict) {
1048                         destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1049                         destroyedByEvictorCount.incrementAndGet();
1050                     } else {
1051                         if (testWhileIdle) {
1052                             boolean active = false;
1053                             try {
1054                                 factory.activateObject(evictionKey, underTest);
1055                                 active = true;
1056                             } catch (final Exception e) {
1057                                 destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1058                                 destroyedByEvictorCount.incrementAndGet();
1059                             }
1060                             if (active) {
1061                                 boolean validate = false;
1062                                 Throwable validationThrowable = null;
1063                                 try {
1064                                     validate = factory.validateObject(evictionKey, underTest);
1065                                 } catch (final Throwable t) {
1066                                     PoolUtils.checkRethrow(t);
1067                                     validationThrowable = t;
1068                                 }
1069                                 if (!validate) {
1070                                     destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1071                                     destroyedByEvictorCount.incrementAndGet();
1072                                     if (validationThrowable != null) {
1073                                         if (validationThrowable instanceof RuntimeException) {
1074                                             throw (RuntimeException) validationThrowable;
1075                                         }
1076                                         throw (Error) validationThrowable;
1077                                     }
1078                                 } else {
1079                                     try {
1080                                         factory.passivateObject(evictionKey, underTest);
1081                                     } catch (final Exception e) {
1082                                         destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1083                                         destroyedByEvictorCount.incrementAndGet();
1084                                     }
1085                                 }
1086                             }
1087                         }
1088                         underTest.endEvictionTest(idleObjects);
1089                         // TODO - May need to add code here once additional
1090                         // states are used
1091                     }
1092                 }
1093             }
1094         }
1095         final AbandonedConfig ac = this.abandonedConfig;
1096         if (ac != null && ac.getRemoveAbandonedOnMaintenance()) {
1097             removeAbandoned(ac);
1098         }
1099     }
1100 
1101     /**
1102      * Gets a reference to the factory used to create, destroy and validate
1103      * the objects used by this pool.
1104      *
1105      * @return the factory
1106      */
1107     public KeyedPooledObjectFactory<K, T> getFactory() {
1108         return factory;
1109     }
1110 
1111     /**
1112      * Gets a copy of the pool key list.
1113      *
1114      * @return a copy of the pool key list.
1115      * @since 2.12.0
1116      */
1117     @Override
1118     @SuppressWarnings("unchecked")
1119     public List<K> getKeys() {
1120         return (List<K>) poolKeyList.clone();
1121     }
1122 
1123     /**
1124      * Gets the cap on the number of "idle" instances per key in the pool.
1125      * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1126      * you will see objects being destroyed and almost immediately new objects
1127      * being created. This is a result of the active threads momentarily
1128      * returning objects faster than they are requesting them, causing the
1129      * number of idle objects to rise above maxIdlePerKey. The best value for
1130      * maxIdlePerKey for heavily loaded system will vary but the default is a
1131      * good starting point.
1132      *
1133      * @return the maximum number of "idle" instances that can be held in a
1134      *         given keyed sub-pool or a negative value if there is no limit
1135      *
1136      * @see #setMaxIdlePerKey
1137      */
1138     @Override
1139     public int getMaxIdlePerKey() {
1140         return maxIdlePerKey;
1141     }
1142 
1143     /**
1144      * Gets the limit on the number of object instances allocated by the pool
1145      * (checked out or idle), per key. When the limit is reached, the sub-pool
1146      * is said to be exhausted. A negative value indicates no limit.
1147      *
1148      * @return the limit on the number of active instances per key
1149      * @see #setMaxTotalPerKey
1150      */
1151     @Override
1152     public int getMaxTotalPerKey() {
1153         return maxTotalPerKey;
1154     }
1155 
1156     /**
1157      * Gets the target for the minimum number of idle objects to maintain in
1158      * each of the keyed sub-pools. This setting only has an effect if it is
1159      * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1160      * zero. If this is the case, an attempt is made to ensure that each
1161      * sub-pool has the required minimum number of instances during idle object
1162      * eviction runs.
1163      * <p>
1164      * If the configured value of minIdlePerKey is greater than the configured
1165      * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1166      * instead.
1167      * </p>
1168      *
1169      * @return minimum size of the each keyed pool
1170      * @see #setTimeBetweenEvictionRunsMillis
1171      */
1172     @Override
1173     public int getMinIdlePerKey() {
1174         final int maxIdlePerKeySave = getMaxIdlePerKey();
1175         return Math.min(this.minIdlePerKey, maxIdlePerKeySave);
1176     }
1177 
1178     @Override
1179     public int getNumActive() {
1180         return numTotal.get() - getNumIdle();
1181     }
1182 
1183     @Override
1184     public int getNumActive(final K key) {
1185         final ObjectDeque<T> objectDeque = poolMap.get(key);
1186         if (objectDeque != null) {
1187             return objectDeque.getAllObjects().size() -
1188                     objectDeque.getIdleObjects().size();
1189         }
1190         return 0;
1191     }
1192 
1193     @Override
1194     public Map<String, Integer> getNumActivePerKey() {
1195         return poolMap.entrySet().stream().collect(Collectors.toMap(
1196                 e -> e.getKey().toString(),
1197                 e -> Integer.valueOf(e.getValue().getAllObjects().size() - e.getValue().getIdleObjects().size()),
1198                 (t, u) -> u));
1199     }
1200 
1201     @Override
1202     public int getNumIdle() {
1203         return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().size()).sum();
1204     }
1205 
1206     @Override
1207     public int getNumIdle(final K key) {
1208         final ObjectDeque<T> objectDeque = poolMap.get(key);
1209         return objectDeque != null ? objectDeque.getIdleObjects().size() : 0;
1210     }
1211 
1212     /**
1213      * Gets the number of objects to test in a run of the idle object
1214      * evictor.
1215      *
1216      * @return The number of objects to test for validity
1217      */
1218     private int getNumTests() {
1219         final int totalIdle = getNumIdle();
1220         final int numTests = getNumTestsPerEvictionRun();
1221         if (numTests >= 0) {
1222             return Math.min(numTests, totalIdle);
1223         }
1224         return (int) Math.ceil(totalIdle / Math.abs((double) numTests));
1225     }
1226 
1227     /**
1228      * Gets an estimate of the number of threads currently blocked waiting for
1229      * an object from the pool. This is intended for monitoring only, not for
1230      * synchronization control.
1231      *
1232      * @return The estimate of the number of threads currently blocked waiting
1233      *         for an object from the pool
1234      */
1235     @Override
1236     public int getNumWaiters() {
1237         if (getBlockWhenExhausted()) {
1238             // Assume no overflow
1239             return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().getTakeQueueLength()).sum();
1240         }
1241         return 0;
1242     }
1243 
1244     /**
1245      * Gets an estimate of the number of threads currently blocked waiting for
1246      * an object from the pool for each key. This is intended for
1247      * monitoring only, not for synchronization control.
1248      *
1249      * @return The estimate of the number of threads currently blocked waiting
1250      *         for an object from the pool for each key
1251      */
1252     @Override
1253     public Map<String, Integer> getNumWaitersByKey() {
1254         final Map<String, Integer> result = new HashMap<>();
1255         poolMap.forEach((k, deque) -> result.put(k.toString(), getBlockWhenExhausted() ?
1256                 Integer.valueOf(deque.getIdleObjects().getTakeQueueLength()) :
1257                 ZERO));
1258         return result;
1259     }
1260 
1261     @Override
1262     String getStatsString() {
1263         // Simply listed in AB order.
1264         return super.getStatsString() +
1265                 String.format(", fairness=%s, maxIdlePerKey%,d, maxTotalPerKey=%,d, minIdlePerKey=%,d, numTotal=%,d",
1266                         fairness, maxIdlePerKey, maxTotalPerKey, minIdlePerKey, numTotal.get());
1267     }
1268 
1269     /**
1270      * Tests to see if there are any threads currently waiting to borrow
1271      * objects but are blocked waiting for more objects to become available.
1272      *
1273      * @return {@code true} if there is at least one thread waiting otherwise
1274      *         {@code false}
1275      */
1276     private boolean hasBorrowWaiters() {
1277         return getBlockWhenExhausted() && poolMap.values().stream().anyMatch(deque -> deque.getIdleObjects().hasTakeWaiters());
1278     }
1279 
1280     /**
1281      * {@inheritDoc}
1282      * <p>
1283      * Activation of this method decrements the active count associated with
1284      * the given keyed pool and attempts to destroy {@code obj.}
1285      * </p>
1286      *
1287      * @param key pool key
1288      * @param obj instance to invalidate
1289      * @throws Exception             if an exception occurs destroying the
1290      *                               object
1291      * @throws IllegalStateException if obj does not belong to the pool
1292      *                               under the given key
1293      */
1294     @Override
1295     public void invalidateObject(final K key, final T obj) throws Exception {
1296         invalidateObject(key, obj, DestroyMode.NORMAL);
1297     }
1298 
1299     /**
1300      * {@inheritDoc}
1301      * <p>
1302      * Activation of this method decrements the active count associated with
1303      * the given keyed pool and attempts to destroy {@code obj.}
1304      * </p>
1305      *
1306      * @param key pool key
1307      * @param obj instance to invalidate
1308      * @param destroyMode DestroyMode context provided to factory
1309      * @throws Exception             if an exception occurs destroying the
1310      *                               object
1311      * @throws IllegalStateException if obj does not belong to the pool
1312      *                               under the given key
1313      * @since 2.9.0
1314      */
1315     @Override
1316     public void invalidateObject(final K key, final T obj, final DestroyMode destroyMode) throws Exception {
1317         final ObjectDeque<T> objectDeque = poolMap.get(key);
1318         final PooledObject<T> p = objectDeque != null ? objectDeque.getAllObjects().get(new IdentityWrapper<>(obj)) : null;
1319         if (p == null) {
1320             throw new IllegalStateException(appendStats("Object not currently part of this pool"));
1321         }
1322         synchronized (p) {
1323             if (p.getState() != PooledObjectState.INVALID) {
1324                 destroy(key, p, true, destroyMode);
1325                 reuseCapacity();
1326             }
1327         }
1328     }
1329 
1330     /**
1331      * Provides information on all the objects in the pool, both idle (waiting
1332      * to be borrowed) and active (currently borrowed).
1333      * <p>
1334      * Note: This is named listAllObjects so it is presented as an operation via
1335      * JMX. That means it won't be invoked unless the explicitly requested
1336      * whereas all attributes will be automatically requested when viewing the
1337      * attributes for an object in a tool like JConsole.
1338      * </p>
1339      *
1340      * @return Information grouped by key on all the objects in the pool
1341      */
1342     @Override
1343     public Map<String, List<DefaultPooledObjectInfo>> listAllObjects() {
1344         return poolMap.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(),
1345                 e -> e.getValue().getAllObjects().values().stream().map(DefaultPooledObjectInfo::new).collect(Collectors.toList())));
1346     }
1347 
1348     /**
1349      * Registers a key for pool control and ensures that
1350      * {@link #getMinIdlePerKey()} idle instances are created.
1351      *
1352      * @param key   The key to register for pool control.
1353      * @throws Exception If the associated factory throws an exception
1354      */
1355     public void preparePool(final K key) throws Exception {
1356         final int minIdlePerKeySave = getMinIdlePerKey();
1357         if (minIdlePerKeySave < 1) {
1358             return;
1359         }
1360         ensureMinIdle(key);
1361     }
1362 
1363     /**
1364      * Register the use of a key by an object.
1365      * <p>
1366      * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
1367      * </p>
1368      *
1369      * @param k The key to register
1370      * @return The objects currently associated with the given key. If this
1371      *         method returns without throwing an exception then it will never
1372      *         return null.
1373      */
1374     private ObjectDeque<T> register(final K k) {
1375         Lock lock = keyLock.readLock();
1376         ObjectDeque<T> objectDeque = null;
1377         try {
1378             lock.lock();
1379             objectDeque = poolMap.get(k);
1380             if (objectDeque == null) {
1381                 // Upgrade to write lock
1382                 lock.unlock();
1383                 lock = keyLock.writeLock();
1384                 lock.lock();
1385                 final AtomicBoolean allocated = new AtomicBoolean();
1386                 objectDeque = poolMap.computeIfAbsent(k, key -> {
1387                     allocated.set(true);
1388                     final ObjectDeque<T> deque = new ObjectDeque<>(fairness);
1389                     deque.getNumInterested().incrementAndGet();
1390                     // NOTE: Keys must always be added to both poolMap and
1391                     //       poolKeyList at the same time while protected by
1392                     //       the write lock
1393                     poolKeyList.add(k);
1394                     return deque;
1395                 });
1396                 if (!allocated.get()) {
1397                     // Another thread could have beaten us to creating the deque, while we were
1398                     // waiting for the write lock, so re-get it from the map.
1399                     objectDeque = poolMap.get(k);
1400                     objectDeque.getNumInterested().incrementAndGet();
1401                 }
1402             } else {
1403                 objectDeque.getNumInterested().incrementAndGet();
1404             }
1405         } finally {
1406             lock.unlock();
1407         }
1408         return objectDeque;
1409     }
1410 
1411     /**
1412      * Recovers abandoned objects which have been checked out but
1413      * not used since longer than the removeAbandonedTimeout.
1414      *
1415      * @param abandonedConfig The configuration to use to identify abandoned objects
1416      */
1417     @SuppressWarnings("resource") // The PrintWriter is managed elsewhere
1418     private void removeAbandoned(final AbandonedConfig abandonedConfig) {
1419         poolMap.forEach((key, value) -> {
1420             // Generate a list of abandoned objects to remove
1421             final ArrayList<PooledObject<T>> remove = createRemoveList(abandonedConfig, value.getAllObjects());
1422             // Now remove the abandoned objects
1423             remove.forEach(pooledObject -> {
1424                 if (abandonedConfig.getLogAbandoned()) {
1425                     pooledObject.printStackTrace(abandonedConfig.getLogWriter());
1426                 }
1427                 try {
1428                     invalidateObject(key, pooledObject.getObject(), DestroyMode.ABANDONED);
1429                 } catch (final Exception e) {
1430                     swallowException(e);
1431                 }
1432             });
1433         });
1434     }
1435 
1436     /**
1437      * Returns an object to a keyed sub-pool.
1438      * <p>
1439      * If {@link #getMaxIdlePerKey() maxIdle} is set to a positive value and the
1440      * number of idle instances under the given key has reached this value, the
1441      * returning instance is destroyed.
1442      * </p>
1443      * <p>
1444      * If {@link #getTestOnReturn() testOnReturn} == true, the returning
1445      * instance is validated before being returned to the idle instance sub-pool
1446      * under the given key. In this case, if validation fails, the instance is
1447      * destroyed.
1448      * </p>
1449      * <p>
1450      * Exceptions encountered destroying objects for any reason are swallowed
1451      * but notified via a {@link SwallowedExceptionListener}.
1452      * </p>
1453      *
1454      * @param key pool key
1455      * @param obj instance to return to the keyed pool
1456      * @throws IllegalStateException if an object is returned to the pool that
1457      *                               was not borrowed from it or if an object is
1458      *                               returned to the pool multiple times
1459      */
1460     @Override
1461     public void returnObject(final K key, final T obj) {
1462 
1463         final ObjectDeque<T> objectDeque = poolMap.get(key);
1464 
1465         if (objectDeque == null) {
1466             throw new IllegalStateException("No keyed pool found under the given key.");
1467         }
1468 
1469         final PooledObject<T> p = objectDeque.getAllObjects().get(new IdentityWrapper<>(obj));
1470 
1471         if (PooledObject.isNull(p)) {
1472             throw new IllegalStateException("Returned object not currently part of this pool");
1473         }
1474 
1475         markReturningState(p);
1476 
1477         final Duration activeTime = p.getActiveDuration();
1478 
1479         try {
1480             if (getTestOnReturn() && !factory.validateObject(key, p)) {
1481                 try {
1482                     destroy(key, p, true, DestroyMode.NORMAL);
1483                 } catch (final Exception e) {
1484                     swallowException(e);
1485                 }
1486                 whenWaitersAddObject(key, objectDeque.idleObjects);
1487                 return;
1488             }
1489 
1490             try {
1491                 factory.passivateObject(key, p);
1492             } catch (final Exception e1) {
1493                 swallowException(e1);
1494                 try {
1495                     destroy(key, p, true, DestroyMode.NORMAL);
1496                 } catch (final Exception e) {
1497                     swallowException(e);
1498                 }
1499                 whenWaitersAddObject(key, objectDeque.idleObjects);
1500                 return;
1501             }
1502 
1503             if (!p.deallocate()) {
1504                 throw new IllegalStateException("Object has already been returned to this pool");
1505             }
1506 
1507             final int maxIdle = getMaxIdlePerKey();
1508             final LinkedBlockingDeque<PooledObject<T>> idleObjects = objectDeque.getIdleObjects();
1509 
1510             if (isClosed() || maxIdle > -1 && maxIdle <= idleObjects.size()) {
1511                 try {
1512                     destroy(key, p, true, DestroyMode.NORMAL);
1513                 } catch (final Exception e) {
1514                     swallowException(e);
1515                 }
1516             } else {
1517                 if (getLifo()) {
1518                     idleObjects.addFirst(p);
1519                 } else {
1520                     idleObjects.addLast(p);
1521                 }
1522                 if (isClosed()) {
1523                     // Pool closed while object was being added to idle objects.
1524                     // Make sure the returned object is destroyed rather than left
1525                     // in the idle object pool (which would effectively be a leak)
1526                     clear(key);
1527                 }
1528             }
1529         } finally {
1530             if (hasBorrowWaiters()) {
1531                 reuseCapacity();
1532             }
1533             updateStatsReturn(activeTime);
1534         }
1535     }
1536 
1537     /**
1538      * Attempt to create one new instance to serve from the most heavily
1539      * loaded pool that can add a new instance.
1540      *
1541      * This method exists to ensure liveness in the pool when threads are
1542      * parked waiting and capacity to create instances under the requested keys
1543      * subsequently becomes available.
1544      *
1545      * This method is not guaranteed to create an instance and its selection
1546      * of the most loaded pool that can create an instance may not always be
1547      * correct, since it does not lock the pool and instances may be created,
1548      * borrowed, returned or destroyed by other threads while it is executing.
1549      */
1550     private void reuseCapacity() {
1551         final int maxTotalPerKeySave = getMaxTotalPerKey();
1552         int maxQueueLength = 0;
1553         LinkedBlockingDeque<PooledObject<T>> mostLoadedPool = null;
1554         K mostLoadedKey = null;
1555 
1556         // Find the most loaded pool that could take a new instance
1557         for (final Map.Entry<K, ObjectDeque<T>> entry : poolMap.entrySet()) {
1558             final K k = entry.getKey();
1559             final LinkedBlockingDeque<PooledObject<T>> pool = entry.getValue().getIdleObjects();
1560             final int queueLength = pool.getTakeQueueLength();
1561             if (getNumActive(k) < maxTotalPerKeySave && queueLength > maxQueueLength) {
1562                 maxQueueLength = queueLength;
1563                 mostLoadedPool = pool;
1564                 mostLoadedKey = k;
1565             }
1566         }
1567 
1568         // Attempt to add an instance to the most loaded pool.
1569         if (mostLoadedPool != null) {
1570             register(mostLoadedKey);
1571             try {
1572                 // If there is no capacity to add, create will return null
1573                 // and addIdleObject will no-op.
1574                 addIdleObject(mostLoadedKey, create(mostLoadedKey));
1575             } catch (final Exception e) {
1576                 swallowException(e);
1577             } finally {
1578                 deregister(mostLoadedKey);
1579             }
1580         }
1581     }
1582 
1583     /**
1584      * Call {@link #reuseCapacity()} repeatedly.
1585      * <p>
1586      * Always activates {@link #reuseCapacity()} at least once.
1587      *
1588      * @param newCapacity number of new instances to attempt to create.
1589      */
1590     private void reuseCapacity(final int newCapacity) {
1591         final int bound = newCapacity < 1 ? 1 : newCapacity;
1592         for (int i = 0; i < bound; i++) {
1593             reuseCapacity();
1594         }
1595     }
1596 
1597     /**
1598      * Sets the configuration.
1599      *
1600      * @param conf the new configuration to use. This is used by value.
1601      * @see GenericKeyedObjectPoolConfig
1602      */
1603     public void setConfig(final GenericKeyedObjectPoolConfig<T> conf) {
1604         super.setConfig(conf);
1605         setMaxIdlePerKey(conf.getMaxIdlePerKey());
1606         setMaxTotalPerKey(conf.getMaxTotalPerKey());
1607         setMaxTotal(conf.getMaxTotal());
1608         setMinIdlePerKey(conf.getMinIdlePerKey());
1609     }
1610 
1611     /**
1612      * Sets the cap on the number of "idle" instances per key in the pool.
1613      * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1614      * you will see objects being destroyed and almost immediately new objects
1615      * being created. This is a result of the active threads momentarily
1616      * returning objects faster than they are requesting them, causing the
1617      * number of idle objects to rise above maxIdlePerKey. The best value for
1618      * maxIdlePerKey for heavily loaded system will vary but the default is a
1619      * good starting point.
1620      *
1621      * @param maxIdlePerKey the maximum number of "idle" instances that can be
1622      *                      held in a given keyed sub-pool. Use a negative value
1623      *                      for no limit
1624      *
1625      * @see #getMaxIdlePerKey
1626      */
1627     public void setMaxIdlePerKey(final int maxIdlePerKey) {
1628         this.maxIdlePerKey = maxIdlePerKey;
1629     }
1630 
1631     /**
1632      * Sets the limit on the number of object instances allocated by the pool
1633      * (checked out or idle), per key. When the limit is reached, the sub-pool
1634      * is said to be exhausted. A negative value indicates no limit.
1635      *
1636      * @param maxTotalPerKey the limit on the number of active instances per key
1637      * @see #getMaxTotalPerKey
1638      */
1639     public void setMaxTotalPerKey(final int maxTotalPerKey) {
1640         this.maxTotalPerKey = maxTotalPerKey;
1641     }
1642 
1643     /**
1644      * Sets the target for the minimum number of idle objects to maintain in
1645      * each of the keyed sub-pools. This setting only has an effect if it is
1646      * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1647      * zero. If this is the case, an attempt is made to ensure that each
1648      * sub-pool has the required minimum number of instances during idle object
1649      * eviction runs.
1650      * <p>
1651      * If the configured value of minIdlePerKey is greater than the configured
1652      * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1653      * instead.
1654      * </p>
1655      *
1656      * @param minIdlePerKey The minimum size of the each keyed pool
1657      * @see #getMinIdlePerKey()
1658      * @see #getMaxIdlePerKey()
1659      * @see #setDurationBetweenEvictionRuns(Duration)
1660      */
1661     public void setMinIdlePerKey(final int minIdlePerKey) {
1662         this.minIdlePerKey = minIdlePerKey;
1663     }
1664 
1665     @Override
1666     protected void toStringAppendFields(final StringBuilder builder) {
1667         super.toStringAppendFields(builder);
1668         builder.append(", maxIdlePerKey=");
1669         builder.append(maxIdlePerKey);
1670         builder.append(", minIdlePerKey=");
1671         builder.append(minIdlePerKey);
1672         builder.append(", maxTotalPerKey=");
1673         builder.append(maxTotalPerKey);
1674         builder.append(", factory=");
1675         builder.append(factory);
1676         builder.append(", fairness=");
1677         builder.append(fairness);
1678         builder.append(", poolMap=");
1679         builder.append(poolMap);
1680         builder.append(", poolKeyList=");
1681         builder.append(poolKeyList);
1682         builder.append(", keyLock=");
1683         builder.append(keyLock);
1684         builder.append(", numTotal=");
1685         builder.append(numTotal);
1686         builder.append(", evictionKeyIterator=");
1687         builder.append(evictionKeyIterator);
1688         builder.append(", evictionKey=");
1689         builder.append(evictionKey);
1690         builder.append(", abandonedConfig=");
1691         builder.append(abandonedConfig);
1692     }
1693 
1694     /**
1695      * @since 2.10.0
1696      */
1697     @Override
1698     public void use(final T pooledObject) {
1699         final AbandonedConfig abandonedCfg = this.abandonedConfig;
1700         if (abandonedCfg != null && abandonedCfg.getUseUsageTracking()) {
1701             poolMap.values().stream()
1702                 .map(pool -> pool.getAllObjects().get(new IdentityWrapper<>(pooledObject)))
1703                 .filter(Objects::nonNull)
1704                 .findFirst()
1705                 .ifPresent(PooledObject::use);
1706         }
1707     }
1708 
1709     /**
1710      * When there is at least one thread waiting on the given deque, try to add an instance to pool under the given key.
1711      * NOTE: there is no check that the key corresponds to the deque (it is assumed that the key was used to get the deque
1712      * from the poolMap)
1713      *
1714      * @param key pool key.
1715      * @param idleObjects deque backing the pool under the given key
1716      */
1717     private void whenWaitersAddObject(final K key, final LinkedBlockingDeque<PooledObject<T>> idleObjects) {
1718         if (idleObjects.hasTakeWaiters()) {
1719             try {
1720                 addObject(key);
1721             } catch (final Exception e) {
1722                 swallowException(e);
1723             }
1724         }
1725     }
1726 
1727 }