View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.pool2.impl;
19  
20  
21  import static org.junit.jupiter.api.Assertions.assertEquals;
22  import static org.junit.jupiter.api.Assertions.assertFalse;
23  import static org.junit.jupiter.api.Assertions.assertNotEquals;
24  import static org.junit.jupiter.api.Assertions.assertNotNull;
25  import static org.junit.jupiter.api.Assertions.assertNotSame;
26  import static org.junit.jupiter.api.Assertions.assertSame;
27  import static org.junit.jupiter.api.Assertions.assertThrows;
28  import static org.junit.jupiter.api.Assertions.assertTrue;
29  import static org.junit.jupiter.api.Assertions.fail;
30  
31  import java.lang.management.ManagementFactory;
32  import java.time.Duration;
33  import java.util.ArrayList;
34  import java.util.Arrays;
35  import java.util.HashSet;
36  import java.util.List;
37  import java.util.NoSuchElementException;
38  import java.util.Objects;
39  import java.util.Random;
40  import java.util.Set;
41  import java.util.Timer;
42  import java.util.TimerTask;
43  import java.util.concurrent.ConcurrentHashMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Executors;
47  import java.util.concurrent.Future;
48  import java.util.concurrent.Semaphore;
49  import java.util.concurrent.ThreadFactory;
50  import java.util.concurrent.TimeUnit;
51  import java.util.concurrent.atomic.AtomicInteger;
52  
53  import javax.management.MBeanServer;
54  import javax.management.ObjectName;
55  
56  import org.apache.commons.lang3.exception.ExceptionUtils;
57  import org.apache.commons.pool2.AbstractTestKeyedObjectPool;
58  import org.apache.commons.pool2.BaseKeyedPooledObjectFactory;
59  import org.apache.commons.pool2.DestroyMode;
60  import org.apache.commons.pool2.KeyedObjectPool;
61  import org.apache.commons.pool2.KeyedPooledObjectFactory;
62  import org.apache.commons.pool2.PooledObject;
63  import org.apache.commons.pool2.VisitTracker;
64  import org.apache.commons.pool2.VisitTrackerFactory;
65  import org.apache.commons.pool2.Waiter;
66  import org.apache.commons.pool2.WaiterFactory;
67  import org.junit.jupiter.api.AfterEach;
68  import org.junit.jupiter.api.BeforeEach;
69  import org.junit.jupiter.api.Test;
70  import org.junit.jupiter.api.Timeout;
71  import org.junit.jupiter.params.ParameterizedTest;
72  import org.junit.jupiter.params.provider.EnumSource;
73  
74  /**
75   */
76  public class TestGenericKeyedObjectPool extends AbstractTestKeyedObjectPool {
77  
78      private static class DaemonThreadFactory implements ThreadFactory {
79          @Override
80          public Thread newThread(final Runnable r) {
81              final Thread t = new Thread(r);
82              t.setDaemon(true);
83              return t;
84          }
85      }
86  
87      private static class DummyFactory extends BaseKeyedPooledObjectFactory<Object, Object> {
88          @Override
89          public Object create(final Object key) {
90              return null;
91          }
92          @Override
93          public PooledObject<Object> wrap(final Object value) {
94              return new DefaultPooledObject<>(value);
95          }
96      }
97  
98      /**
99       * Factory that creates HashSets.  Note that this means
100      *  0) All instances are initially equal (not discernible by equals)
101      *  1) Instances are mutable and mutation can cause change in identity / hash code.
102      */
103     private static final class HashSetFactory
104             extends BaseKeyedPooledObjectFactory<String, HashSet<String>> {
105         @Override
106         public HashSet<String> create(final String key) {
107             return new HashSet<>();
108         }
109         @Override
110         public PooledObject<HashSet<String>> wrap(final HashSet<String> value) {
111             return new DefaultPooledObject<>(value);
112         }
113     }
114 
115     /**
116      * Attempts to invalidate an object, swallowing IllegalStateException.
117      */
118     static class InvalidateThread implements Runnable {
119 
120         private final String obj;
121         private final KeyedObjectPool<String, String> pool;
122         private final String key;
123         private boolean done;
124 
125         public InvalidateThread(final KeyedObjectPool<String, String> pool, final String key, final String obj) {
126             this.obj = obj;
127             this.pool = pool;
128             this.key = key;
129         }
130 
131         public boolean complete() {
132             return done;
133         }
134 
135         @Override
136         public void run() {
137             try {
138                 pool.invalidateObject(key, obj);
139             } catch (final IllegalStateException ex) {
140                 // Ignore
141             } catch (final Exception ex) {
142                 fail("Unexpected exception " + ex.toString());
143             } finally {
144                 done = true;
145             }
146         }
147     }
148 
149     private static class ObjectFactory
150         extends BaseKeyedPooledObjectFactory<Integer, Object> {
151 
152         @Override
153         public Object create(final Integer key) {
154             return new Object();
155         }
156 
157         @Override
158         public PooledObject<Object> wrap(final Object value) {
159             return new DefaultPooledObject<>(value);
160         }
161     }
162 
163     public static class SimpleFactory<K> implements KeyedPooledObjectFactory<K, String> {
164         volatile int counter;
165         final boolean valid;
166         int activeCount;
167         int validateCounter;
168         boolean evenValid = true;
169         boolean oddValid = true;
170         boolean enableValidation;
171 
172         long destroyLatency;
173         long makeLatency;
174         long validateLatency;
175         volatile int maxTotalPerKey = Integer.MAX_VALUE;
176         boolean exceptionOnPassivate;
177         boolean exceptionOnActivate;
178         boolean exceptionOnDestroy;
179         boolean exceptionOnValidate;
180 
181         boolean exceptionOnCreate;
182 
183         public SimpleFactory() {
184             this(true);
185         }
186 
187         public SimpleFactory(final boolean valid) {
188             this.valid = valid;
189         }
190 
191         @Override
192         public void activateObject(final K key, final PooledObject<String> obj) throws Exception {
193             if (exceptionOnActivate && !(validateCounter++ % 2 == 0 ? evenValid : oddValid)) {
194                 throw new Exception();
195             }
196         }
197 
198         @Override
199         public void destroyObject(final K key, final PooledObject<String> obj) throws Exception {
200             doWait(destroyLatency);
201             synchronized(this) {
202                 activeCount--;
203             }
204             if (exceptionOnDestroy) {
205                 throw new Exception();
206             }
207         }
208 
209         private void doWait(final long latency) {
210             Waiter.sleepQuietly(latency);
211         }
212 
213         @Override
214         public PooledObject<String> makeObject(final K key) throws Exception {
215             if (exceptionOnCreate) {
216                 throw new Exception();
217             }
218             doWait(makeLatency);
219             String out = null;
220             synchronized(this) {
221                 activeCount++;
222                 if (activeCount > maxTotalPerKey) {
223                     throw new IllegalStateException(
224                         "Too many active instances: " + activeCount);
225                 }
226                 out = String.valueOf(key) + String.valueOf(counter++);
227             }
228             return new DefaultPooledObject<>(out);
229         }
230 
231         @Override
232         public void passivateObject(final K key, final PooledObject<String> obj) throws Exception {
233             if (exceptionOnPassivate) {
234                 throw new Exception();
235             }
236         }
237 
238         public void setDestroyLatency(final long destroyLatency) {
239             this.destroyLatency = destroyLatency;
240         }
241 
242         void setEvenValid(final boolean valid) {
243             evenValid = valid;
244         }
245 
246         public void setMakeLatency(final long makeLatency) {
247             this.makeLatency = makeLatency;
248         }
249 
250         public void setMaxTotalPerKey(final int maxTotalPerKey) {
251             this.maxTotalPerKey = maxTotalPerKey;
252         }
253 
254         public void setThrowExceptionOnActivate(final boolean b) {
255             exceptionOnActivate = b;
256         }
257 
258         public void setThrowExceptionOnDestroy(final boolean b) {
259             exceptionOnDestroy = b;
260         }
261 
262         public void setThrowExceptionOnPassivate(final boolean b) {
263             exceptionOnPassivate = b;
264         }
265 
266         public void setThrowExceptionOnValidate(final boolean b) {
267             exceptionOnValidate = b;
268         }
269 
270         void setValid(final boolean valid) {
271             evenValid = valid;
272             oddValid = valid;
273         }
274 
275         public void setValidateLatency(final long validateLatency) {
276             this.validateLatency = validateLatency;
277         }
278 
279         public void setValidationEnabled(final boolean b) {
280             enableValidation = b;
281         }
282 
283         @Override
284         public boolean validateObject(final K key, final PooledObject<String> obj) {
285             doWait(validateLatency);
286             if (exceptionOnValidate) {
287                 throw new RuntimeException("validation failed");
288             }
289             if (enableValidation) {
290                 return validateCounter++ % 2 == 0 ? evenValid : oddValid;
291             }
292             return valid;
293         }
294     }
295 
296     private static class SimplePerKeyFactory extends BaseKeyedPooledObjectFactory<Object, Object> {
297         final ConcurrentHashMap<Object, AtomicInteger> map = new ConcurrentHashMap<>();
298 
299         @Override
300         public Object create(final Object key) {
301             int counter = map.computeIfAbsent(key, k -> new AtomicInteger(-1)).incrementAndGet();
302             return String.valueOf(key) + String.valueOf(counter);
303         }
304 
305         @Override
306         public PooledObject<Object> wrap(final Object value) {
307             return new DefaultPooledObject<>(value);
308         }
309     }
310 
311     /**
312      * Very simple test thread that just tries to borrow an object from
313      * the provided pool with the specified key and returns it
314      */
315     static class SimpleTestThread<T> implements Runnable {
316         private final KeyedObjectPool<String, T> pool;
317         private final String key;
318 
319         public SimpleTestThread(final KeyedObjectPool<String, T> pool, final String key) {
320             this.pool = pool;
321             this.key = key;
322         }
323 
324         @Override
325         public void run() {
326             try {
327                 pool.returnObject(key, pool.borrowObject(key));
328             } catch (final Exception e) {
329                 // Ignore
330             }
331         }
332     }
333 
334     /**
335      * DefaultEvictionPolicy modified to add latency
336      */
337     private static class SlowEvictionPolicy<T> extends DefaultEvictionPolicy<T> {
338         private final long delay;
339 
340         /**
341          * Constructs SlowEvictionPolicy with the given delay in ms
342          *
343          * @param delay number of ms of latency to inject in evict
344          */
345         public SlowEvictionPolicy(final long delay) {
346             this.delay = delay;
347         }
348 
349         @Override
350         public boolean evict(final EvictionConfig config, final PooledObject<T> underTest,
351                 final int idleCount) {
352             Waiter.sleepQuietly(delay);
353             return super.evict(config, underTest, idleCount);
354         }
355     }
356 
357     static class TestThread<T> implements Runnable {
358         private final java.util.Random random = new java.util.Random();
359 
360         /** GKOP to hit */
361         private final KeyedObjectPool<String, T> pool;
362         /** number of borrow/return iterations */
363         private final int iter;
364         /** delay before borrow */
365         private final int startDelay;
366         /** delay before return */
367         private final int holdTime;
368         /** whether or not delays are random (with max = configured values) */
369         private final boolean randomDelay;
370         /** expected object */
371         private final T expectedObject;
372         /** key used in borrow / return sequence - null means random */
373         private final String key;
374 
375         private volatile boolean complete;
376         private volatile boolean failed;
377         private volatile Exception exception;
378 
379         public TestThread(final KeyedObjectPool<String, T> pool) {
380             this(pool, 100, 50, 50, true, null, null);
381         }
382 
383         public TestThread(final KeyedObjectPool<String, T> pool, final int iter) {
384             this(pool, iter, 50, 50, true, null, null);
385         }
386 
387         public TestThread(final KeyedObjectPool<String, T> pool, final int iter, final int delay) {
388             this(pool, iter, delay, delay, true, null, null);
389         }
390 
391         public TestThread(final KeyedObjectPool<String, T> pool, final int iter, final int startDelay,
392             final int holdTime, final boolean randomDelay, final T expectedObject, final String key) {
393             this.pool = pool;
394             this.iter = iter;
395             this.startDelay = startDelay;
396             this.holdTime = holdTime;
397             this.randomDelay = randomDelay;
398             this.expectedObject = expectedObject;
399             this.key = key;
400 
401         }
402 
403         public boolean complete() {
404             return complete;
405         }
406 
407         public boolean failed() {
408             return failed;
409         }
410 
411         @Override
412         public void run() {
413             for (int i = 0; i < iter; i++) {
414                 final String actualKey = key == null ? String.valueOf(random.nextInt(3)) : key;
415                 Waiter.sleepQuietly(randomDelay ? random.nextInt(startDelay) : startDelay);
416                 T obj = null;
417                 try {
418                     obj = pool.borrowObject(actualKey);
419                 } catch (final Exception e) {
420                     exception = e;
421                     failed = true;
422                     complete = true;
423                     break;
424                 }
425 
426                 if (expectedObject != null && !expectedObject.equals(obj)) {
427                     exception = new Exception("Expected: " + expectedObject + " found: " + obj);
428                     failed = true;
429                     complete = true;
430                     break;
431                 }
432 
433                 Waiter.sleepQuietly(randomDelay ? random.nextInt(holdTime) : holdTime);
434                 try {
435                     pool.returnObject(actualKey, obj);
436                 } catch (final Exception e) {
437                     exception = e;
438                     failed = true;
439                     complete = true;
440                     break;
441                 }
442             }
443             complete = true;
444         }
445     }
446 
447     /*
448      * Very simple test thread that just tries to borrow an object from
449      * the provided pool with the specified key and returns it after a wait
450      */
451     static class WaitingTestThread extends Thread {
452         private final KeyedObjectPool<String, String> pool;
453         private final String key;
454         private final long pause;
455         private Throwable thrown;
456 
457         private long preBorrowMillis; // just before borrow
458         private long postBorrowMillis; //  borrow returned
459         private long postReturnMillis; // after object was returned
460         private long endedMillis;
461         private String objectId;
462 
463         public WaitingTestThread(final KeyedObjectPool<String, String> pool, final String key, final long pause) {
464             this.pool = pool;
465             this.key = key;
466             this.pause = pause;
467             this.thrown = null;
468         }
469 
470         @Override
471         public void run() {
472             try {
473                 preBorrowMillis = System.currentTimeMillis();
474                 final String obj = pool.borrowObject(key);
475                 objectId = obj;
476                 postBorrowMillis = System.currentTimeMillis();
477                 Thread.sleep(pause);
478                 pool.returnObject(key, obj);
479                 postReturnMillis = System.currentTimeMillis();
480             } catch (final Exception e) {
481                 thrown = e;
482             } finally{
483                 endedMillis = System.currentTimeMillis();
484             }
485         }
486     }
487 
488     private static final Integer KEY_ZERO = Integer.valueOf(0);
489 
490     private static final Integer KEY_ONE = Integer.valueOf(1);
491 
492     private static final Integer KEY_TWO = Integer.valueOf(2);
493 
494     private static final boolean DISPLAY_THREAD_DETAILS=
495     Boolean.getBoolean("TestGenericKeyedObjectPool.display.thread.details");
496     // To pass this to a Maven test, use:
497     // mvn test -DargLine="-DTestGenericKeyedObjectPool.display.thread.details=true"
498     // @see https://issues.apache.org/jira/browse/SUREFIRE-121
499 
500     /** setUp(): {@code new GenericKeyedObjectPool<String,String>(factory)} */
501     private GenericKeyedObjectPool<String, String> gkoPool;
502 
503     /** setUp(): {@code new SimpleFactory<String>()} */
504     private SimpleFactory<String> simpleFactory;
505 
506     private void checkEvictionOrder(final boolean lifo) throws Exception {
507         final SimpleFactory<Integer> intFactory = new SimpleFactory<>();
508         try (final GenericKeyedObjectPool<Integer, String> intPool = new GenericKeyedObjectPool<>(intFactory)) {
509             intPool.setNumTestsPerEvictionRun(2);
510             intPool.setMinEvictableIdleTime(Duration.ofMillis(100));
511             intPool.setLifo(lifo);
512 
513             for (int i = 0; i < 3; i++) {
514                 final Integer key = Integer.valueOf(i);
515                 for (int j = 0; j < 5; j++) {
516                     intPool.addObject(key);
517                 }
518             }
519 
520             // Make all evictable
521             Thread.sleep(200);
522 
523             /*
524              * Initial state (Key, Object) pairs in order of age:
525              *
526              * (0,0), (0,1), (0,2), (0,3), (0,4) (1,5), (1,6), (1,7), (1,8), (1,9) (2,10), (2,11), (2,12), (2,13),
527              * (2,14)
528              */
529 
530             intPool.evict(); // Kill (0,0),(0,1)
531             assertEquals(3, intPool.getNumIdle(KEY_ZERO));
532             final String objZeroA = intPool.borrowObject(KEY_ZERO);
533             assertTrue(lifo ? objZeroA.equals("04") : objZeroA.equals("02"));
534             assertEquals(2, intPool.getNumIdle(KEY_ZERO));
535             final String objZeroB = intPool.borrowObject(KEY_ZERO);
536             assertEquals("03", objZeroB);
537             assertEquals(1, intPool.getNumIdle(KEY_ZERO));
538 
539             intPool.evict(); // Kill remaining 0 survivor and (1,5)
540             assertEquals(0, intPool.getNumIdle(KEY_ZERO));
541             assertEquals(4, intPool.getNumIdle(KEY_ONE));
542             final String objOneA = intPool.borrowObject(KEY_ONE);
543             assertTrue(lifo ? objOneA.equals("19") : objOneA.equals("16"));
544             assertEquals(3, intPool.getNumIdle(KEY_ONE));
545             final String objOneB = intPool.borrowObject(KEY_ONE);
546             assertTrue(lifo ? objOneB.equals("18") : objOneB.equals("17"));
547             assertEquals(2, intPool.getNumIdle(KEY_ONE));
548 
549             intPool.evict(); // Kill remaining 1 survivors
550             assertEquals(0, intPool.getNumIdle(KEY_ONE));
551             intPool.evict(); // Kill (2,10), (2,11)
552             assertEquals(3, intPool.getNumIdle(KEY_TWO));
553             final String objTwoA = intPool.borrowObject(KEY_TWO);
554             assertTrue(lifo ? objTwoA.equals("214") : objTwoA.equals("212"));
555             assertEquals(2, intPool.getNumIdle(KEY_TWO));
556             intPool.evict(); // All dead now
557             assertEquals(0, intPool.getNumIdle(KEY_TWO));
558 
559             intPool.evict(); // Should do nothing - make sure no exception
560             // Currently 2 zero, 2 one and 1 two active. Return them
561             intPool.returnObject(KEY_ZERO, objZeroA);
562             intPool.returnObject(KEY_ZERO, objZeroB);
563             intPool.returnObject(KEY_ONE, objOneA);
564             intPool.returnObject(KEY_ONE, objOneB);
565             intPool.returnObject(KEY_TWO, objTwoA);
566             // Remove all idle objects
567             intPool.clear();
568 
569             // Reload
570             intPool.setMinEvictableIdleTime(Duration.ofMillis(500));
571             intFactory.counter = 0; // Reset counter
572             for (int i = 0; i < 3; i++) {
573                 final Integer key = Integer.valueOf(i);
574                 for (int j = 0; j < 5; j++) {
575                     intPool.addObject(key);
576                 }
577                 Thread.sleep(200);
578             }
579 
580             // 0's are evictable, others not
581             intPool.evict(); // Kill (0,0),(0,1)
582             assertEquals(3, intPool.getNumIdle(KEY_ZERO));
583             intPool.evict(); // Kill (0,2),(0,3)
584             assertEquals(1, intPool.getNumIdle(KEY_ZERO));
585             intPool.evict(); // Kill (0,4), leave (1,5)
586             assertEquals(0, intPool.getNumIdle(KEY_ZERO));
587             assertEquals(5, intPool.getNumIdle(KEY_ONE));
588             assertEquals(5, intPool.getNumIdle(KEY_TWO));
589             intPool.evict(); // (1,6), (1,7)
590             assertEquals(5, intPool.getNumIdle(KEY_ONE));
591             assertEquals(5, intPool.getNumIdle(KEY_TWO));
592             intPool.evict(); // (1,8), (1,9)
593             assertEquals(5, intPool.getNumIdle(KEY_ONE));
594             assertEquals(5, intPool.getNumIdle(KEY_TWO));
595             intPool.evict(); // (2,10), (2,11)
596             assertEquals(5, intPool.getNumIdle(KEY_ONE));
597             assertEquals(5, intPool.getNumIdle(KEY_TWO));
598             intPool.evict(); // (2,12), (2,13)
599             assertEquals(5, intPool.getNumIdle(KEY_ONE));
600             assertEquals(5, intPool.getNumIdle(KEY_TWO));
601             intPool.evict(); // (2,14), (1,5)
602             assertEquals(5, intPool.getNumIdle(KEY_ONE));
603             assertEquals(5, intPool.getNumIdle(KEY_TWO));
604             Thread.sleep(200); // Ones now timed out
605             intPool.evict(); // kill (1,6), (1,7) - (1,5) missed
606             assertEquals(3, intPool.getNumIdle(KEY_ONE));
607             assertEquals(5, intPool.getNumIdle(KEY_TWO));
608             final String obj = intPool.borrowObject(KEY_ONE);
609             if (lifo) {
610                 assertEquals("19", obj);
611             } else {
612                 assertEquals("15", obj);
613             }
614         }
615     }
616 
617     private void checkEvictorVisiting(final boolean lifo) throws Exception {
618         VisitTrackerFactory<Integer> trackerFactory = new VisitTrackerFactory<>();
619         try (GenericKeyedObjectPool<Integer, VisitTracker<Integer>> intPool = new GenericKeyedObjectPool<>(
620                 trackerFactory)) {
621             intPool.setNumTestsPerEvictionRun(2);
622             intPool.setMinEvictableIdleTime(Duration.ofMillis(-1));
623             intPool.setTestWhileIdle(true);
624             intPool.setLifo(lifo);
625             intPool.setTestOnReturn(false);
626             intPool.setTestOnBorrow(false);
627             for (int i = 0; i < 3; i++) {
628                 trackerFactory.resetId();
629                 final Integer key = Integer.valueOf(i);
630                 for (int j = 0; j < 8; j++) {
631                     intPool.addObject(key);
632                 }
633             }
634             intPool.evict(); // Visit oldest 2 - 00 and 01
635             VisitTracker<Integer> obj = intPool.borrowObject(KEY_ZERO);
636             intPool.returnObject(KEY_ZERO, obj);
637             obj = intPool.borrowObject(KEY_ZERO);
638             intPool.returnObject(KEY_ZERO, obj);
639             // borrow, return, borrow, return
640             // FIFO will move 0 and 1 to end - 2,3,4,5,6,7,0,1
641             // LIFO, 7 out, then in, then out, then in - 7,6,5,4,3,2,1,0
642             intPool.evict(); // Should visit 02 and 03 in either case
643             for (int i = 0; i < 8; i++) {
644                 final VisitTracker<Integer> tracker = intPool.borrowObject(KEY_ZERO);
645                 if (tracker.getId() >= 4) {
646                     assertEquals(0, tracker.getValidateCount(), "Unexpected instance visited " + tracker.getId());
647                 } else {
648                     assertEquals(1,
649                             tracker.getValidateCount(),"Instance " + tracker.getId() + " visited wrong number of times.");
650                 }
651             }
652             // 0's are all out
653 
654             intPool.setNumTestsPerEvictionRun(3);
655 
656             intPool.evict(); // 10, 11, 12
657             intPool.evict(); // 13, 14, 15
658 
659             obj = intPool.borrowObject(KEY_ONE);
660             intPool.returnObject(KEY_ONE, obj);
661             obj = intPool.borrowObject(KEY_ONE);
662             intPool.returnObject(KEY_ONE, obj);
663             obj = intPool.borrowObject(KEY_ONE);
664             intPool.returnObject(KEY_ONE, obj);
665             // borrow, return, borrow, return
666             // FIFO 3,4,5,^,6,7,0,1,2
667             // LIFO 7,6,^,5,4,3,2,1,0
668             // In either case, pointer should be at 6
669             intPool.evict();
670             // LIFO - 16, 17, 20
671             // FIFO - 16, 17, 10
672             intPool.evict();
673             // LIFO - 21, 22, 23
674             // FIFO - 11, 12, 20
675             intPool.evict();
676             // LIFO - 24, 25, 26
677             // FIFO - 21, 22, 23
678             intPool.evict();
679             // LIFO - 27, 10, 11
680             // FIFO - 24, 25, 26
681             for (int i = 0; i < 8; i++) {
682                 final VisitTracker<Integer> tracker = intPool.borrowObject(KEY_ONE);
683                 if (lifo && tracker.getId() > 1 || !lifo && tracker.getId() > 2) {
684                     assertEquals(1, tracker.getValidateCount(), "Instance " + tracker.getId() + " visited wrong number of times.");
685                 } else {
686                     assertEquals(2, tracker.getValidateCount(), "Instance " + tracker.getId() + " visited wrong number of times.");
687                 }
688             }
689         }
690 
691         // Randomly generate some pools with random numTests
692         // and make sure evictor cycles through elements appropriately
693         final int[] smallPrimes = { 2, 3, 5, 7 };
694         final Random random = new Random();
695         random.setSeed(System.currentTimeMillis());
696         for (int i = 0; i < smallPrimes.length; i++) {
697             for (int j = 0; j < 5; j++) {// Try the tests a few times
698                 // Can't use clear as some objects are still active so create
699                 // a new pool
700                 trackerFactory = new VisitTrackerFactory<>();
701                 try (GenericKeyedObjectPool<Integer, VisitTracker<Integer>> intPool = new GenericKeyedObjectPool<>(
702                         trackerFactory)) {
703                     intPool.setMaxIdlePerKey(-1);
704                     intPool.setMaxTotalPerKey(-1);
705                     intPool.setNumTestsPerEvictionRun(smallPrimes[i]);
706                     intPool.setMinEvictableIdleTime(Duration.ofMillis(-1));
707                     intPool.setTestWhileIdle(true);
708                     intPool.setLifo(lifo);
709                     intPool.setTestOnReturn(false);
710                     intPool.setTestOnBorrow(false);
711 
712                     final int zeroLength = 10 + random.nextInt(20);
713                     for (int k = 0; k < zeroLength; k++) {
714                         intPool.addObject(KEY_ZERO);
715                     }
716                     final int oneLength = 10 + random.nextInt(20);
717                     for (int k = 0; k < oneLength; k++) {
718                         intPool.addObject(KEY_ONE);
719                     }
720                     final int twoLength = 10 + random.nextInt(20);
721                     for (int k = 0; k < twoLength; k++) {
722                         intPool.addObject(KEY_TWO);
723                     }
724 
725                     // Choose a random number of evictor runs
726                     final int runs = 10 + random.nextInt(50);
727                     for (int k = 0; k < runs; k++) {
728                         intPool.evict();
729                     }
730 
731                     // Total instances in pool
732                     final int totalInstances = zeroLength + oneLength + twoLength;
733 
734                     // Number of times evictor should have cycled through pools
735                     final int cycleCount = runs * intPool.getNumTestsPerEvictionRun() / totalInstances;
736 
737                     // Look at elements and make sure they are visited cycleCount
738                     // or cycleCount + 1 times
739                     VisitTracker<Integer> tracker = null;
740                     int visitCount = 0;
741                     for (int k = 0; k < zeroLength; k++) {
742                         tracker = intPool.borrowObject(KEY_ZERO);
743                         visitCount = tracker.getValidateCount();
744                         if (visitCount < cycleCount || visitCount > cycleCount + 1) {
745                             fail(formatSettings("ZERO", "runs", runs, "lifo", lifo, "i", i, "j", j, "k", k,
746                                     "visitCount", visitCount, "cycleCount", cycleCount, "totalInstances",
747                                     totalInstances, zeroLength, oneLength, twoLength));
748                         }
749                     }
750                     for (int k = 0; k < oneLength; k++) {
751                         tracker = intPool.borrowObject(KEY_ONE);
752                         visitCount = tracker.getValidateCount();
753                         if (visitCount < cycleCount || visitCount > cycleCount + 1) {
754                             fail(formatSettings("ONE", "runs", runs, "lifo", lifo, "i", i, "j", j, "k", k, "visitCount",
755                                     visitCount, "cycleCount", cycleCount, "totalInstances", totalInstances, zeroLength,
756                                     oneLength, twoLength));
757                         }
758                     }
759                     final int[] visits = new int[twoLength];
760                     for (int k = 0; k < twoLength; k++) {
761                         tracker = intPool.borrowObject(KEY_TWO);
762                         visitCount = tracker.getValidateCount();
763                         visits[k] = visitCount;
764                         if (visitCount < cycleCount || visitCount > cycleCount + 1) {
765                             final StringBuilder sb = new StringBuilder("Visits:");
766                             for (int l = 0; l <= k; l++) {
767                                 sb.append(visits[l]).append(' ');
768                             }
769                             fail(formatSettings("TWO " + sb.toString(), "runs", runs, "lifo", lifo, "i", i, "j", j, "k",
770                                     k, "visitCount", visitCount, "cycleCount", cycleCount, "totalInstances",
771                                     totalInstances, zeroLength, oneLength, twoLength));
772                         }
773                     }
774                 }
775             }
776         }
777     }
778 
779     private String formatSettings(final String title, final String s, final int i, final String s0, final boolean b0, final String s1, final int i1, final String s2, final int i2, final String s3, final int i3,
780             final String s4, final int i4, final String s5, final int i5, final String s6, final int i6, final int zeroLength, final int oneLength, final int twoLength){
781         final StringBuilder sb = new StringBuilder(80);
782         sb.append(title).append(' ');
783         sb.append(s).append('=').append(i).append(' ');
784         sb.append(s0).append('=').append(b0).append(' ');
785         sb.append(s1).append('=').append(i1).append(' ');
786         sb.append(s2).append('=').append(i2).append(' ');
787         sb.append(s3).append('=').append(i3).append(' ');
788         sb.append(s4).append('=').append(i4).append(' ');
789         sb.append(s5).append('=').append(i5).append(' ');
790         sb.append(s6).append('=').append(i6).append(' ');
791         sb.append("Lengths=").append(zeroLength).append(',').append(oneLength).append(',').append(twoLength).append(' ');
792         return sb.toString();
793     }
794 
795     @Override
796     protected Object getNthObject(final Object key, final int n) {
797         return String.valueOf(key) + String.valueOf(n);
798     }
799 
800 
801     @Override
802     protected boolean isFifo() {
803         return false;
804     }
805 
806     @Override
807     protected boolean isLifo() {
808         return true;
809     }
810 
811     @SuppressWarnings("unchecked")
812     @Override
813     protected <E extends Exception> KeyedObjectPool<Object, Object> makeEmptyPool(final int minCapacity) {
814         final KeyedPooledObjectFactory<Object, Object> perKeyFactory = new SimplePerKeyFactory();
815         final GenericKeyedObjectPool<Object, Object> perKeyPool = new GenericKeyedObjectPool<>(perKeyFactory);
816         perKeyPool.setMaxTotalPerKey(minCapacity);
817         perKeyPool.setMaxIdlePerKey(minCapacity);
818         return (KeyedObjectPool<Object, Object>) perKeyPool;
819     }
820 
821     @Override
822     protected <E extends Exception> KeyedObjectPool<Object, Object> makeEmptyPool(final KeyedPooledObjectFactory<Object, Object> fac) {
823         return new GenericKeyedObjectPool<>(fac);
824     }
825 
826     @Override
827     protected Object makeKey(final int n) {
828         return String.valueOf(n);
829     }
830 
831     /**
832      * Kicks off {@code numThreads} test threads, each of which will go
833      * through {@code iterations} borrow-return cycles with random delay
834      * times &lt;= delay in between.
835      *
836      * @param <T>           Type of object in pool
837      * @param numThreads    Number of test threads
838      * @param iterations    Number of iterations for each thread
839      * @param delay         Maximum delay between iterations
840      * @param gkopPool      The keyed object pool to use
841      */
842     public <T> void runTestThreads(final int numThreads, final int iterations, final int delay,
843             final GenericKeyedObjectPool<String, T> gkopPool) {
844         final ArrayList<TestThread<T>> threads = new ArrayList<>();
845         for (int i = 0; i < numThreads; i++) {
846             final TestThread<T> testThread = new TestThread<>(gkopPool, iterations, delay);
847             threads.add(testThread);
848             final Thread t = new Thread(testThread);
849             t.start();
850         }
851         for (final TestThread<T> testThread : threads) {
852             while (!testThread.complete()) {
853                 Waiter.sleepQuietly(500L);
854             }
855             if (testThread.failed()) {
856                 fail("Thread failed: " + threads.indexOf(testThread) + "\n" + ExceptionUtils.getStackTrace(testThread.exception));
857             }
858         }
859     }
860 
861     @BeforeEach
862     public void setUp() {
863         simpleFactory = new SimpleFactory<>();
864         gkoPool = new GenericKeyedObjectPool<>(simpleFactory);
865     }
866 
867     @AfterEach
868     public void tearDownJmx() throws Exception {
869         super.tearDown();
870         final ObjectName jmxName = gkoPool.getJmxName();
871         final String poolName = Objects.toString(jmxName, null);
872         gkoPool.clear();
873         gkoPool.close();
874         gkoPool = null;
875         simpleFactory = null;
876 
877         final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
878         final Set<ObjectName> result = mbs.queryNames(new ObjectName(
879                 "org.apache.commoms.pool2:type=GenericKeyedObjectPool,*"),
880                 null);
881         // There should be no registered pools at this point
882         final int registeredPoolCount = result.size();
883         final StringBuilder msg = new StringBuilder("Current pool is: ");
884         msg.append(poolName);
885         msg.append("  Still open pools are: ");
886         for (final ObjectName name : result) {
887             // Clean these up ready for the next test
888             msg.append(name.toString());
889             msg.append(" created via\n");
890             msg.append(mbs.getAttribute(name, "CreationStackTrace"));
891             msg.append('\n');
892             mbs.unregisterMBean(name);
893         }
894         assertEquals(0, registeredPoolCount, msg.toString());
895     }
896 
897     @Test
898     public void testAppendStats() {
899         assertFalse(gkoPool.getMessageStatistics());
900         assertEquals("foo", gkoPool.appendStats("foo"));
901         try (final GenericKeyedObjectPool<?, ?> pool = new GenericKeyedObjectPool<>(new SimpleFactory<>())) {
902             pool.setMessagesStatistics(true);
903             assertNotEquals("foo", pool.appendStats("foo"));
904             pool.setMessagesStatistics(false);
905             assertEquals("foo", pool.appendStats("foo"));
906         }
907     }
908 
909     @Test
910     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
911     public void testBlockedKeyDoesNotBlockPool() throws Exception {
912         gkoPool.setBlockWhenExhausted(true);
913         gkoPool.setMaxWaitMillis(5000);
914         gkoPool.setMaxTotalPerKey(1);
915         gkoPool.setMaxTotal(-1);
916         gkoPool.borrowObject("one");
917         final long startMillis = System.currentTimeMillis();
918         // Needs to be in a separate thread as this will block
919         final Runnable simple = new SimpleTestThread<>(gkoPool, "one");
920         new Thread(simple).start();
921         // This should be almost instant. If it isn't it means this thread got
922         // stuck behind the thread created above which is bad.
923         // Give other thread a chance to start
924         Thread.sleep(1000);
925         gkoPool.borrowObject("two");
926         final long endMillis = System.currentTimeMillis();
927         // If it fails it will be more than 4000ms (5000 less the 1000 sleep)
928         // If it passes it should be almost instant
929         // Use 3000ms as the threshold - should avoid timing issues on most
930         // (all? platforms)
931         assertTrue(endMillis - startMillis < 4000,
932                 "Elapsed time: " + (endMillis - startMillis) + " should be less than 4000");
933 
934     }
935 
936     /*
937      * Note: This test relies on timing for correct execution. There *should* be
938      * enough margin for this to work correctly on most (all?) systems but be
939      * aware of this if you see a failure of this test.
940      */
941     @SuppressWarnings("rawtypes")
942     @Test
943     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
944     public void testBorrowObjectFairness() throws Exception {
945 
946         final int numThreads = 40;
947         final int maxTotal = 40;
948 
949         final GenericKeyedObjectPoolConfig<String> config = new GenericKeyedObjectPoolConfig<>();
950         config.setMaxTotalPerKey(maxTotal);
951         config.setFairness(true);
952         config.setLifo(false);
953         config.setMaxIdlePerKey(maxTotal);
954 
955         gkoPool = new GenericKeyedObjectPool<>(simpleFactory, config);
956 
957         // Exhaust the pool
958         final String[] objects = new String[maxTotal];
959         for (int i = 0; i < maxTotal; i++) {
960             objects[i] = gkoPool.borrowObject("0");
961         }
962 
963         // Start and park threads waiting to borrow objects
964         final TestThread[] threads = new TestThread[numThreads];
965         for (int i = 0; i < numThreads; i++) {
966             threads[i] = new TestThread<>(gkoPool, 1, 0, 2000, false, "0" + String.valueOf(i % maxTotal), "0");
967             final Thread t = new Thread(threads[i]);
968             t.start();
969             // Short delay to ensure threads start in correct order
970             try {
971                 Thread.sleep(10);
972             } catch (final InterruptedException e) {
973                 fail(e.toString());
974             }
975         }
976 
977         // Return objects, other threads should get served in order
978         for (int i = 0; i < maxTotal; i++) {
979             gkoPool.returnObject("0", objects[i]);
980         }
981 
982         // Wait for threads to finish
983         for (int i = 0; i < numThreads; i++) {
984             while (!threads[i].complete()) {
985                 Waiter.sleepQuietly(500L);
986             }
987             if (threads[i].failed()) {
988                 fail("Thread " + i + " failed: " + ExceptionUtils.getStackTrace(threads[i].exception));
989             }
990         }
991     }
992 
993     /**
994      * POOL-192
995      * Verify that clear(key) does not leak capacity.
996      *
997      * @throws Exception May occur in some failure modes
998      */
999     @Test
1000     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1001     public void testClear() throws Exception {
1002         gkoPool.setMaxTotal(2);
1003         gkoPool.setMaxTotalPerKey(2);
1004         gkoPool.setBlockWhenExhausted(false);
1005         gkoPool.addObject("one");
1006         gkoPool.addObject("one");
1007         assertEquals(2, gkoPool.getNumIdle());
1008         gkoPool.clear("one");
1009         assertEquals(0, gkoPool.getNumIdle());
1010         assertEquals(0, gkoPool.getNumIdle("one"));
1011         final String obj1 = gkoPool.borrowObject("one");
1012         final String obj2 = gkoPool.borrowObject("one");
1013         gkoPool.returnObject("one", obj1);
1014         gkoPool.returnObject("one", obj2);
1015         gkoPool.clear();
1016         assertEquals(0, gkoPool.getNumIdle());
1017         assertEquals(0, gkoPool.getNumIdle("one"));
1018         gkoPool.borrowObject("one");
1019         gkoPool.borrowObject("one");
1020         gkoPool.close();
1021     }
1022 
1023     /**
1024      * Test to make sure that clearOldest does not destroy instances that have been checked out.
1025      *
1026      * @throws Exception May occur in some failure modes
1027      */
1028     @Test
1029     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1030     public void testClearOldest() throws Exception {
1031         // Make destroy have some latency so clearOldest takes some time
1032         final WaiterFactory<String> waiterFactory = new WaiterFactory<>(0, 20, 0, 0, 0, 0, 50, 5, 0);
1033         try (final GenericKeyedObjectPool<String, Waiter> waiterPool = new GenericKeyedObjectPool<>(waiterFactory)) {
1034             waiterPool.setMaxTotalPerKey(5);
1035             waiterPool.setMaxTotal(50);
1036             waiterPool.setLifo(false);
1037             // Load the pool with idle instances - 5 each for 10 keys
1038             for (int i = 0; i < 10; i++) {
1039                 final String key = Integer.toString(i);
1040                 for (int j = 0; j < 5; j++) {
1041                     waiterPool.addObject(key);
1042                 }
1043                 // Make sure order is maintained
1044                 Thread.sleep(20);
1045             }
1046             // Now set up a race - one thread wants a new instance, triggering clearOldest
1047             // Other goes after an element on death row
1048             // See if we end up with dead man walking
1049             final SimpleTestThread<Waiter> t2 = new SimpleTestThread<>(waiterPool, "51");
1050             final Thread thread2 = new Thread(t2);
1051             thread2.start(); // Triggers clearOldest, killing all of the 0's and the 2 oldest 1's
1052             Thread.sleep(50); // Wait for clearOldest to kick off, but not long enough to reach the 1's
1053             final Waiter waiter = waiterPool.borrowObject("1");
1054             Thread.sleep(200); // Wait for execution to happen
1055             waiterPool.returnObject("1", waiter); // Will throw IllegalStateException if dead
1056         }
1057     }
1058 
1059     /**
1060        * POOL-391 Verify that when clear(key) is called with reuseCapacity true,
1061        * capacity freed is reused and allocated to most loaded pools.
1062        *
1063        * @throws Exception May occur in some failure modes
1064        */
1065       @Test
1066       public void testClearReuseCapacity() throws Exception {
1067           gkoPool.setMaxTotalPerKey(6);
1068           gkoPool.setMaxTotal(6);
1069           gkoPool.setMaxWait(Duration.ofSeconds(5));
1070           // Create one thread to wait on "one", two on "two", three on "three"
1071           final ArrayList<Thread> testThreads = new ArrayList<>();
1072           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "one")));
1073           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "two")));
1074           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "two")));
1075           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "three")));
1076           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "three")));
1077           testThreads.add(new Thread(new SimpleTestThread<>(gkoPool, "three")));
1078           // Borrow two each from "four", "five", "six" - using all capacity
1079           final String four = gkoPool.borrowObject("four");
1080           final String four2 = gkoPool.borrowObject("four");
1081           final String five = gkoPool.borrowObject("five");
1082           final String five2 = gkoPool.borrowObject("five");
1083           final String six = gkoPool.borrowObject("six");
1084           final String six2 = gkoPool.borrowObject("six");
1085           Thread.sleep(100);
1086           // Launch the waiters - all will be blocked waiting
1087           for (final Thread t : testThreads) {
1088               t.start();
1089           }
1090           Thread.sleep(100);
1091           // Return and clear the fours - at least one "three" should get served
1092           // Other should be a two or a three (three was most loaded)
1093           gkoPool.returnObject("four", four);
1094           gkoPool.returnObject("four", four2);
1095           gkoPool.clear("four");
1096           Thread.sleep(20);
1097           assertTrue(!testThreads.get(3).isAlive() || !testThreads.get(4).isAlive() || !testThreads.get(5).isAlive());
1098           // Now clear the fives
1099           gkoPool.returnObject("five", five);
1100           gkoPool.returnObject("five", five2);
1101           gkoPool.clear("five");
1102           Thread.sleep(20);
1103           // Clear the sixes
1104           gkoPool.returnObject("six", six);
1105           gkoPool.returnObject("six", six2);
1106           gkoPool.clear("six");
1107           Thread.sleep(20);
1108           for (final Thread t : testThreads) {
1109               assertFalse(t.isAlive());
1110           }
1111       }
1112 
1113     /**
1114      * POOL-391 Adapted from code in the JIRA ticket.
1115      */
1116     @Test
1117     @Timeout(value = 10000, unit = TimeUnit.MILLISECONDS)
1118     public void testClearUnblocksWaiters() throws Exception {
1119         final GenericKeyedObjectPoolConfig<Integer> config = new GenericKeyedObjectPoolConfig<>();
1120         config.setMaxTotalPerKey(1);
1121         config.setMaxTotal(1);
1122         config.setMaxWait(Duration.ofMillis(500));
1123         final GenericKeyedObjectPool<Integer, Integer> testPool = new GenericKeyedObjectPool<>(
1124                 new KeyedPooledObjectFactory<Integer, Integer>() {
1125                     @Override
1126                     public void activateObject(final Integer key, final PooledObject<Integer> p) {
1127                         // do nothing
1128                     }
1129 
1130                     @Override
1131                     public void destroyObject(final Integer key, final PooledObject<Integer> p)
1132                             throws InterruptedException {
1133                         Thread.sleep(10);
1134                     }
1135 
1136                     @Override
1137                     public PooledObject<Integer> makeObject(final Integer key) {
1138                         return new DefaultPooledObject<>(10);
1139                     }
1140 
1141                     @Override
1142                     public void passivateObject(final Integer key, final PooledObject<Integer> p) {
1143                         // do nothing
1144                     }
1145 
1146                     @Override
1147                     public boolean validateObject(final Integer key, final PooledObject<Integer> p) {
1148                         return true;
1149                     }
1150                 }, config);
1151         final Integer borrowKey = 10;
1152         final int iterations = 100;
1153         final ExecutorService executor = Executors.newFixedThreadPool(2);
1154         final Thread t = new Thread(() -> {
1155             try {
1156                 for (int i = 0; i < iterations; i++) {
1157                     final Integer integer = testPool.borrowObject(borrowKey);
1158                     testPool.returnObject(borrowKey, integer);
1159                     Thread.sleep(10);
1160                 }
1161 
1162             } catch (final Exception e) {
1163                 fail(e);
1164             }
1165         });
1166         final Thread t2 = new Thread(() -> {
1167             try {
1168                 for (int i = 0; i < iterations; i++) {
1169                     testPool.clear(borrowKey);
1170                     Thread.sleep(10);
1171                 }
1172             } catch (final Exception e) {
1173                 fail(e);
1174             }
1175         });
1176         final Future<?> f1 = executor.submit(t);
1177         final Future<?> f2 = executor.submit(t2);
1178         f2.get();
1179         f1.get();
1180     }
1181 
1182     // POOL-259
1183     @Test
1184     public void testClientWaitStats() throws Exception {
1185         final SimpleFactory<String> factory = new SimpleFactory<>();
1186         // Give makeObject a little latency
1187         factory.setMakeLatency(200);
1188         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(factory,
1189                 new GenericKeyedObjectPoolConfig<>())) {
1190             final String s = pool.borrowObject("one");
1191             // First borrow waits on create, so wait time should be at least 200 ms
1192             // Allow 100ms error in clock times
1193             assertTrue(pool.getMaxBorrowWaitTimeMillis() >= 100);
1194             assertTrue(pool.getMeanBorrowWaitTimeMillis() >= 100);
1195             pool.returnObject("one", s);
1196             pool.borrowObject("one");
1197             // Second borrow does not have to wait on create, average should be about 100
1198             assertTrue(pool.getMaxBorrowWaitTimeMillis() > 100);
1199             assertTrue(pool.getMeanBorrowWaitTimeMillis() < 200);
1200             assertTrue(pool.getMeanBorrowWaitTimeMillis() > 20);
1201         }
1202     }
1203 
1204 
1205     /**
1206      * Tests POOL-411, or least tries to reproduce the NPE, but does not.
1207      *
1208      * @throws TestException a test failure.
1209      */
1210     @Test
1211     public void testConcurrentBorrowAndClear() throws Exception {
1212         final int threadCount = 64;
1213         final int taskCount = 64;
1214         final int addCount = 1;
1215         final int borrowCycles = 1024;
1216         final int clearCycles = 1024;
1217         final boolean useYield = true;
1218 
1219         testConcurrentBorrowAndClear(threadCount, taskCount, addCount, borrowCycles, clearCycles, useYield);
1220     }
1221 
1222     /**
1223      * Tests POOL-411, or least tries to reproduce the NPE, but does not.
1224      *
1225      * @throws Exception a test failure.
1226      */
1227     private void testConcurrentBorrowAndClear(final int threadCount, final int taskCount, final int addCount, final int borrowCycles, final int clearCycles,
1228             final boolean useYield) throws Exception {
1229         final GenericKeyedObjectPoolConfig<String> config = new GenericKeyedObjectPoolConfig<>();
1230         final int maxTotalPerKey = borrowCycles + 1;
1231         config.setMaxTotalPerKey(threadCount);
1232         config.setMaxIdlePerKey(threadCount);
1233         config.setMaxTotal(maxTotalPerKey * threadCount);
1234         config.setBlockWhenExhausted(false); // pool exhausted indicates a bug in the test
1235 
1236         gkoPool = new GenericKeyedObjectPool<>(simpleFactory, config);
1237         final String key = "0";
1238         gkoPool.addObjects(Arrays.asList(key), threadCount);
1239         // all objects in the pool are now idle.
1240 
1241         final ExecutorService threadPool = Executors.newFixedThreadPool(threadCount);
1242         final List<Future<?>> futures = new ArrayList<>();
1243         try {
1244             for (int t = 0; t < taskCount; t++) {
1245                 futures.add(threadPool.submit(() -> {
1246                     for (int i = 0; i < clearCycles; i++) {
1247                         if (useYield) {
1248                             Thread.yield();
1249                         }
1250                         gkoPool.clear(key, true);
1251                         try {
1252                             gkoPool.addObjects(Arrays.asList(key), addCount);
1253                         } catch (Exception e) {
1254                             fail(e);
1255                         }
1256                     }
1257                 }));
1258                 futures.add(threadPool.submit(() -> {
1259                     try {
1260                         for (int i = 0; i < borrowCycles; i++) {
1261                             if (useYield) {
1262                                 Thread.yield();
1263                             }
1264                             final String pooled = gkoPool.borrowObject(key);
1265                             gkoPool.returnObject(key, pooled);
1266                         }
1267                     } catch (Exception e) {
1268                         fail(e);
1269                     }
1270                 }));
1271             }
1272             futures.forEach(f -> {
1273                 try {
1274                     f.get();
1275                 } catch (InterruptedException | ExecutionException e) {
1276                     fail(e);
1277                 }
1278             });
1279         } finally {
1280             threadPool.shutdownNow();
1281         }
1282     }
1283 
1284     /**
1285      * See https://issues.apache.org/jira/browse/POOL-411?focusedCommentId=17741156&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-17741156
1286      *
1287      * @throws TestException a test failure.
1288      */
1289     @Test
1290     public void testConcurrentBorrowAndClear_JiraComment17741156() throws Exception {
1291         final int threadCount = 2;
1292         final int taskCount = 2;
1293         final int addCount = 1;
1294         final int borrowCycles = 5_000;
1295         final int clearCycles = 5_000;
1296         final boolean useYield = false;
1297 
1298         testConcurrentBorrowAndClear(threadCount, taskCount, addCount, borrowCycles, clearCycles, useYield);
1299     }
1300 
1301     /**
1302      * POOL-231 - verify that concurrent invalidates of the same object do not
1303      * corrupt pool destroyCount.
1304      *
1305      * @throws Exception May occur in some failure modes
1306      */
1307     @Test
1308     public void testConcurrentInvalidate() throws Exception {
1309         // Get allObjects and idleObjects loaded with some instances
1310         final int nObjects = 1000;
1311         final String key = "one";
1312         gkoPool.setMaxTotal(nObjects);
1313         gkoPool.setMaxTotalPerKey(nObjects);
1314         gkoPool.setMaxIdlePerKey(nObjects);
1315         final String [] obj = new String[nObjects];
1316         for (int i = 0; i < nObjects; i++) {
1317             obj[i] = gkoPool.borrowObject(key);
1318         }
1319         for (int i = 0; i < nObjects; i++) {
1320             if (i % 2 == 0) {
1321                 gkoPool.returnObject(key, obj[i]);
1322             }
1323         }
1324         final int nThreads = 20;
1325         final int nIterations = 60;
1326         final InvalidateThread[] threads = new InvalidateThread[nThreads];
1327         // Randomly generated list of distinct invalidation targets
1328         final ArrayList<Integer> targets = new ArrayList<>();
1329         final Random random = new Random();
1330         for (int j = 0; j < nIterations; j++) {
1331             // Get a random invalidation target
1332             Integer targ = Integer.valueOf(random.nextInt(nObjects));
1333             while (targets.contains(targ)) {
1334                 targ = Integer.valueOf(random.nextInt(nObjects));
1335             }
1336             targets.add(targ);
1337             // Launch nThreads threads all trying to invalidate the target
1338             for (int i = 0; i < nThreads; i++) {
1339                 threads[i] = new InvalidateThread(gkoPool, key, obj[targ.intValue()]);
1340             }
1341             for (int i = 0; i < nThreads; i++) {
1342                 new Thread(threads[i]).start();
1343             }
1344             boolean done = false;
1345             while (!done) {
1346                 done = true;
1347                 for (int i = 0; i < nThreads; i++) {
1348                     done = done && threads[i].complete();
1349                 }
1350                 Thread.sleep(100);
1351             }
1352         }
1353         assertEquals(nIterations, gkoPool.getDestroyedCount());
1354     }
1355 
1356     @Test
1357     public void testConstructorNullFactory() {
1358         // add dummy assert (won't be invoked because of IAE) to avoid "unused" warning
1359         assertThrows(IllegalArgumentException.class,
1360                 () -> new GenericKeyedObjectPool<>(null));
1361     }
1362 
1363     @SuppressWarnings("deprecation")
1364     @Test
1365     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1366     public void testConstructors() {
1367 
1368         // Make constructor arguments all different from defaults
1369         final int maxTotalPerKey = 1;
1370         final int minIdle = 2;
1371         final Duration maxWaitDuration = Duration.ofMillis(3);
1372         final long maxWaitMillis = maxWaitDuration.toMillis();
1373         final int maxIdle = 4;
1374         final int maxTotal = 5;
1375         final long minEvictableIdleTimeMillis = 6;
1376         final int numTestsPerEvictionRun = 7;
1377         final boolean testOnBorrow = true;
1378         final boolean testOnReturn = true;
1379         final boolean testWhileIdle = true;
1380         final long timeBetweenEvictionRunsMillis = 8;
1381         final boolean blockWhenExhausted = false;
1382         final boolean lifo = false;
1383         final KeyedPooledObjectFactory<Object, Object> dummyFactory = new DummyFactory();
1384 
1385         try (GenericKeyedObjectPool<Object, Object> objPool = new GenericKeyedObjectPool<>(dummyFactory)) {
1386             assertEquals(GenericKeyedObjectPoolConfig.DEFAULT_MAX_TOTAL_PER_KEY, objPool.getMaxTotalPerKey());
1387             assertEquals(GenericKeyedObjectPoolConfig.DEFAULT_MAX_IDLE_PER_KEY, objPool.getMaxIdlePerKey());
1388             assertEquals(BaseObjectPoolConfig.DEFAULT_MAX_WAIT_MILLIS, objPool.getMaxWaitMillis());
1389             assertEquals(GenericKeyedObjectPoolConfig.DEFAULT_MIN_IDLE_PER_KEY, objPool.getMinIdlePerKey());
1390             assertEquals(GenericKeyedObjectPoolConfig.DEFAULT_MAX_TOTAL, objPool.getMaxTotal());
1391             //
1392             assertEquals(BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,
1393                     objPool.getMinEvictableIdleTimeMillis());
1394             assertEquals(BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_TIME,
1395                     objPool.getMinEvictableIdleTime());
1396             assertEquals(BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_TIME,
1397                     objPool.getMinEvictableIdleDuration());
1398             //
1399             assertEquals(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN, objPool.getNumTestsPerEvictionRun());
1400             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW),
1401                     Boolean.valueOf(objPool.getTestOnBorrow()));
1402             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN),
1403                     Boolean.valueOf(objPool.getTestOnReturn()));
1404             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE),
1405                     Boolean.valueOf(objPool.getTestWhileIdle()));
1406             //
1407             assertEquals(BaseObjectPoolConfig.DEFAULT_DURATION_BETWEEN_EVICTION_RUNS,
1408                     objPool.getDurationBetweenEvictionRuns());
1409             assertEquals(BaseObjectPoolConfig.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,
1410                     objPool.getTimeBetweenEvictionRunsMillis());
1411             assertEquals(BaseObjectPoolConfig.DEFAULT_TIME_BETWEEN_EVICTION_RUNS,
1412                     objPool.getTimeBetweenEvictionRuns());
1413             //
1414             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_BLOCK_WHEN_EXHAUSTED),
1415                     Boolean.valueOf(objPool.getBlockWhenExhausted()));
1416             assertEquals(Boolean.valueOf(BaseObjectPoolConfig.DEFAULT_LIFO), Boolean.valueOf(objPool.getLifo()));
1417         }
1418 
1419         final GenericKeyedObjectPoolConfig<Object> config = new GenericKeyedObjectPoolConfig<>();
1420         config.setLifo(lifo);
1421         config.setMaxTotalPerKey(maxTotalPerKey);
1422         config.setMaxIdlePerKey(maxIdle);
1423         config.setMinIdlePerKey(minIdle);
1424         config.setMaxTotal(maxTotal);
1425         config.setMaxWait(maxWaitDuration);
1426         config.setMinEvictableIdleDuration(Duration.ofMillis(minEvictableIdleTimeMillis));
1427         config.setMinEvictableIdleTime(Duration.ofMillis(minEvictableIdleTimeMillis));
1428         config.setNumTestsPerEvictionRun(numTestsPerEvictionRun);
1429         config.setTestOnBorrow(testOnBorrow);
1430         config.setTestOnReturn(testOnReturn);
1431         config.setTestWhileIdle(testWhileIdle);
1432         config.setTimeBetweenEvictionRuns(Duration.ofMillis(timeBetweenEvictionRunsMillis));
1433         config.setBlockWhenExhausted(blockWhenExhausted);
1434         try (GenericKeyedObjectPool<Object, Object> objPool = new GenericKeyedObjectPool<>(dummyFactory, config)) {
1435             assertEquals(maxTotalPerKey, objPool.getMaxTotalPerKey());
1436             assertEquals(maxIdle, objPool.getMaxIdlePerKey());
1437             assertEquals(maxWaitDuration, objPool.getMaxWaitDuration());
1438             assertEquals(maxWaitMillis, objPool.getMaxWaitMillis());
1439             assertEquals(minIdle, objPool.getMinIdlePerKey());
1440             assertEquals(maxTotal, objPool.getMaxTotal());
1441             assertEquals(minEvictableIdleTimeMillis, objPool.getMinEvictableIdleDuration().toMillis());
1442             assertEquals(minEvictableIdleTimeMillis, objPool.getMinEvictableIdleTimeMillis());
1443             assertEquals(minEvictableIdleTimeMillis, objPool.getMinEvictableIdleTime().toMillis());
1444             assertEquals(numTestsPerEvictionRun, objPool.getNumTestsPerEvictionRun());
1445             assertEquals(Boolean.valueOf(testOnBorrow), Boolean.valueOf(objPool.getTestOnBorrow()));
1446             assertEquals(Boolean.valueOf(testOnReturn), Boolean.valueOf(objPool.getTestOnReturn()));
1447             assertEquals(Boolean.valueOf(testWhileIdle), Boolean.valueOf(objPool.getTestWhileIdle()));
1448             assertEquals(timeBetweenEvictionRunsMillis, objPool.getDurationBetweenEvictionRuns().toMillis());
1449             assertEquals(timeBetweenEvictionRunsMillis, objPool.getTimeBetweenEvictionRunsMillis());
1450             assertEquals(timeBetweenEvictionRunsMillis, objPool.getTimeBetweenEvictionRuns().toMillis());
1451             assertEquals(Boolean.valueOf(blockWhenExhausted), Boolean.valueOf(objPool.getBlockWhenExhausted()));
1452             assertEquals(Boolean.valueOf(lifo), Boolean.valueOf(objPool.getLifo()));
1453         }
1454     }
1455 
1456     /**
1457      * JIRA: POOL-270 - make sure constructor correctly sets run
1458      * frequency of evictor timer.
1459      */
1460     @Test
1461     public void testContructorEvictionConfig() throws Exception {
1462         final GenericKeyedObjectPoolConfig<String> config = new GenericKeyedObjectPoolConfig<>();
1463         config.setTimeBetweenEvictionRuns(Duration.ofMillis(500));
1464         config.setMinEvictableIdleDuration(Duration.ofMillis(50));
1465         config.setNumTestsPerEvictionRun(5);
1466         try (final GenericKeyedObjectPool<String, String> p = new GenericKeyedObjectPool<>(simpleFactory, config)) {
1467             for (int i = 0; i < 5; i++) {
1468                 p.addObject("one");
1469             }
1470             Waiter.sleepQuietly(100);
1471             assertEquals(5, p.getNumIdle("one"));
1472             Waiter.sleepQuietly(500);
1473             assertEquals(0, p.getNumIdle("one"));
1474         }
1475     }
1476 
1477     /**
1478      * Verifies that when a factory's makeObject produces instances that are not
1479      * discernible by equals, the pool can handle them.
1480      *
1481      * JIRA: POOL-283
1482      */
1483     @Test
1484     public void testEqualsIndiscernible() throws Exception {
1485         final HashSetFactory factory = new HashSetFactory();
1486         try (final GenericKeyedObjectPool<String, HashSet<String>> pool = new GenericKeyedObjectPool<>(factory,
1487                 new GenericKeyedObjectPoolConfig<>())) {
1488             final HashSet<String> s1 = pool.borrowObject("a");
1489             final HashSet<String> s2 = pool.borrowObject("a");
1490             pool.returnObject("a", s1);
1491             pool.returnObject("a", s2);
1492         }
1493     }
1494 
1495     @Test
1496     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1497     public void testEviction() throws Exception {
1498         gkoPool.setMaxIdlePerKey(500);
1499         gkoPool.setMaxTotalPerKey(500);
1500         gkoPool.setNumTestsPerEvictionRun(100);
1501         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(250));
1502         gkoPool.setDurationBetweenEvictionRuns(Duration.ofMillis(500));
1503 
1504         final String[] active = new String[500];
1505         for (int i = 0; i < 500; i++) {
1506             active[i] = gkoPool.borrowObject("");
1507         }
1508         for (int i = 0; i < 500; i++) {
1509             gkoPool.returnObject("", active[i]);
1510         }
1511 
1512         Waiter.sleepQuietly(1000L);
1513         assertTrue(gkoPool.getNumIdle("") < 500, "Should be less than 500 idle, found " + gkoPool.getNumIdle(""));
1514         Waiter.sleepQuietly(600L);
1515         assertTrue(gkoPool.getNumIdle("") < 400, "Should be less than 400 idle, found " + gkoPool.getNumIdle(""));
1516         Waiter.sleepQuietly(600L);
1517         assertTrue(gkoPool.getNumIdle("") < 300, "Should be less than 300 idle, found " + gkoPool.getNumIdle(""));
1518         Waiter.sleepQuietly(600L);
1519         assertTrue(gkoPool.getNumIdle("") < 200, "Should be less than 200 idle, found " + gkoPool.getNumIdle(""));
1520         Waiter.sleepQuietly(600L);
1521         assertTrue(gkoPool.getNumIdle("") < 100, "Should be less than 100 idle, found " + gkoPool.getNumIdle(""));
1522         Waiter.sleepQuietly(600L);
1523         assertEquals(0, gkoPool.getNumIdle(""), "Should be zero idle, found " + gkoPool.getNumIdle(""));
1524 
1525         for (int i = 0; i < 500; i++) {
1526             active[i] = gkoPool.borrowObject("");
1527         }
1528         for (int i = 0; i < 500; i++) {
1529             gkoPool.returnObject("", active[i]);
1530         }
1531 
1532         Waiter.sleepQuietly(1000L);
1533         assertTrue(gkoPool.getNumIdle("") < 500, "Should be less than 500 idle, found " + gkoPool.getNumIdle(""));
1534         Waiter.sleepQuietly(600L);
1535         assertTrue(gkoPool.getNumIdle("") < 400, "Should be less than 400 idle, found " + gkoPool.getNumIdle(""));
1536         Waiter.sleepQuietly(600L);
1537         assertTrue(gkoPool.getNumIdle("") < 300, "Should be less than 300 idle, found " + gkoPool.getNumIdle(""));
1538         Waiter.sleepQuietly(600L);
1539         assertTrue(gkoPool.getNumIdle("") < 200, "Should be less than 200 idle, found " + gkoPool.getNumIdle(""));
1540         Waiter.sleepQuietly(600L);
1541         assertTrue(gkoPool.getNumIdle("") < 100, "Should be less than 100 idle, found " + gkoPool.getNumIdle(""));
1542         Waiter.sleepQuietly(600L);
1543         assertEquals(0, gkoPool.getNumIdle(""), "Should be zero idle, found " + gkoPool.getNumIdle(""));
1544     }
1545 
1546     @Test
1547     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1548     public void testEviction2() throws Exception {
1549         gkoPool.setMaxIdlePerKey(500);
1550         gkoPool.setMaxTotalPerKey(500);
1551         gkoPool.setNumTestsPerEvictionRun(100);
1552         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(500));
1553         gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(500));
1554 
1555         final String[] active = new String[500];
1556         final String[] active2 = new String[500];
1557         for (int i = 0; i < 500; i++) {
1558             active[i] = gkoPool.borrowObject("");
1559             active2[i] = gkoPool.borrowObject("2");
1560         }
1561         for (int i = 0; i < 500; i++) {
1562             gkoPool.returnObject("", active[i]);
1563             gkoPool.returnObject("2", active2[i]);
1564         }
1565 
1566         Waiter.sleepQuietly(1100L);
1567         assertTrue(gkoPool.getNumIdle() < 1000, "Should be less than 1000 idle, found " + gkoPool.getNumIdle());
1568         final long sleepMillisPart2 = 600L;
1569         Waiter.sleepQuietly(sleepMillisPart2);
1570         assertTrue(gkoPool.getNumIdle() < 900, "Should be less than 900 idle, found " + gkoPool.getNumIdle());
1571         Waiter.sleepQuietly(sleepMillisPart2);
1572         assertTrue(gkoPool.getNumIdle() < 800, "Should be less than 800 idle, found " + gkoPool.getNumIdle());
1573         Waiter.sleepQuietly(sleepMillisPart2);
1574         assertTrue(gkoPool.getNumIdle() < 700, "Should be less than 700 idle, found " + gkoPool.getNumIdle());
1575         Waiter.sleepQuietly(sleepMillisPart2);
1576         assertTrue(gkoPool.getNumIdle() < 600, "Should be less than 600 idle, found " + gkoPool.getNumIdle());
1577         Waiter.sleepQuietly(sleepMillisPart2);
1578         assertTrue(gkoPool.getNumIdle() < 500, "Should be less than 500 idle, found " + gkoPool.getNumIdle());
1579         Waiter.sleepQuietly(sleepMillisPart2);
1580         assertTrue(gkoPool.getNumIdle() < 400, "Should be less than 400 idle, found " + gkoPool.getNumIdle());
1581         Waiter.sleepQuietly(sleepMillisPart2);
1582         assertTrue(gkoPool.getNumIdle() < 300, "Should be less than 300 idle, found " + gkoPool.getNumIdle());
1583         Waiter.sleepQuietly(sleepMillisPart2);
1584         assertTrue(gkoPool.getNumIdle() < 200, "Should be less than 200 idle, found " + gkoPool.getNumIdle());
1585         Waiter.sleepQuietly(sleepMillisPart2);
1586         assertTrue(gkoPool.getNumIdle() < 100, "Should be less than 100 idle, found " + gkoPool.getNumIdle());
1587         Waiter.sleepQuietly(sleepMillisPart2);
1588         assertEquals(0, gkoPool.getNumIdle(), "Should be zero idle, found " + gkoPool.getNumIdle());
1589     }
1590 
1591     /**
1592      * Test to make sure evictor visits least recently used objects first,
1593      * regardless of FIFO/LIFO
1594      *
1595      * JIRA: POOL-86
1596      *
1597      * @throws Exception May occur in some failure modes
1598      */
1599     @Test
1600     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1601     public void testEvictionOrder() throws Exception {
1602         checkEvictionOrder(false);
1603         checkEvictionOrder(true);
1604     }
1605 
1606     // POOL-326
1607     @Test
1608     public void testEvictorClearOldestRace() throws Exception {
1609         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(100));
1610         gkoPool.setNumTestsPerEvictionRun(1);
1611 
1612         // Introduce latency between when evictor starts looking at an instance and when
1613         // it decides to destroy it
1614         gkoPool.setEvictionPolicy(new SlowEvictionPolicy<>(1000));
1615 
1616         // Borrow an instance
1617         final String val = gkoPool.borrowObject("foo");
1618 
1619         // Add another idle one
1620         gkoPool.addObject("foo");
1621 
1622         // Sleep long enough so idle one is eligible for eviction
1623         Thread.sleep(1000);
1624 
1625         // Start evictor and race with clearOldest
1626         gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(10));
1627 
1628         // Wait for evictor to start
1629         Thread.sleep(100);
1630         gkoPool.clearOldest();
1631 
1632         // Wait for slow evictor to complete
1633         Thread.sleep(1500);
1634 
1635         // See if we get NPE on return (POOL-326)
1636         gkoPool.returnObject("foo", val);
1637     }
1638 
1639     /**
1640      * Verifies that the evictor visits objects in expected order
1641      * and frequency.
1642      *
1643      * @throws Exception May occur in some failure modes
1644      */
1645     @Test
1646     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1647     public void testEvictorVisiting() throws Exception {
1648         checkEvictorVisiting(true);
1649         checkEvictorVisiting(false);
1650     }
1651 
1652     @Test
1653     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1654     public void testExceptionInValidationDuringEviction() throws Exception {
1655         gkoPool.setMaxIdlePerKey(1);
1656         gkoPool.setMinEvictableIdleTime(Duration.ZERO);
1657         gkoPool.setTestWhileIdle(true);
1658 
1659         final String obj = gkoPool.borrowObject("one");
1660         gkoPool.returnObject("one", obj);
1661 
1662         simpleFactory.setThrowExceptionOnValidate(true);
1663         assertThrows(RuntimeException.class, gkoPool::evict);
1664         assertEquals(0, gkoPool.getNumActive());
1665         assertEquals(0, gkoPool.getNumIdle());
1666     }
1667 
1668 
1669     @Test
1670     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1671     public void testExceptionOnActivateDuringBorrow() throws Exception {
1672         final String obj1 = gkoPool.borrowObject("one");
1673         final String obj2 = gkoPool.borrowObject("one");
1674         gkoPool.returnObject("one", obj1);
1675         gkoPool.returnObject("one", obj2);
1676         simpleFactory.setThrowExceptionOnActivate(true);
1677         simpleFactory.setEvenValid(false);
1678         // Activation will now throw every other time
1679         // First attempt throws, but loop continues and second succeeds
1680         final String obj = gkoPool.borrowObject("one");
1681         assertEquals(1, gkoPool.getNumActive("one"));
1682         assertEquals(0, gkoPool.getNumIdle("one"));
1683         assertEquals(1, gkoPool.getNumActive());
1684         assertEquals(0, gkoPool.getNumIdle());
1685 
1686         gkoPool.returnObject("one", obj);
1687         simpleFactory.setValid(false);
1688         // Validation will now fail on activation when borrowObject returns
1689         // an idle instance, and then when attempting to create a new instance
1690         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject("one"));
1691         assertEquals(0, gkoPool.getNumActive("one"));
1692         assertEquals(0, gkoPool.getNumIdle("one"));
1693         assertEquals(0, gkoPool.getNumActive());
1694         assertEquals(0, gkoPool.getNumIdle());
1695     }
1696 
1697     @Test
1698     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1699     public void testExceptionOnDestroyDuringBorrow() throws Exception {
1700         simpleFactory.setThrowExceptionOnDestroy(true);
1701         simpleFactory.setValidationEnabled(true);
1702         gkoPool.setTestOnBorrow(true);
1703         gkoPool.borrowObject("one");
1704         simpleFactory.setValid(false); // Make validation fail on next borrow attempt
1705         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject("one"));
1706         assertEquals(1, gkoPool.getNumActive("one"));
1707         assertEquals(0, gkoPool.getNumIdle("one"));
1708         assertEquals(1, gkoPool.getNumActive());
1709         assertEquals(0, gkoPool.getNumIdle());
1710     }
1711 
1712     @Test
1713     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1714     public void testExceptionOnDestroyDuringReturn() throws Exception {
1715         simpleFactory.setThrowExceptionOnDestroy(true);
1716         simpleFactory.setValidationEnabled(true);
1717         gkoPool.setTestOnReturn(true);
1718         final String obj1 = gkoPool.borrowObject("one");
1719         gkoPool.borrowObject("one");
1720         simpleFactory.setValid(false); // Make validation fail
1721         gkoPool.returnObject("one", obj1);
1722         assertEquals(1, gkoPool.getNumActive("one"));
1723         assertEquals(0, gkoPool.getNumIdle("one"));
1724         assertEquals(1, gkoPool.getNumActive());
1725         assertEquals(0, gkoPool.getNumIdle());
1726     }
1727 
1728     @Test
1729     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1730     public void testExceptionOnPassivateDuringReturn() throws Exception {
1731         final String obj = gkoPool.borrowObject("one");
1732         simpleFactory.setThrowExceptionOnPassivate(true);
1733         gkoPool.returnObject("one", obj);
1734         assertEquals(0, gkoPool.getNumIdle());
1735         gkoPool.close();
1736     }
1737 
1738     @Test
1739     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1740     public void testFIFO() throws Exception {
1741         gkoPool.setLifo(false);
1742         final String key = "key";
1743         gkoPool.addObject(key); // "key0"
1744         gkoPool.addObject(key); // "key1"
1745         gkoPool.addObject(key); // "key2"
1746         assertEquals("key0", gkoPool.borrowObject(key), "Oldest");
1747         assertEquals("key1", gkoPool.borrowObject(key), "Middle");
1748         assertEquals("key2", gkoPool.borrowObject(key), "Youngest");
1749         final String s = gkoPool.borrowObject(key);
1750         assertEquals("key3", s, "new-3");
1751         gkoPool.returnObject(key, s);
1752         assertEquals(s, gkoPool.borrowObject(key), "returned");
1753         assertEquals("key4", gkoPool.borrowObject(key), "new-4");
1754     }
1755 
1756     @Test
1757     @Timeout(value = 60, unit = TimeUnit.SECONDS)
1758     public void testGetKeys() throws Exception {
1759         gkoPool.addObject("one");
1760         assertEquals(1, gkoPool.getKeys().size());
1761         gkoPool.addObject("two");
1762         assertEquals(2, gkoPool.getKeys().size());
1763         gkoPool.clear("one");
1764         assertEquals(1, gkoPool.getKeys().size());
1765         assertEquals("two", gkoPool.getKeys().get(0));
1766         gkoPool.clear();
1767     }
1768 
1769     @Test
1770     public void testGetStatsString() {
1771         assertNotNull(gkoPool.getStatsString());
1772     }
1773 
1774     /**
1775      * Verify that threads waiting on a depleted pool get served when a checked out object is
1776      * invalidated.
1777      *
1778      * JIRA: POOL-240
1779      * @throws InterruptedException Custom exception
1780      * @throws Exception May occur in some failure modes
1781      */
1782     @Test
1783     public void testInvalidateFreesCapacity() throws Exception {
1784         final SimpleFactory<String> factory = new SimpleFactory<>();
1785         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(factory)) {
1786             pool.setMaxTotalPerKey(2);
1787             pool.setMaxWaitMillis(500);
1788             // Borrow an instance and hold if for 5 seconds
1789             final WaitingTestThread thread1 = new WaitingTestThread(pool, "one", 5000);
1790             thread1.start();
1791             // Borrow another instance
1792             final String obj = pool.borrowObject("one");
1793             // Launch another thread - will block, but fail in 500 ms
1794             final WaitingTestThread thread2 = new WaitingTestThread(pool, "one", 100);
1795             thread2.start();
1796             // Invalidate the object borrowed by this thread - should allow thread2 to create
1797             Thread.sleep(20);
1798             pool.invalidateObject("one", obj);
1799             Thread.sleep(600); // Wait for thread2 to timeout
1800             if (thread2.thrown != null) {
1801                 fail(thread2.thrown.toString());
1802             }
1803         }
1804     }
1805 
1806     @Test
1807     public void testInvalidateFreesCapacityForOtherKeys() throws Exception {
1808         gkoPool.setMaxTotal(1);
1809         gkoPool.setMaxWait(Duration.ofMillis(500));
1810         final Thread borrower = new Thread(new SimpleTestThread<>(gkoPool, "two"));
1811         final String obj = gkoPool.borrowObject("one");
1812         borrower.start();  // Will block
1813         Thread.sleep(100);  // Make sure borrower has started
1814         gkoPool.invalidateObject("one", obj);  // Should free capacity to serve the other key
1815         Thread.sleep(20);  // Should have been served by now
1816         assertFalse(borrower.isAlive());
1817     }
1818 
1819     @Test
1820     public void testInvalidateMissingKey() throws Exception {
1821         assertThrows(IllegalStateException.class, () -> gkoPool.invalidateObject("UnknownKey", "Ignored"));
1822     }
1823 
1824     @ParameterizedTest
1825     @EnumSource(DestroyMode.class)
1826     public void testInvalidateMissingKeyForDestroyMode(final DestroyMode destroyMode) throws Exception {
1827         assertThrows(IllegalStateException.class, () -> gkoPool.invalidateObject("UnknownKey", "Ignored", destroyMode));
1828     }
1829 
1830     /**
1831      * Verify that threads blocked waiting on a depleted pool get served when a checked out instance
1832      * is invalidated.
1833      *
1834      * JIRA: POOL-240
1835      *
1836      * @throws Exception May occur in some failure modes
1837      */
1838     @Test
1839     public void testInvalidateWaiting()
1840             throws Exception {
1841 
1842         final GenericKeyedObjectPoolConfig<Object> config = new GenericKeyedObjectPoolConfig<>();
1843         config.setMaxTotal(2);
1844         config.setBlockWhenExhausted(true);
1845         config.setMinIdlePerKey(0);
1846         config.setMaxWait(Duration.ofMillis(-1));
1847         config.setNumTestsPerEvictionRun(Integer.MAX_VALUE); // always test all idle objects
1848         config.setTestOnBorrow(true);
1849         config.setTestOnReturn(false);
1850         config.setTestWhileIdle(true);
1851         config.setTimeBetweenEvictionRuns(Duration.ofMillis(-1));
1852 
1853         try (final GenericKeyedObjectPool<Integer, Object> pool = new GenericKeyedObjectPool<>(new ObjectFactory(), config)) {
1854 
1855             // Allocate both objects with this thread
1856             pool.borrowObject(Integer.valueOf(1)); // object1
1857             final Object object2 = pool.borrowObject(Integer.valueOf(1));
1858 
1859             // Cause a thread to block waiting for an object
1860             final ExecutorService executorService = Executors.newSingleThreadExecutor(new DaemonThreadFactory());
1861             final Semaphore signal = new Semaphore(0);
1862             final Future<Exception> result = executorService.submit(() -> {
1863                 try {
1864                     signal.release();
1865                     final Object object3 = pool.borrowObject(Integer.valueOf(1));
1866                     pool.returnObject(Integer.valueOf(1), object3);
1867                     signal.release();
1868                 } catch (final Exception e1) {
1869                     return e1;
1870                 } catch (final Throwable e2) {
1871                     return new Exception(e2);
1872                 }
1873 
1874                 return null;
1875             });
1876 
1877             // Wait for the thread to start
1878             assertTrue(signal.tryAcquire(5, TimeUnit.SECONDS));
1879 
1880             // Attempt to ensure that test thread is blocked waiting for an object
1881             Thread.sleep(500);
1882 
1883             pool.invalidateObject(Integer.valueOf(1), object2);
1884 
1885             assertTrue(signal.tryAcquire(2, TimeUnit.SECONDS),"Call to invalidateObject did not unblock pool waiters.");
1886 
1887             if (result.get() != null) {
1888                 throw new AssertionError(result.get());
1889             }
1890         }
1891     }
1892 
1893     /**
1894      * Ensure the pool is registered.
1895      */
1896     @Test
1897     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1898     public void testJmxRegistration() {
1899         final ObjectName oname = gkoPool.getJmxName();
1900         final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
1901         final Set<ObjectName> result = mbs.queryNames(oname, null);
1902         assertEquals(1, result.size());
1903     }
1904 
1905     @Test
1906     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1907     public void testLIFO() throws Exception {
1908         gkoPool.setLifo(true);
1909         final String key = "key";
1910         gkoPool.addObject(key); // "key0"
1911         gkoPool.addObject(key); // "key1"
1912         gkoPool.addObject(key); // "key2"
1913         assertEquals("key2", gkoPool.borrowObject(key), "Youngest");
1914         assertEquals("key1", gkoPool.borrowObject(key), "Middle");
1915         assertEquals("key0", gkoPool.borrowObject(key), "Oldest");
1916         final String s = gkoPool.borrowObject(key);
1917         assertEquals("key3", s, "new-3");
1918         gkoPool.returnObject(key, s);
1919         assertEquals(s, gkoPool.borrowObject(key), "returned");
1920         assertEquals("key4", gkoPool.borrowObject(key), "new-4");
1921     }
1922 
1923     /**
1924      * Verifies that threads that get parked waiting for keys not in use
1925      * when the pool is at maxTotal eventually get served.
1926      *
1927      * @throws Exception May occur in some failure modes
1928      */
1929     @Test
1930     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1931     public void testLivenessPerKey() throws Exception {
1932         gkoPool.setMaxIdlePerKey(3);
1933         gkoPool.setMaxTotal(3);
1934         gkoPool.setMaxTotalPerKey(3);
1935         gkoPool.setMaxWaitMillis(3000);  // Really a timeout for the test
1936 
1937         // Check out and briefly hold 3 "1"s
1938         final WaitingTestThread t1 = new WaitingTestThread(gkoPool, "1", 100);
1939         final WaitingTestThread t2 = new WaitingTestThread(gkoPool, "1", 100);
1940         final WaitingTestThread t3 = new WaitingTestThread(gkoPool, "1", 100);
1941         t1.start();
1942         t2.start();
1943         t3.start();
1944 
1945         // Try to get a "2" while all capacity is in use.
1946         // Thread will park waiting on empty queue. Verify it gets served.
1947         gkoPool.borrowObject("2");
1948     }
1949 
1950     /**
1951      * Verify that factory exceptions creating objects do not corrupt per key create count.
1952      *
1953      * JIRA: POOL-243
1954      *
1955      * @throws Exception May occur in some failure modes
1956      */
1957     @Test
1958     public void testMakeObjectException() throws Exception {
1959         final SimpleFactory<String> factory = new SimpleFactory<>();
1960         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(factory)) {
1961             pool.setMaxTotalPerKey(1);
1962             pool.setBlockWhenExhausted(false);
1963             factory.exceptionOnCreate = true;
1964             assertThrows(Exception.class, () -> pool.borrowObject("One"));
1965             factory.exceptionOnCreate = false;
1966             pool.borrowObject("One");
1967         }
1968     }
1969 
1970     /**
1971      * Test case for POOL-180.
1972      */
1973     @Test
1974     @Timeout(value = 200000, unit = TimeUnit.MILLISECONDS)
1975     public void testMaxActivePerKeyExceeded() {
1976         final WaiterFactory<String> waiterFactory = new WaiterFactory<>(0, 20, 0, 0, 0, 0, 8, 5, 0);
1977         // TODO Fix this. Can't use local pool since runTestThreads uses the
1978         // protected pool field
1979         try (final GenericKeyedObjectPool<String, Waiter> waiterPool = new GenericKeyedObjectPool<>(waiterFactory)) {
1980             waiterPool.setMaxTotalPerKey(5);
1981             waiterPool.setMaxTotal(8);
1982             waiterPool.setTestOnBorrow(true);
1983             waiterPool.setMaxIdlePerKey(5);
1984             waiterPool.setMaxWaitMillis(-1);
1985             runTestThreads(20, 300, 250, waiterPool);
1986         }
1987     }
1988 
1989     @Test
1990     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
1991     public void testMaxIdle() throws Exception {
1992         gkoPool.setMaxTotalPerKey(100);
1993         gkoPool.setMaxIdlePerKey(8);
1994         final String[] active = new String[100];
1995         for (int i = 0; i < 100; i++) {
1996             active[i] = gkoPool.borrowObject("");
1997         }
1998         assertEquals(100, gkoPool.getNumActive(""));
1999         assertEquals(0, gkoPool.getNumIdle(""));
2000         for (int i = 0; i < 100; i++) {
2001             gkoPool.returnObject("", active[i]);
2002             assertEquals(99 - i, gkoPool.getNumActive(""));
2003             assertEquals(i < 8 ? i + 1 : 8, gkoPool.getNumIdle(""));
2004         }
2005 
2006         for (int i = 0; i < 100; i++) {
2007             active[i] = gkoPool.borrowObject("a");
2008         }
2009         assertEquals(100, gkoPool.getNumActive("a"));
2010         assertEquals(0, gkoPool.getNumIdle("a"));
2011         for (int i = 0; i < 100; i++) {
2012             gkoPool.returnObject("a", active[i]);
2013             assertEquals(99 - i, gkoPool.getNumActive("a"));
2014             assertEquals(i < 8 ? i + 1 : 8, gkoPool.getNumIdle("a"));
2015         }
2016 
2017         // total number of idle instances is twice maxIdle
2018         assertEquals(16, gkoPool.getNumIdle());
2019         // Each pool is at the sup
2020         assertEquals(8, gkoPool.getNumIdle(""));
2021         assertEquals(8, gkoPool.getNumIdle("a"));
2022 
2023     }
2024 
2025     @Test
2026     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2027     public void testMaxTotal() throws Exception {
2028         gkoPool.setMaxTotalPerKey(2);
2029         gkoPool.setMaxTotal(3);
2030         gkoPool.setBlockWhenExhausted(false);
2031 
2032         final String o1 = gkoPool.borrowObject("a");
2033         assertNotNull(o1);
2034         final String o2 = gkoPool.borrowObject("a");
2035         assertNotNull(o2);
2036         final String o3 = gkoPool.borrowObject("b");
2037         assertNotNull(o3);
2038         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject("c"));
2039 
2040         assertEquals(0, gkoPool.getNumIdle());
2041 
2042         gkoPool.returnObject("b", o3);
2043         assertEquals(1, gkoPool.getNumIdle());
2044         assertEquals(1, gkoPool.getNumIdle("b"));
2045 
2046         final Object o4 = gkoPool.borrowObject("b");
2047         assertNotNull(o4);
2048         assertEquals(0, gkoPool.getNumIdle());
2049         assertEquals(0, gkoPool.getNumIdle("b"));
2050 
2051         gkoPool.setMaxTotal(4);
2052         final Object o5 = gkoPool.borrowObject("b");
2053         assertNotNull(o5);
2054 
2055         assertEquals(2, gkoPool.getNumActive("a"));
2056         assertEquals(2, gkoPool.getNumActive("b"));
2057         assertEquals(gkoPool.getMaxTotal(),
2058                 gkoPool.getNumActive("b") + gkoPool.getNumActive("b"));
2059         assertEquals(gkoPool.getNumActive(),
2060                 gkoPool.getMaxTotal());
2061     }
2062 
2063     /**
2064      * Verifies that maxTotal is not exceeded when factory destroyObject
2065      * has high latency, testOnReturn is set and there is high incidence of
2066      * validation failures.
2067      */
2068     @Test
2069     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2070     public void testMaxTotalInvariant() {
2071         final int maxTotal = 15;
2072         simpleFactory.setEvenValid(false);     // Every other validation fails
2073         simpleFactory.setDestroyLatency(100);  // Destroy takes 100 ms
2074         simpleFactory.setMaxTotalPerKey(maxTotal);  // (makes - destroys) bound
2075         simpleFactory.setValidationEnabled(true);
2076         gkoPool.setMaxTotal(maxTotal);
2077         gkoPool.setMaxIdlePerKey(-1);
2078         gkoPool.setTestOnReturn(true);
2079         gkoPool.setMaxWaitMillis(10000L);
2080         runTestThreads(5, 10, 50, gkoPool);
2081     }
2082 
2083     @Test
2084     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2085     public void testMaxTotalLRU() throws Exception {
2086         gkoPool.setMaxTotalPerKey(2);
2087         gkoPool.setMaxTotal(3);
2088 
2089         final String o1 = gkoPool.borrowObject("a");
2090         assertNotNull(o1);
2091         gkoPool.returnObject("a", o1);
2092         Thread.sleep(25);
2093 
2094         final String o2 = gkoPool.borrowObject("b");
2095         assertNotNull(o2);
2096         gkoPool.returnObject("b", o2);
2097         Thread.sleep(25);
2098 
2099         final String o3 = gkoPool.borrowObject("c");
2100         assertNotNull(o3);
2101         gkoPool.returnObject("c", o3);
2102         Thread.sleep(25);
2103 
2104         final String o4 = gkoPool.borrowObject("a");
2105         assertNotNull(o4);
2106         gkoPool.returnObject("a", o4);
2107         Thread.sleep(25);
2108 
2109         assertSame(o1, o4);
2110 
2111         // this should cause b to be bumped out of the pool
2112         final String o5 = gkoPool.borrowObject("d");
2113         assertNotNull(o5);
2114         gkoPool.returnObject("d", o5);
2115         Thread.sleep(25);
2116 
2117         // now re-request b, we should get a different object because it should
2118         // have been expelled from pool (was oldest because a was requested after b)
2119         final String o6 = gkoPool.borrowObject("b");
2120         assertNotNull(o6);
2121         gkoPool.returnObject("b", o6);
2122 
2123         assertNotSame(o1, o6);
2124         assertNotSame(o2, o6);
2125 
2126         // second a is still in there
2127         final String o7 = gkoPool.borrowObject("a");
2128         assertNotNull(o7);
2129         gkoPool.returnObject("a", o7);
2130 
2131         assertSame(o4, o7);
2132     }
2133 
2134     @Test
2135     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2136     public void testMaxTotalPerKey() throws Exception {
2137         gkoPool.setMaxTotalPerKey(3);
2138         gkoPool.setBlockWhenExhausted(false);
2139 
2140         gkoPool.borrowObject("");
2141         gkoPool.borrowObject("");
2142         gkoPool.borrowObject("");
2143         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject(""));
2144     }
2145 
2146 
2147     @Test
2148     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2149     public void testMaxTotalPerKeyZero() {
2150         gkoPool.setMaxTotalPerKey(0);
2151         gkoPool.setBlockWhenExhausted(false);
2152 
2153         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject("a"));
2154     }
2155 
2156 
2157     /**
2158      * Verifies that if a borrow of a new key is blocked because maxTotal has
2159      * been reached, that borrow continues once another object is returned.
2160      *
2161      * JIRA: POOL-310
2162      */
2163     @Test
2164     public void testMaxTotalWithThreads() throws Exception {
2165 
2166         gkoPool.setMaxTotalPerKey(2);
2167         gkoPool.setMaxTotal(1);
2168 
2169         final int holdTime = 2000;
2170 
2171         final TestThread<String> testA = new TestThread<>(gkoPool, 1, 0, holdTime, false, null, "a");
2172         final TestThread<String> testB = new TestThread<>(gkoPool, 1, 0, holdTime, false, null, "b");
2173 
2174         final Thread threadA = new Thread(testA);
2175         final Thread threadB = new Thread(testB);
2176 
2177         threadA.start();
2178         threadB.start();
2179 
2180         Thread.sleep(holdTime * 2);
2181 
2182         // Both threads should be complete now.
2183         boolean threadRunning = true;
2184         int count = 0;
2185         while (threadRunning && count < 15) {
2186             threadRunning = threadA.isAlive();
2187             threadRunning = threadB.isAlive();
2188             Thread.sleep(200);
2189             count++;
2190         }
2191         assertFalse(threadA.isAlive());
2192         assertFalse(threadB.isAlive());
2193 
2194         assertFalse(testA.failed);
2195         assertFalse(testB.failed);
2196     }
2197 
2198     @Test
2199     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2200     public void testMaxTotalZero() {
2201         gkoPool.setMaxTotal(0);
2202         gkoPool.setBlockWhenExhausted(false);
2203 
2204         assertThrows(NoSuchElementException.class, () -> gkoPool.borrowObject("a"));
2205     }
2206 
2207     /*
2208      * Test multi-threaded pool access.
2209      * Multiple keys, multiple threads, but maxActive only allows half the threads to succeed.
2210      *
2211      * This test was prompted by Continuum build failures in the Commons DBCP test case:
2212      * TestSharedPoolDataSource.testMultipleThreads2()
2213      * Let's see if the this fails on Continuum too!
2214      */
2215     @Test
2216     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2217     public void testMaxWaitMultiThreaded() throws Exception {
2218         final long maxWait = 500; // wait for connection
2219         final long holdTime = 4 * maxWait; // how long to hold connection
2220         final int keyCount = 4; // number of different keys
2221         final int threadsPerKey = 5; // number of threads to grab the key initially
2222         gkoPool.setBlockWhenExhausted(true);
2223         gkoPool.setMaxWaitMillis(maxWait);
2224         gkoPool.setMaxTotalPerKey(threadsPerKey);
2225         // Create enough threads so half the threads will have to wait
2226         final WaitingTestThread[] wtt = new WaitingTestThread[keyCount * threadsPerKey * 2];
2227         for (int i = 0; i < wtt.length; i++) {
2228             wtt[i] = new WaitingTestThread(gkoPool, Integer.toString(i % keyCount), holdTime);
2229         }
2230         final long originMillis = System.currentTimeMillis() - 1000;
2231         for (final WaitingTestThread element : wtt) {
2232             element.start();
2233         }
2234         int failed = 0;
2235         for (final WaitingTestThread element : wtt) {
2236             element.join();
2237             if (element.thrown != null){
2238                 failed++;
2239             }
2240         }
2241         if (DISPLAY_THREAD_DETAILS || wtt.length/2 != failed){
2242             System.out.println(
2243                     "MaxWait: " + maxWait +
2244                     " HoldTime: " + holdTime +
2245                     " KeyCount: " + keyCount +
2246                     " MaxActive: " + threadsPerKey +
2247                     " Threads: " + wtt.length +
2248                     " Failed: " + failed
2249                     );
2250             for (final WaitingTestThread wt : wtt) {
2251                 System.out.println(
2252                         "Preborrow: " + (wt.preBorrowMillis - originMillis) +
2253                         " Postborrow: " + (wt.postBorrowMillis != 0 ? wt.postBorrowMillis - originMillis : -1) +
2254                         " BorrowTime: " + (wt.postBorrowMillis != 0 ? wt.postBorrowMillis - wt.preBorrowMillis : -1) +
2255                         " PostReturn: " + (wt.postReturnMillis != 0 ? wt.postReturnMillis - originMillis : -1) +
2256                         " Ended: " + (wt.endedMillis - originMillis) +
2257                         " Key: " + wt.key +
2258                         " ObjId: " + wt.objectId
2259                         );
2260             }
2261         }
2262         assertEquals(wtt.length / 2, failed, "Expected half the threads to fail");
2263     }
2264 
2265     @Test
2266     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2267     public void testMinIdle() throws Exception {
2268         gkoPool.setMaxIdlePerKey(500);
2269         gkoPool.setMinIdlePerKey(5);
2270         gkoPool.setMaxTotalPerKey(10);
2271         gkoPool.setNumTestsPerEvictionRun(0);
2272         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(50));
2273         gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(100));
2274         gkoPool.setTestWhileIdle(true);
2275 
2276         // Generate a random key
2277         final String key = "A";
2278 
2279         gkoPool.preparePool(key);
2280 
2281         Waiter.sleepQuietly(150L);
2282         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2283 
2284         final String[] active = new String[5];
2285         active[0] = gkoPool.borrowObject(key);
2286 
2287         Waiter.sleepQuietly(150L);
2288         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2289 
2290         for (int i = 1; i < 5; i++) {
2291             active[i] = gkoPool.borrowObject(key);
2292         }
2293 
2294         Waiter.sleepQuietly(150L);
2295         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2296 
2297         for (int i = 0; i < 5; i++) {
2298             gkoPool.returnObject(key, active[i]);
2299         }
2300 
2301         Waiter.sleepQuietly(150L);
2302         assertEquals(10, gkoPool.getNumIdle(), "Should be 10 idle, found " + gkoPool.getNumIdle());
2303     }
2304 
2305     @Test
2306     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2307     public void testMinIdleMaxTotalPerKey() throws Exception {
2308         gkoPool.setMaxIdlePerKey(500);
2309         gkoPool.setMinIdlePerKey(5);
2310         gkoPool.setMaxTotalPerKey(10);
2311         gkoPool.setNumTestsPerEvictionRun(0);
2312         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(50));
2313         gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(100));
2314         gkoPool.setTestWhileIdle(true);
2315 
2316         final String key = "A";
2317 
2318         gkoPool.preparePool(key);
2319         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " +
2320                 gkoPool.getNumIdle());
2321 
2322         Waiter.sleepQuietly(150L);
2323         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2324 
2325         final String[] active = new String[10];
2326 
2327         Waiter.sleepQuietly(150L);
2328         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2329 
2330         for(int i=0 ; i<5 ; i++) {
2331             active[i] = gkoPool.borrowObject(key);
2332         }
2333 
2334         Waiter.sleepQuietly(150L);
2335         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2336 
2337         for(int i=0 ; i<5 ; i++) {
2338             gkoPool.returnObject(key, active[i]);
2339         }
2340 
2341         Waiter.sleepQuietly(150L);
2342         assertEquals(10, gkoPool.getNumIdle(), "Should be 10 idle, found " + gkoPool.getNumIdle());
2343 
2344         for(int i=0 ; i<10 ; i++) {
2345             active[i] = gkoPool.borrowObject(key);
2346         }
2347 
2348         Waiter.sleepQuietly(150L);
2349         assertEquals(0, gkoPool.getNumIdle(), "Should be 0 idle, found " + gkoPool.getNumIdle());
2350 
2351         for(int i=0 ; i<10 ; i++) {
2352             gkoPool.returnObject(key, active[i]);
2353         }
2354 
2355         Waiter.sleepQuietly(150L);
2356         assertEquals(10, gkoPool.getNumIdle(), "Should be 10 idle, found " + gkoPool.getNumIdle());
2357     }
2358 
2359     @Test
2360     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2361     public void testMinIdleNoPreparePool() throws Exception {
2362         gkoPool.setMaxIdlePerKey(500);
2363         gkoPool.setMinIdlePerKey(5);
2364         gkoPool.setMaxTotalPerKey(10);
2365         gkoPool.setNumTestsPerEvictionRun(0);
2366         gkoPool.setMinEvictableIdleTime(Duration.ofMillis(50));
2367         gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(100));
2368         gkoPool.setTestWhileIdle(true);
2369 
2370 
2371         //Generate a random key
2372         final String key = "A";
2373 
2374         Waiter.sleepQuietly(150L);
2375         assertEquals(0, gkoPool.getNumIdle(), "Should be 0 idle, found " + gkoPool.getNumIdle());
2376 
2377         final Object active = gkoPool.borrowObject(key);
2378         assertNotNull(active);
2379 
2380         Waiter.sleepQuietly(150L);
2381         assertEquals(5, gkoPool.getNumIdle(), "Should be 5 idle, found " + gkoPool.getNumIdle());
2382     }
2383 
2384     /**
2385      * Verifies that returning an object twice (without borrow in between) causes ISE
2386      * but does not re-validate or re-passivate the instance.
2387      *
2388      * JIRA: POOL-285
2389      */
2390     @Test
2391     public void testMultipleReturn() throws Exception {
2392         final WaiterFactory<String> factory = new WaiterFactory<>(0, 0, 0, 0, 0, 0);
2393         try (final GenericKeyedObjectPool<String, Waiter> pool = new GenericKeyedObjectPool<>(factory)) {
2394             pool.setTestOnReturn(true);
2395             final Waiter waiter = pool.borrowObject("a");
2396             pool.returnObject("a", waiter);
2397             assertEquals(1, waiter.getValidationCount());
2398             assertEquals(1, waiter.getPassivationCount());
2399             try {
2400                 pool.returnObject("a", waiter);
2401                 fail("Expecting IllegalStateException from multiple return");
2402             } catch (final IllegalStateException ex) {
2403                 // Exception is expected, now check no repeat validation/passivation
2404                 assertEquals(1, waiter.getValidationCount());
2405                 assertEquals(1, waiter.getPassivationCount());
2406             }
2407         }
2408     }
2409 
2410     /**
2411      * Verifies that when a borrowed object is mutated in a way that does not
2412      * preserve equality and hash code, the pool can recognized it on return.
2413      *
2414      * JIRA: POOL-284
2415      */
2416     @Test
2417     public void testMutable() throws Exception {
2418         final HashSetFactory factory = new HashSetFactory();
2419         try (final GenericKeyedObjectPool<String, HashSet<String>> pool = new GenericKeyedObjectPool<>(factory,
2420                 new GenericKeyedObjectPoolConfig<>())) {
2421             final HashSet<String> s1 = pool.borrowObject("a");
2422             final HashSet<String> s2 = pool.borrowObject("a");
2423             s1.add("One");
2424             s2.add("One");
2425             pool.returnObject("a", s1);
2426             pool.returnObject("a", s2);
2427         }
2428     }
2429 
2430     @Test
2431     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2432     public void testNegativeMaxTotalPerKey() throws Exception {
2433         gkoPool.setMaxTotalPerKey(-1);
2434         gkoPool.setBlockWhenExhausted(false);
2435         final String obj = gkoPool.borrowObject("");
2436         assertEquals("0", obj);
2437         gkoPool.returnObject("", obj);
2438     }
2439 
2440     /**
2441      * Verify that when a factory returns a null object, pool methods throw NPE.
2442      */
2443     @Test
2444     @Timeout(value = 1000, unit = TimeUnit.MILLISECONDS)
2445     public void testNPEOnFactoryNull() {
2446         // @formatter:off
2447         final DisconnectingWaiterFactory<String> factory = new DisconnectingWaiterFactory<>(
2448             () -> null,  // Override default to always return null from makeObject
2449             DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_LIFECYCLE_ACTION,
2450             DisconnectingWaiterFactory.DEFAULT_DISCONNECTED_VALIDATION_ACTION
2451         );
2452         // @formatter:on
2453         try (final GenericKeyedObjectPool<String, Waiter> pool = new GenericKeyedObjectPool<>(factory)) {
2454             final String key = "one";
2455             pool.setTestOnBorrow(true);
2456             pool.setMaxTotal(-1);
2457             pool.setMinIdlePerKey(1);
2458             // Disconnect the factory - will always return null in this state
2459             factory.disconnect();
2460             assertThrows(NullPointerException.class, () -> pool.borrowObject(key));
2461             assertThrows(NullPointerException.class, () -> pool.addObject(key));
2462             assertThrows(NullPointerException.class, pool::ensureMinIdle);
2463         }
2464     }
2465 
2466     @Test
2467     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2468     public void testNumActiveNumIdle2() throws Exception {
2469         assertEquals(0, gkoPool.getNumActive());
2470         assertEquals(0, gkoPool.getNumIdle());
2471         assertEquals(0, gkoPool.getNumActive("A"));
2472         assertEquals(0, gkoPool.getNumIdle("A"));
2473         assertEquals(0, gkoPool.getNumActive("B"));
2474         assertEquals(0, gkoPool.getNumIdle("B"));
2475 
2476         final String objA0 = gkoPool.borrowObject("A");
2477         final String objB0 = gkoPool.borrowObject("B");
2478 
2479         assertEquals(2, gkoPool.getNumActive());
2480         assertEquals(0, gkoPool.getNumIdle());
2481         assertEquals(1, gkoPool.getNumActive("A"));
2482         assertEquals(0, gkoPool.getNumIdle("A"));
2483         assertEquals(1, gkoPool.getNumActive("B"));
2484         assertEquals(0, gkoPool.getNumIdle("B"));
2485 
2486         final String objA1 = gkoPool.borrowObject("A");
2487         final String objB1 = gkoPool.borrowObject("B");
2488 
2489         assertEquals(4, gkoPool.getNumActive());
2490         assertEquals(0, gkoPool.getNumIdle());
2491         assertEquals(2, gkoPool.getNumActive("A"));
2492         assertEquals(0, gkoPool.getNumIdle("A"));
2493         assertEquals(2, gkoPool.getNumActive("B"));
2494         assertEquals(0, gkoPool.getNumIdle("B"));
2495 
2496         gkoPool.returnObject("A", objA0);
2497         gkoPool.returnObject("B", objB0);
2498 
2499         assertEquals(2, gkoPool.getNumActive());
2500         assertEquals(2, gkoPool.getNumIdle());
2501         assertEquals(1, gkoPool.getNumActive("A"));
2502         assertEquals(1, gkoPool.getNumIdle("A"));
2503         assertEquals(1, gkoPool.getNumActive("B"));
2504         assertEquals(1, gkoPool.getNumIdle("B"));
2505 
2506         gkoPool.returnObject("A", objA1);
2507         gkoPool.returnObject("B", objB1);
2508 
2509         assertEquals(0, gkoPool.getNumActive());
2510         assertEquals(4, gkoPool.getNumIdle());
2511         assertEquals(0, gkoPool.getNumActive("A"));
2512         assertEquals(2, gkoPool.getNumIdle("A"));
2513         assertEquals(0, gkoPool.getNumActive("B"));
2514         assertEquals(2, gkoPool.getNumIdle("B"));
2515     }
2516 
2517     @Test
2518     public void testReturnObjectThrowsIllegalStateException() {
2519         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(new SimpleFactory<>())) {
2520             assertThrows(IllegalStateException.class,
2521                     () ->  pool.returnObject("Foo", "Bar"));
2522         }
2523     }
2524 
2525 
2526     @Test
2527     public void testReturnObjectWithBlockWhenExhausted() throws Exception {
2528         gkoPool.setBlockWhenExhausted(true);
2529         gkoPool.setMaxTotal(1);
2530 
2531         // Test return object with no take waiters
2532         final String obj = gkoPool.borrowObject("0");
2533         gkoPool.returnObject("0", obj);
2534 
2535         // Test return object with a take waiter
2536         final TestThread<String> testA = new TestThread<>(gkoPool, 1, 0, 500, false, null, "0");
2537         final TestThread<String> testB = new TestThread<>(gkoPool, 1, 0, 0, false, null, "1");
2538         final Thread threadA = new Thread(testA);
2539         final Thread threadB = new Thread(testB);
2540         threadA.start();
2541         threadB.start();
2542         threadA.join();
2543         threadB.join();
2544     }
2545 
2546     @Test
2547     public void testReturnObjectWithoutBlockWhenExhausted() throws Exception {
2548         gkoPool.setBlockWhenExhausted(false);
2549 
2550         // Test return object with no take waiters
2551         final String obj = gkoPool.borrowObject("0");
2552         gkoPool.returnObject("0", obj);
2553     }
2554 
2555     /**
2556      * JIRA: POOL-287
2557      *
2558      * Verify that when an attempt is made to borrow an instance from the pool
2559      * while the evictor is visiting it, there is no capacity leak.
2560      *
2561      * Test creates the scenario described in POOL-287.
2562      */
2563     @Test
2564     public void testReturnToHead() throws Exception {
2565         final SimpleFactory<String> factory = new SimpleFactory<>();
2566         factory.setValidateLatency(100);
2567         factory.setValid(true); // Validation always succeeds
2568         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(factory)) {
2569             pool.setMaxWaitMillis(1000);
2570             pool.setTestWhileIdle(true);
2571             pool.setMaxTotalPerKey(2);
2572             pool.setNumTestsPerEvictionRun(1);
2573             pool.setTimeBetweenEvictionRuns(Duration.ofMillis(500));
2574 
2575             // Load pool with two objects
2576             pool.addObject("one"); // call this o1
2577             pool.addObject("one"); // call this o2
2578             // Default is LIFO, so "one" pool is now [o2, o1] in offer order.
2579             // Evictor will visit in oldest-to-youngest order, so o1 then o2
2580 
2581             Thread.sleep(800); // Wait for first eviction run to complete
2582 
2583             // At this point, one eviction run should have completed, visiting o1
2584             // and eviction cursor should be pointed at o2, which is the next offered instance
2585             Thread.sleep(250); // Wait for evictor to start
2586             final String o1 = pool.borrowObject("one"); // o2 is under eviction, so this will return o1
2587             final String o2 = pool.borrowObject("one"); // Once validation completes, o2 should be offered
2588             pool.returnObject("one", o1);
2589             pool.returnObject("one", o2);
2590         }
2591     }
2592 
2593     @Test
2594     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2595     public void testSettersAndGetters() {
2596         {
2597             gkoPool.setMaxTotalPerKey(123);
2598             assertEquals(123, gkoPool.getMaxTotalPerKey());
2599         }
2600         {
2601             gkoPool.setMaxIdlePerKey(12);
2602             assertEquals(12, gkoPool.getMaxIdlePerKey());
2603         }
2604         {
2605             gkoPool.setMaxWaitMillis(1234L);
2606             assertEquals(1234L, gkoPool.getMaxWaitMillis());
2607         }
2608         {
2609             gkoPool.setMinEvictableIdleTimeMillis(12345L);
2610             assertEquals(12345L, gkoPool.getMinEvictableIdleTimeMillis());
2611         }
2612         {
2613             gkoPool.setMinEvictableIdleTime(Duration.ofMillis(12345L));
2614             assertEquals(12345L, gkoPool.getMinEvictableIdleTime().toMillis());
2615         }
2616         {
2617             gkoPool.setMinEvictableIdleTime(Duration.ofMillis(12345L));
2618             assertEquals(12345L, gkoPool.getMinEvictableIdleDuration().toMillis());
2619         }
2620         {
2621             gkoPool.setNumTestsPerEvictionRun(11);
2622             assertEquals(11, gkoPool.getNumTestsPerEvictionRun());
2623         }
2624         {
2625             gkoPool.setTestOnBorrow(true);
2626             assertTrue(gkoPool.getTestOnBorrow());
2627             gkoPool.setTestOnBorrow(false);
2628             assertFalse(gkoPool.getTestOnBorrow());
2629         }
2630         {
2631             gkoPool.setTestOnReturn(true);
2632             assertTrue(gkoPool.getTestOnReturn());
2633             gkoPool.setTestOnReturn(false);
2634             assertFalse(gkoPool.getTestOnReturn());
2635         }
2636         {
2637             gkoPool.setTestWhileIdle(true);
2638             assertTrue(gkoPool.getTestWhileIdle());
2639             gkoPool.setTestWhileIdle(false);
2640             assertFalse(gkoPool.getTestWhileIdle());
2641         }
2642         {
2643             gkoPool.setTimeBetweenEvictionRunsMillis(11235L);
2644             assertEquals(11235L, gkoPool.getDurationBetweenEvictionRuns().toMillis());
2645             assertEquals(11235L, gkoPool.getTimeBetweenEvictionRuns().toMillis());
2646             assertEquals(11235L, gkoPool.getTimeBetweenEvictionRunsMillis());
2647         }
2648         {
2649             gkoPool.setTimeBetweenEvictionRuns(Duration.ofMillis(11235L));
2650             assertEquals(11235L, gkoPool.getDurationBetweenEvictionRuns().toMillis());
2651             assertEquals(11235L, gkoPool.getTimeBetweenEvictionRuns().toMillis());
2652             assertEquals(11235L, gkoPool.getTimeBetweenEvictionRunsMillis());
2653         }
2654         {
2655             gkoPool.setBlockWhenExhausted(true);
2656             assertTrue(gkoPool.getBlockWhenExhausted());
2657             gkoPool.setBlockWhenExhausted(false);
2658             assertFalse(gkoPool.getBlockWhenExhausted());
2659         }
2660     }
2661 
2662     @Test
2663     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2664     public void testThreaded1() {
2665         gkoPool.setMaxTotalPerKey(15);
2666         gkoPool.setMaxIdlePerKey(15);
2667         gkoPool.setMaxWaitMillis(1000L);
2668         runTestThreads(20, 100, 50, gkoPool);
2669     }
2670 
2671     // Pool-361
2672     @Test
2673     public void testValidateOnCreate() throws Exception {
2674         gkoPool.setTestOnCreate(true);
2675         simpleFactory.setValidationEnabled(true);
2676         gkoPool.addObject("one");
2677         assertEquals(1, simpleFactory.validateCounter);
2678     }
2679 
2680    @Test
2681 public void testValidateOnCreateFailure() throws Exception {
2682     gkoPool.setTestOnCreate(true);
2683     gkoPool.setTestOnBorrow(false);
2684     gkoPool.setMaxTotal(2);
2685     simpleFactory.setValidationEnabled(true);
2686     simpleFactory.setValid(false);
2687     // Make sure failed validations do not leak capacity
2688     gkoPool.addObject("one");
2689     gkoPool.addObject("one");
2690     assertEquals(0, gkoPool.getNumIdle());
2691     assertEquals(0, gkoPool.getNumActive());
2692     simpleFactory.setValid(true);
2693     final String obj = gkoPool.borrowObject("one");
2694     assertNotNull(obj);
2695     gkoPool.addObject("one");
2696     // Should have one idle, one out now
2697     assertEquals(1, gkoPool.getNumIdle());
2698     assertEquals(1, gkoPool.getNumActive());
2699 }
2700 
2701     /**
2702      * Verify that threads waiting on a depleted pool get served when a returning object fails
2703      * validation.
2704      *
2705      * JIRA: POOL-240
2706      *
2707      * @throws Exception May occur in some failure modes
2708      */
2709     @Test
2710     public void testValidationFailureOnReturnFreesCapacity()
2711             throws Exception {
2712         final SimpleFactory<String> factory = new SimpleFactory<>();
2713         factory.setValid(false); // Validate will always fail
2714         factory.setValidationEnabled(true);
2715         try (final GenericKeyedObjectPool<String, String> pool = new GenericKeyedObjectPool<>(factory)) {
2716             pool.setMaxTotalPerKey(2);
2717             pool.setMaxWaitMillis(1500);
2718             pool.setTestOnReturn(true);
2719             pool.setTestOnBorrow(false);
2720             // Borrow an instance and hold if for 5 seconds
2721             final WaitingTestThread thread1 = new WaitingTestThread(pool, "one", 5000);
2722             thread1.start();
2723             // Borrow another instance and return it after 500 ms (validation will fail)
2724             final WaitingTestThread thread2 = new WaitingTestThread(pool, "one", 500);
2725             thread2.start();
2726             Thread.sleep(50);
2727             // Try to borrow an object
2728             final String obj = pool.borrowObject("one");
2729             pool.returnObject("one", obj);
2730         }
2731     }
2732 
2733     // POOL-276
2734     @Test
2735     public void testValidationOnCreateOnly() throws Exception {
2736         simpleFactory.enableValidation = true;
2737 
2738         gkoPool.setMaxTotal(1);
2739         gkoPool.setTestOnCreate(true);
2740         gkoPool.setTestOnBorrow(false);
2741         gkoPool.setTestOnReturn(false);
2742         gkoPool.setTestWhileIdle(false);
2743 
2744         final String o1 = gkoPool.borrowObject("KEY");
2745         assertEquals("KEY0", o1);
2746         final Timer t = new Timer();
2747         t.schedule(
2748                 new TimerTask() {
2749                     @Override
2750                     public void run() {
2751                         gkoPool.returnObject("KEY", o1);
2752                     }
2753                 }, 3000);
2754 
2755         final String o2 = gkoPool.borrowObject("KEY");
2756         assertEquals("KEY0", o2);
2757 
2758         assertEquals(1, simpleFactory.validateCounter);
2759     }
2760 
2761     @Test
2762     public void testWaiterCounts() throws Exception {
2763         final String[] keys = {"one", "two", "three"};
2764         final String[] objects = new String[keys.length];
2765         gkoPool.setMaxTotalPerKey(1);
2766         gkoPool.setBlockWhenExhausted(true);
2767         // Empty pool - waiter counts should be 0
2768         assertTrue(gkoPool.getNumWaitersByKey().isEmpty());
2769         assertEquals(0, gkoPool.getNumWaiters());
2770 
2771         // Exhaust the pool
2772         for (int i = 0; i < keys.length; i++) {
2773             objects[i] = gkoPool.borrowObject(keys[i]);
2774         }
2775 
2776         // Start 1 waiter for "one", 2 for "two", 3 for "three"
2777         // Configure them to hold borrowed object for 100ms
2778         final WaitingTestThread[] waiters = new WaitingTestThread[6];
2779         for (int i = 0; i < 3; i++) {
2780             for (int j = 0; j < i + 1; j++) {
2781                 waiters[i + j] = new WaitingTestThread(gkoPool, keys[i], 100);
2782                 waiters[i + j].start();
2783             }
2784         }
2785         Thread.sleep(10);
2786 
2787         // Check waiter counts
2788         assertEquals(6, gkoPool.getNumWaiters());
2789         for (int i = 0; i < 3; i++) {
2790             assertEquals(i + 1, gkoPool.getNumWaitersByKey().get(keys[i]).intValue());
2791         }
2792 
2793         // Return objects to the pool
2794         for (int i = 0; i < 3; i++) {
2795             gkoPool.returnObject(keys[i], objects[i]);
2796         }
2797         // One waiter for each key should get served, holding for 50ms
2798         Thread.sleep(10);
2799 
2800         // Recheck counts - should be 0 for "one", 1 for "two", 2 for "three
2801         assertEquals(3, gkoPool.getNumWaiters());
2802         for (int i = 0; i < 3; i++) {
2803             assertEquals(i, gkoPool.getNumWaitersByKey().get(keys[i]).intValue());
2804         }
2805 
2806         // Eventually, all get served
2807         for (int i = 0; i < 5; i++) {
2808             waiters[i].join();
2809         }
2810 
2811         // Should be no waiters now, one idle instance under each key
2812         assertEquals(0, gkoPool.getNumWaiters());
2813         for (int i = 0; i < 3; i++) {
2814             assertEquals(0, gkoPool.getNumWaitersByKey().get(keys[i]).intValue());
2815         }
2816         assertEquals(3, gkoPool.getNumIdle());
2817         for (String key : keys) {
2818             assertEquals(1, gkoPool.getNumIdle(key));
2819         }
2820 
2821         // Clear the pool - should clear the map
2822         gkoPool.clear();
2823         assertTrue(gkoPool.getNumWaitersByKey().isEmpty());
2824         assertEquals(0, gkoPool.getNumWaiters());
2825 
2826         gkoPool.close();
2827     }
2828 
2829     /**
2830      * POOL-189
2831      *
2832      * @throws Exception May occur in some failure modes
2833      */
2834     @Test
2835     @Timeout(value = 60_000, unit = TimeUnit.MILLISECONDS)
2836     public void testWhenExhaustedBlockClosePool() throws Exception {
2837         gkoPool.setMaxTotalPerKey(1);
2838         gkoPool.setBlockWhenExhausted(true);
2839         gkoPool.setMaxWaitMillis(-1);
2840         final String obj1 = gkoPool.borrowObject("a");
2841 
2842         // Make sure an object was obtained
2843         assertNotNull(obj1);
2844 
2845         // Create a separate thread to try and borrow another object
2846         final WaitingTestThread wtt = new WaitingTestThread(gkoPool, "a", 200);
2847         wtt.start();
2848         // Give wtt time to start
2849         Thread.sleep(200);
2850 
2851         // close the pool (Bug POOL-189)
2852         gkoPool.close();
2853 
2854         // Give interrupt time to take effect
2855         Thread.sleep(200);
2856 
2857         // Check thread was interrupted
2858         assertTrue(wtt.thrown instanceof InterruptedException);
2859     }
2860 
2861 }
2862 
2863