001 package org.apache.jcs.auxiliary.disk.indexed;
002
003 /*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements. See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership. The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License. You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied. See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022 import java.io.File;
023 import java.io.FileNotFoundException;
024 import java.io.IOException;
025 import java.io.Serializable;
026 import java.util.ArrayList;
027 import java.util.Arrays;
028 import java.util.Comparator;
029 import java.util.ConcurrentModificationException;
030 import java.util.HashMap;
031 import java.util.HashSet;
032 import java.util.Iterator;
033 import java.util.LinkedList;
034 import java.util.List;
035 import java.util.Map;
036 import java.util.Set;
037 import java.util.concurrent.atomic.AtomicInteger;
038 import java.util.concurrent.locks.ReentrantReadWriteLock;
039
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042 import org.apache.jcs.auxiliary.AuxiliaryCacheAttributes;
043 import org.apache.jcs.auxiliary.disk.AbstractDiskCache;
044 import org.apache.jcs.auxiliary.disk.LRUMapJCS;
045 import org.apache.jcs.engine.CacheConstants;
046 import org.apache.jcs.engine.behavior.ICacheElement;
047 import org.apache.jcs.engine.behavior.IElementSerializer;
048 import org.apache.jcs.engine.control.group.GroupAttrName;
049 import org.apache.jcs.engine.control.group.GroupId;
050 import org.apache.jcs.engine.logging.behavior.ICacheEvent;
051 import org.apache.jcs.engine.logging.behavior.ICacheEventLogger;
052 import org.apache.jcs.engine.stats.StatElement;
053 import org.apache.jcs.engine.stats.Stats;
054 import org.apache.jcs.engine.stats.behavior.IStatElement;
055 import org.apache.jcs.engine.stats.behavior.IStats;
056 import org.apache.jcs.utils.struct.SortedPreferentialArray;
057 import org.apache.jcs.utils.timing.ElapsedTimer;
058
059 /**
060 * Disk cache that uses a RandomAccessFile with keys stored in memory. The maximum number of keys
061 * stored in memory is configurable. The disk cache tries to recycle spots on disk to limit file
062 * expansion.
063 */
064 public class IndexedDiskCache<K extends Serializable, V extends Serializable>
065 extends AbstractDiskCache<K, V>
066 {
067 /** Don't change */
068 private static final long serialVersionUID = -265035607729729629L;
069
070 /** The logger */
071 protected static final Log log = LogFactory.getLog( IndexedDiskCache.class );
072
073 /** Cache name used in log messages */
074 protected final String logCacheName;
075
076 /** The name of the file where the data is stored */
077 private final String fileName;
078
079 /** The IndexedDisk manages reads and writes to the data file. */
080 private IndexedDisk dataFile;
081
082 /** The IndexedDisk manages reads and writes to the key file. */
083 private IndexedDisk keyFile;
084
085 /** Map containing the keys and disk offsets. */
086 private Map<K, IndexedDiskElementDescriptor> keyHash;
087
088 /** The maximum number of keys that we will keep in memory. */
089 private final int maxKeySize;
090
091 /** A handle on the data file. */
092 private File rafDir;
093
094 /** Should we keep adding to the recycle bin. False during optimization. */
095 boolean doRecycle = true;
096
097 /** Should we optimize real time */
098 boolean isRealTimeOptimizationEnabled = true;
099
100 /** Should we optimize on shutdown. */
101 boolean isShutdownOptimizationEnabled = true;
102
103 /** are we currently optimizing the files */
104 boolean isOptimizing = false;
105
106 /** The number of times the file has been optimized. */
107 private int timesOptimized = 0;
108
109 /** The thread optimizing the file. */
110 protected volatile Thread currentOptimizationThread;
111
112 /** used for counting the number of requests */
113 private int removeCount = 0;
114
115 /** Should we queue puts. True when optimizing. We write the queue post optimization. */
116 private boolean queueInput = false;
117
118 /** list where puts made during optimization are made */
119 private final LinkedList<IndexedDiskElementDescriptor> queuedPutList =
120 new LinkedList<IndexedDiskElementDescriptor>();
121
122 /** RECYLCE BIN -- array of empty spots */
123 private SortedPreferentialArray<IndexedDiskElementDescriptor> recycle;
124
125 /** User configurable parameters */
126 private final IndexedDiskCacheAttributes cattr;
127
128 /** How many slots have we recycled. */
129 private int recycleCnt = 0;
130
131 /** How many items were there on startup. */
132 private int startupSize = 0;
133
134 /** the number of bytes free on disk. */
135 private long bytesFree = 0;
136
137 /** simple stat */
138 private AtomicInteger hitCount = new AtomicInteger(0);
139
140 /**
141 * Use this lock to synchronize reads and writes to the underlying storage mechanism.
142 */
143 protected ReentrantReadWriteLock storageLock = new ReentrantReadWriteLock();
144
145 /**
146 * Constructor for the DiskCache object.
147 * <p>
148 * @param cacheAttributes
149 */
150 public IndexedDiskCache( IndexedDiskCacheAttributes cacheAttributes )
151 {
152 this( cacheAttributes, null );
153 }
154
155 /**
156 * Constructor for the DiskCache object.
157 * <p>
158 * @param cattr
159 * @param elementSerializer used if supplied, the super's super will not set a null
160 */
161 public IndexedDiskCache( IndexedDiskCacheAttributes cattr, IElementSerializer elementSerializer )
162 {
163 super( cattr );
164
165 setElementSerializer( elementSerializer );
166
167 this.cattr = cattr;
168 this.maxKeySize = cattr.getMaxKeySize();
169 this.isRealTimeOptimizationEnabled = cattr.getOptimizeAtRemoveCount() > 0;
170 this.isShutdownOptimizationEnabled = cattr.isOptimizeOnShutdown();
171 this.logCacheName = "Region [" + getCacheName() + "] ";
172 this.fileName = getCacheName();
173
174 try
175 {
176 initializeFileSystem( cattr );
177
178 initializeKeysAndData( cattr );
179
180 initializeRecycleBin();
181
182 // Initialization finished successfully, so set alive to true.
183 alive = true;
184 if ( log.isInfoEnabled() )
185 {
186 log.info( logCacheName + "Indexed Disk Cache is alive." );
187 }
188
189 // TODO: Should we improve detection of whether or not the file should be optimized.
190 if ( isRealTimeOptimizationEnabled && keyHash.size() > 0 )
191 {
192 // Kick off a real time optimization, in case we didn't do a final optimization.
193 doOptimizeRealTime();
194 }
195 }
196 catch ( Exception e )
197 {
198 log.error( logCacheName + "Failure initializing for fileName: " + fileName + " and directory: "
199 + this.rafDir.getAbsolutePath(), e );
200 }
201 }
202
203 /**
204 * Tries to create the root directory if it does not already exist.
205 * <p>
206 * @param cattr
207 */
208 private void initializeFileSystem( IndexedDiskCacheAttributes cattr )
209 {
210 String rootDirName = cattr.getDiskPath();
211 this.rafDir = new File( rootDirName );
212 boolean createdDirectories = this.rafDir.mkdirs();
213 if ( log.isInfoEnabled() )
214 {
215 log.info( logCacheName + "Cache file root directory: " + rootDirName );
216 log.info( logCacheName + "Created root directory: " + createdDirectories );
217 }
218 }
219
220 /**
221 * Creates the key and data disk caches.
222 * <p>
223 * Loads any keys if they are present and ClearDiskOnStartup is false.
224 * <p>
225 * @param cattr
226 * @throws FileNotFoundException
227 * @throws IOException
228 * @throws InterruptedException
229 */
230 private void initializeKeysAndData( IndexedDiskCacheAttributes cattr )
231 throws FileNotFoundException, IOException, InterruptedException
232 {
233 this.dataFile = new IndexedDisk( new File( rafDir, fileName + ".data" ), getElementSerializer() );
234
235 this.keyFile = new IndexedDisk( new File( rafDir, fileName + ".key" ), getElementSerializer() );
236
237 if ( cattr.isClearDiskOnStartup() )
238 {
239 if ( log.isInfoEnabled() )
240 {
241 log.info( logCacheName + "ClearDiskOnStartup is set to true. Ingnoring any persisted data." );
242 }
243 initializeEmptyStore();
244 }
245 else if ( keyFile.length() > 0 )
246 {
247 // If the key file has contents, try to initialize the keys
248 // from it. In no keys are loaded reset the data file.
249 initializeStoreFromPersistedData();
250 }
251 else
252 {
253 // Otherwise start with a new empty map for the keys, and reset
254 // the data file if it has contents.
255 initializeEmptyStore();
256 }
257 }
258
259 /**
260 * Initializes an empty disk cache.
261 * <p>
262 * @throws IOException
263 */
264 private void initializeEmptyStore()
265 throws IOException
266 {
267 initializeKeyMap();
268
269 if ( dataFile.length() > 0 )
270 {
271 dataFile.reset();
272 }
273 }
274
275 /**
276 * Loads any persisted data and checks for consistency. If there is a consistency issue, the
277 * files are cleared.
278 * <p>
279 * @throws InterruptedException
280 * @throws IOException
281 */
282 private void initializeStoreFromPersistedData()
283 throws InterruptedException, IOException
284 {
285 loadKeys();
286
287 if ( keyHash.size() == 0 )
288 {
289 dataFile.reset();
290 }
291 else
292 {
293 boolean isOk = checkKeyDataConsistency( false );
294 if ( !isOk )
295 {
296 keyHash.clear();
297 keyFile.reset();
298 dataFile.reset();
299 log.warn( logCacheName + "Corruption detected. Reseting data and keys files." );
300 }
301 else
302 {
303 synchronized (this)
304 {
305 startupSize = keyHash.size();
306 }
307 }
308 }
309 }
310
311 /**
312 * Loads the keys from the .key file. The keys are stored in a HashMap on disk. This is
313 * converted into a LRUMap.
314 * <p>
315 * @throws InterruptedException
316 */
317 protected void loadKeys()
318 throws InterruptedException
319 {
320 if ( log.isDebugEnabled() )
321 {
322 log.debug( logCacheName + "Loading keys for " + keyFile.toString() );
323 }
324
325 storageLock.writeLock().lock();
326
327 try
328 {
329 // create a key map to use.
330 initializeKeyMap();
331
332 HashMap<K, IndexedDiskElementDescriptor> keys =
333 keyFile.readObject( new IndexedDiskElementDescriptor( 0, (int) keyFile.length()
334 - IndexedDisk.HEADER_SIZE_BYTES ) );
335
336 if ( keys != null )
337 {
338 if ( log.isDebugEnabled() )
339 {
340 log.debug( logCacheName + "Found " + keys.size() + " in keys file." );
341 }
342
343 keyHash.putAll( keys );
344
345 if ( log.isInfoEnabled() )
346 {
347 log.info( logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size()
348 + "; up to " + maxKeySize + " will be available." );
349 }
350 }
351
352 if ( log.isDebugEnabled() )
353 {
354 dump( false );
355 }
356 }
357 catch ( Exception e )
358 {
359 log.error( logCacheName + "Problem loading keys for file " + fileName, e );
360 }
361 finally
362 {
363 storageLock.writeLock().unlock();
364 }
365 }
366
367 /**
368 * Check for minimal consistency between the keys and the datafile. Makes sure no starting
369 * positions in the keys exceed the file length.
370 * <p>
371 * The caller should take the appropriate action if the keys and data are not consistent.
372 * @param checkForDedOverlaps if <code>true</code>, do a more thorough check by checking for
373 * data overlap
374 * @return <code>true</code> if the test passes
375 */
376 private boolean checkKeyDataConsistency( boolean checkForDedOverlaps )
377 {
378 ElapsedTimer timer = new ElapsedTimer();
379 log.debug( logCacheName + "Performing inital consistency check" );
380
381 boolean isOk = true;
382 long fileLength = 0;
383 try
384 {
385 fileLength = dataFile.length();
386
387 for (Map.Entry<K, IndexedDiskElementDescriptor> e : keyHash.entrySet())
388 {
389 IndexedDiskElementDescriptor ded = e.getValue();
390
391 isOk = ( ded.pos + IndexedDisk.HEADER_SIZE_BYTES + ded.len <= fileLength );
392
393 if ( !isOk )
394 {
395 log.warn( logCacheName + "The dataFile is corrupted!" + "\n raf.length() = " + fileLength
396 + "\n ded.pos = " + ded.pos );
397 break;
398 }
399 }
400
401 if ( isOk && checkForDedOverlaps )
402 {
403 isOk = checkForDedOverlaps( createPositionSortedDescriptorList() );
404 }
405 }
406 catch ( Exception e )
407 {
408 log.error( e );
409 isOk = false;
410 }
411
412 if ( log.isInfoEnabled() )
413 {
414 log.info( logCacheName + "Finished inital consistency check, isOk = " + isOk + " in "
415 + timer.getElapsedTimeString() );
416 }
417
418 return isOk;
419 }
420
421 /**
422 * Detects any overlapping elements. This expects a sorted list.
423 * <p>
424 * The total length of an item is IndexedDisk.RECORD_HEADER + ded.len.
425 * <p>
426 * @param sortedDescriptors
427 * @return false if there are overlaps.
428 */
429 protected boolean checkForDedOverlaps( IndexedDiskElementDescriptor[] sortedDescriptors )
430 {
431 long start = System.currentTimeMillis();
432 boolean isOk = true;
433 long expectedNextPos = 0;
434 for ( int i = 0; i < sortedDescriptors.length; i++ )
435 {
436 IndexedDiskElementDescriptor ded = sortedDescriptors[i];
437 if ( expectedNextPos > ded.pos )
438 {
439 log.error( logCacheName + "Corrupt file: overlapping deds " + ded );
440 isOk = false;
441 break;
442 }
443 else
444 {
445 expectedNextPos = ded.pos + IndexedDisk.HEADER_SIZE_BYTES + ded.len;
446 }
447 }
448 long end = System.currentTimeMillis();
449 if ( log.isDebugEnabled() )
450 {
451 log.debug( logCacheName + "Check for DED overlaps took " + ( end - start ) + " ms." );
452 }
453
454 return isOk;
455 }
456
457 /**
458 * Saves key file to disk. This converts the LRUMap to a HashMap for deserialization.
459 */
460 protected void saveKeys()
461 {
462 try
463 {
464 if ( log.isInfoEnabled() )
465 {
466 log.info( logCacheName + "Saving keys to: " + fileName + ", key count: " + keyHash.size() );
467 }
468
469 keyFile.reset();
470
471 HashMap<K, IndexedDiskElementDescriptor> keys =
472 new HashMap<K, IndexedDiskElementDescriptor>();
473 keys.putAll( keyHash );
474
475 if ( keys.size() > 0 )
476 {
477 keyFile.writeObject( keys, 0 );
478 }
479
480 if ( log.isInfoEnabled() )
481 {
482 log.info( logCacheName + "Finished saving keys." );
483 }
484 }
485 catch ( Exception e )
486 {
487 log.error( logCacheName + "Problem storing keys.", e );
488 }
489 }
490
491 /**
492 * Update the disk cache. Called from the Queue. Makes sure the Item has not been retrieved from
493 * purgatory while in queue for disk. Remove items from purgatory when they go to disk.
494 * <p>
495 * @param ce The ICacheElement<K, V> to put to disk.
496 */
497 @Override
498 protected void processUpdate( ICacheElement<K, V> ce )
499 {
500 if ( !alive )
501 {
502 log.error( logCacheName + "No longer alive; aborting put of key = " + ce.getKey() );
503 return;
504 }
505
506 if ( log.isDebugEnabled() )
507 {
508 log.debug( logCacheName + "Storing element on disk, key: " + ce.getKey() );
509 }
510
511 IndexedDiskElementDescriptor ded = null;
512
513 // old element with same key
514 IndexedDiskElementDescriptor old = null;
515
516 try
517 {
518 byte[] data = getElementSerializer().serialize( ce );
519
520 // make sure this only locks for one particular cache region
521 storageLock.writeLock().lock();
522 try
523 {
524 old = keyHash.get( ce.getKey() );
525
526 // Item with the same key already exists in file.
527 // Try to reuse the location if possible.
528 if ( old != null && data.length <= old.len )
529 {
530 // Reuse the old ded. The defrag relies on ded updates by reference, not
531 // replacement.
532 ded = old;
533 ded.len = data.length;
534 }
535 else
536 {
537 // we need this to compare in the recycle bin
538 ded = new IndexedDiskElementDescriptor( dataFile.length(), data.length );
539
540 if ( doRecycle )
541 {
542 IndexedDiskElementDescriptor rep = recycle
543 .takeNearestLargerOrEqual( ded );
544 if ( rep != null )
545 {
546 ded = rep;
547 ded.len = data.length;
548 recycleCnt++;
549 this.adjustBytesFree( ded, false );
550 if ( log.isDebugEnabled() )
551 {
552 log.debug( logCacheName + "using recycled ded " + ded.pos + " rep.len = " + rep.len
553 + " ded.len = " + ded.len );
554 }
555 }
556 }
557
558 // Put it in the map
559 keyHash.put( ce.getKey(), ded );
560
561 if ( queueInput )
562 {
563 queuedPutList.add( ded );
564 if ( log.isDebugEnabled() )
565 {
566 log.debug( logCacheName + "added to queued put list." + queuedPutList.size() );
567 }
568 }
569
570 // add the old slot to the recycle bin
571 if ( old != null )
572 {
573 addToRecycleBin( old );
574 }
575 }
576
577 dataFile.write( ded, data );
578 }
579 finally
580 {
581 storageLock.writeLock().unlock();
582 }
583
584 if ( log.isDebugEnabled() )
585 {
586 log.debug( logCacheName + "Put to file: " + fileName + ", key: " + ce.getKey() + ", position: "
587 + ded.pos + ", size: " + ded.len );
588 }
589 }
590 catch ( ConcurrentModificationException cme )
591 {
592 // do nothing, this means it has gone back to memory mid
593 // serialization
594 if ( log.isDebugEnabled() )
595 {
596 // this shouldn't be possible
597 log.debug( logCacheName + "Caught ConcurrentModificationException." + cme );
598 }
599 }
600 catch ( Exception e )
601 {
602 log.error( logCacheName + "Failure updating element, key: " + ce.getKey() + " old: " + old, e );
603 }
604 }
605
606 /**
607 * Gets the key, then goes to disk to get the object.
608 * <p>
609 * @param key
610 * @return ICacheElement<K, V> or null
611 * @see AbstractDiskCache#doGet
612 */
613 @Override
614 protected ICacheElement<K, V> processGet( K key )
615 {
616 if ( !alive )
617 {
618 log.error( logCacheName + "No longer alive so returning null for key = " + key );
619 return null;
620 }
621
622 if ( log.isDebugEnabled() )
623 {
624 log.debug( logCacheName + "Trying to get from disk: " + key );
625 }
626
627 ICacheElement<K, V> object = null;
628 try
629 {
630 storageLock.readLock().lock();
631 try
632 {
633 object = readElement( key );
634 }
635 finally
636 {
637 storageLock.readLock().unlock();
638 }
639
640 if ( object != null )
641 {
642 hitCount.incrementAndGet();
643 }
644 }
645 catch ( IOException ioe )
646 {
647 log.error( logCacheName + "Failure getting from disk, key = " + key, ioe );
648 reset();
649 }
650 catch ( Exception e )
651 {
652 log.error( logCacheName + "Failure getting from disk, key = " + key, e );
653 }
654 return object;
655 }
656
657 /**
658 * Gets matching items from the cache.
659 * <p>
660 * @param pattern
661 * @return a map of K key to ICacheElement<K, V> element, or an empty map if there is no
662 * data in cache matching keys
663 */
664 @Override
665 public Map<K, ICacheElement<K, V>> processGetMatching( String pattern )
666 {
667 Map<K, ICacheElement<K, V>> elements = new HashMap<K, ICacheElement<K, V>>();
668 try
669 {
670 Set<K> keyArray = null;
671 storageLock.readLock().lock();
672 try
673 {
674 keyArray = new HashSet<K>(keyHash.keySet());
675 }
676 finally
677 {
678 storageLock.readLock().unlock();
679 }
680
681 Set<K> matchingKeys = getKeyMatcher().getMatchingKeysFromArray( pattern, keyArray );
682
683 for (K key : matchingKeys)
684 {
685 ICacheElement<K, V> element = processGet( key );
686 if ( element != null )
687 {
688 elements.put( key, element );
689 }
690 }
691 }
692 catch ( Exception e )
693 {
694 log.error( logCacheName + "Failure getting matching from disk, pattern = " + pattern, e );
695 }
696 return elements;
697 }
698
699 /**
700 * Reads the item from disk.
701 * <p>
702 * @param key
703 * @return ICacheElement
704 * @throws IOException
705 */
706 private ICacheElement<K, V> readElement( K key )
707 throws IOException
708 {
709 ICacheElement<K, V> object = null;
710
711 IndexedDiskElementDescriptor ded = keyHash.get( key );
712
713 if ( ded != null )
714 {
715 if ( log.isDebugEnabled() )
716 {
717 log.debug( logCacheName + "Found on disk, key: " + key );
718 }
719 try
720 {
721 ICacheElement<K, V> readObject = dataFile.readObject( ded );
722 object = readObject;
723 // TODO consider checking key equality and throwing if there is a failure
724 }
725 catch ( IOException e )
726 {
727 log.error( logCacheName + "IO Exception, Problem reading object from file", e );
728 throw e;
729 }
730 catch ( Exception e )
731 {
732 log.error( logCacheName + "Exception, Problem reading object from file", e );
733 throw new IOException( logCacheName + "Problem reading object from disk. " + e.getMessage() );
734 }
735 }
736
737 return object;
738 }
739
740 /**
741 * Gets the group keys from the disk.
742 * <p>
743 * @see org.apache.jcs.auxiliary.AuxiliaryCache#getGroupKeys(java.lang.String)
744 */
745 @Override
746 public Set<K> getGroupKeys( String groupName )
747 {
748 GroupId groupId = new GroupId( cacheName, groupName );
749 HashSet<K> keys = new HashSet<K>();
750 try
751 {
752 storageLock.readLock().lock();
753
754 for (Serializable k : keyHash.keySet())
755 {
756 if ( k instanceof GroupAttrName && ( (GroupAttrName<?>) k ).groupId.equals( groupId ) )
757 {
758 @SuppressWarnings("unchecked") // Type checked with instanceof
759 GroupAttrName<K> groupAttrName = (GroupAttrName<K>) k;
760 keys.add( groupAttrName.attrName );
761 }
762 }
763 }
764 finally
765 {
766 storageLock.readLock().unlock();
767 }
768
769 return keys;
770 }
771
772 /**
773 * Gets the group names from the disk.
774 * <p>
775 * @see org.apache.jcs.auxiliary.AuxiliaryCache#getGroupKeys(java.lang.String)
776 */
777 @Override
778 public Set<String> getGroupNames()
779 {
780 HashSet<String> names = new HashSet<String>();
781 try
782 {
783 storageLock.readLock().lock();
784
785 for (K k : keyHash.keySet())
786 {
787 if ( k instanceof GroupAttrName )
788 {
789 @SuppressWarnings("unchecked") // Type checked with instanceof
790 GroupAttrName<K> groupAttrName = (GroupAttrName<K>) k;
791 names.add( groupAttrName.groupId.groupName );
792 }
793 }
794 }
795 finally
796 {
797 storageLock.readLock().unlock();
798 }
799
800 return names;
801 }
802
803 /**
804 * Returns true if the removal was successful; or false if there is nothing to remove. Current
805 * implementation always result in a disk orphan.
806 * <p>
807 * @return true if at least one item was removed.
808 * @param key
809 */
810 @Override
811 protected boolean processRemove( K key )
812 {
813 if ( !alive )
814 {
815 log.error( logCacheName + "No longer alive so returning false for key = " + key );
816 return false;
817 }
818
819 if ( key == null )
820 {
821 return false;
822 }
823
824 boolean reset = false;
825 boolean removed = false;
826 try
827 {
828 storageLock.writeLock().lock();
829
830 if ( key instanceof String && key.toString().endsWith( CacheConstants.NAME_COMPONENT_DELIMITER ) )
831 {
832 removed = performPartialKeyRemoval( (String) key );
833 }
834 else if ( key instanceof GroupAttrName && ((GroupAttrName<?>)key).attrName == null )
835 {
836 removed = performGroupRemoval( ((GroupAttrName<?>)key).groupId );
837 }
838 else
839 {
840 removed = performSingleKeyRemoval( key );
841 }
842 }
843 catch ( Exception e )
844 {
845 log.error( logCacheName + "Problem removing element.", e );
846 reset = true;
847 }
848 finally
849 {
850 storageLock.writeLock().unlock();
851 }
852
853 if ( reset )
854 {
855 reset();
856 }
857
858 // this increments the remove count.
859 // there is no reason to call this if an item was not removed.
860 if ( removed )
861 {
862 doOptimizeRealTime();
863 }
864
865 return removed;
866 }
867
868 /**
869 * Iterates over the keyset. Builds a list of matches. Removes all the keys in the list. Does
870 * not remove via the iterator, since the map impl may not support it.
871 * <p>
872 * This operates under a lock obtained in doRemove().
873 * <p>
874 * @param key
875 * @return true if there was a match
876 */
877 private boolean performPartialKeyRemoval( String key )
878 {
879 boolean removed = false;
880
881 // remove all keys of the same name hierarchy.
882 List<K> itemsToRemove = new LinkedList<K>();
883
884 for (K k : keyHash.keySet())
885 {
886 if ( k instanceof String && k.toString().startsWith( key.toString() ) )
887 {
888 itemsToRemove.add( k );
889 }
890 }
891
892 // remove matches.
893 for (K fullKey : itemsToRemove)
894 {
895 // Don't add to recycle bin here
896 // https://issues.apache.org/jira/browse/JCS-67
897 performSingleKeyRemoval( fullKey );
898 removed = true;
899 // TODO this needs to update the remove count separately
900 }
901
902 return removed;
903 }
904
905 /**
906 * Remove all elements from the group. This does not use the iterator to remove. It builds a
907 * list of group elements and then removes them one by one.
908 * <p>
909 * This operates under a lock obtained in doRemove().
910 * <p>
911 * @param key
912 * @return true if an element was removed
913 */
914 private boolean performGroupRemoval( GroupId key )
915 {
916 boolean removed = false;
917
918 // remove all keys of the same name group.
919 List<K> itemsToRemove = new LinkedList<K>();
920
921 // remove all keys of the same name hierarchy.
922 for (K k : keyHash.keySet())
923 {
924 if ( k instanceof GroupAttrName && ( (GroupAttrName<?>) k ).groupId.equals( key ) )
925 {
926 itemsToRemove.add( k );
927 }
928 }
929
930 // remove matches.
931 for (K fullKey : itemsToRemove)
932 {
933 // Don't add to recycle bin here
934 // https://issues.apache.org/jira/browse/JCS-67
935 performSingleKeyRemoval( fullKey );
936 removed = true;
937 // TODO this needs to update the remove count separately
938 }
939
940 return removed;
941 }
942
943 /**
944 * Removes an individual key from the cache.
945 * <p>
946 * This operates under a lock obtained in doRemove().
947 * <p>
948 * @param key
949 * @return true if an item was removed.
950 */
951 private boolean performSingleKeyRemoval( K key )
952 {
953 boolean removed;
954 // remove single item.
955 IndexedDiskElementDescriptor ded = keyHash.remove( key );
956 removed = ( ded != null );
957 addToRecycleBin( ded );
958
959 if ( log.isDebugEnabled() )
960 {
961 log.debug( logCacheName + "Disk removal: Removed from key hash, key [" + key + "] removed = " + removed );
962 }
963 return removed;
964 }
965
966 /**
967 * Remove all the items from the disk cache by reseting everything.
968 */
969 @Override
970 public void processRemoveAll()
971 {
972 ICacheEvent<String> cacheEvent = createICacheEvent( cacheName, "all", ICacheEventLogger.REMOVEALL_EVENT );
973 try
974 {
975 reset();
976 }
977 catch ( Exception e )
978 {
979 log.error( logCacheName + "Problem removing all.", e );
980 reset();
981 }
982 finally
983 {
984 logICacheEvent( cacheEvent );
985 }
986 }
987
988 /**
989 * Reset effectively clears the disk cache, creating new files, recyclebins, and keymaps.
990 * <p>
991 * It can be used to handle errors by last resort, force content update, or removeall.
992 */
993 private void reset()
994 {
995 if ( log.isWarnEnabled() )
996 {
997 log.warn( logCacheName + "Reseting cache" );
998 }
999
1000 try
1001 {
1002 storageLock.writeLock().lock();
1003
1004 if ( dataFile != null )
1005 {
1006 dataFile.close();
1007 }
1008 File dataFileTemp = new File( rafDir, fileName + ".data" );
1009 dataFileTemp.delete();
1010
1011 if ( keyFile != null )
1012 {
1013 keyFile.close();
1014 }
1015 File keyFileTemp = new File( rafDir, fileName + ".key" );
1016 keyFileTemp.delete();
1017
1018 dataFile = new IndexedDisk( new File( rafDir, fileName + ".data" ), getElementSerializer() );
1019
1020 keyFile = new IndexedDisk( new File( rafDir, fileName + ".key" ), getElementSerializer() );
1021
1022 initializeRecycleBin();
1023
1024 initializeKeyMap();
1025 }
1026 catch ( Exception e )
1027 {
1028 log.error( logCacheName + "Failure reseting state", e );
1029 }
1030 finally
1031 {
1032 storageLock.writeLock().unlock();
1033 }
1034 }
1035
1036 /**
1037 * If the maxKeySize is < 0, use 5000, no way to have an unlimited recycle bin right now, or one
1038 * less than the mazKeySize.
1039 */
1040 private void initializeRecycleBin()
1041 {
1042 int recycleBinSize = cattr.getMaxRecycleBinSize() >= 0 ? cattr.getMaxRecycleBinSize() : 0;
1043 recycle = new SortedPreferentialArray<IndexedDiskElementDescriptor>( recycleBinSize );
1044 if ( log.isDebugEnabled() )
1045 {
1046 log.debug( logCacheName + "Set recycle max Size to MaxRecycleBinSize: '" + recycleBinSize + "'" );
1047 }
1048 }
1049
1050 /**
1051 * Create the map for keys that contain the index position on disk.
1052 */
1053 private void initializeKeyMap()
1054 {
1055 keyHash = null;
1056 if ( maxKeySize >= 0 )
1057 {
1058 keyHash = new LRUMap( maxKeySize );
1059 if ( log.isInfoEnabled() )
1060 {
1061 log.info( logCacheName + "Set maxKeySize to: '" + maxKeySize + "'" );
1062 }
1063 }
1064 else
1065 {
1066 // If no max size, use a plain map for memory and processing efficiency.
1067 keyHash = new HashMap<K, IndexedDiskElementDescriptor>();
1068 // keyHash = Collections.synchronizedMap( new HashMap() );
1069 if ( log.isInfoEnabled() )
1070 {
1071 log.info( logCacheName + "Set maxKeySize to unlimited'" );
1072 }
1073 }
1074 }
1075
1076 /**
1077 * Dispose of the disk cache in a background thread. Joins against this thread to put a cap on
1078 * the disposal time.
1079 * <p>
1080 * TODO make dispose window configurable.
1081 */
1082 @Override
1083 public void processDispose()
1084 {
1085 ICacheEvent<String> cacheEvent = createICacheEvent( cacheName, "none", ICacheEventLogger.DISPOSE_EVENT );
1086 try
1087 {
1088 Runnable disR = new Runnable()
1089 {
1090 public void run()
1091 {
1092 disposeInternal();
1093 }
1094 };
1095 Thread t = new Thread( disR, "IndexedDiskCache-DisposalThread" );
1096 t.start();
1097 // wait up to 60 seconds for dispose and then quit if not done.
1098 try
1099 {
1100 t.join( 60 * 1000 );
1101 }
1102 catch ( InterruptedException ex )
1103 {
1104 log.error( logCacheName + "Interrupted while waiting for disposal thread to finish.", ex );
1105 }
1106 }
1107 finally
1108 {
1109 logICacheEvent( cacheEvent );
1110 }
1111 }
1112
1113 /**
1114 * Internal method that handles the disposal.
1115 */
1116 protected void disposeInternal()
1117 {
1118 if ( !alive )
1119 {
1120 log.error( logCacheName + "Not alive and dispose was called, filename: " + fileName );
1121 return;
1122 }
1123
1124 // Prevents any interaction with the cache while we're shutting down.
1125 alive = false;
1126
1127 Thread optimizationThread = currentOptimizationThread;
1128 if ( isRealTimeOptimizationEnabled && optimizationThread != null )
1129 {
1130 // Join with the current optimization thread.
1131 if ( log.isDebugEnabled() )
1132 {
1133 log.debug( logCacheName + "In dispose, optimization already " + "in progress; waiting for completion." );
1134 }
1135 try
1136 {
1137 optimizationThread.join();
1138 }
1139 catch ( InterruptedException e )
1140 {
1141 log.error( logCacheName + "Unable to join current optimization thread.", e );
1142 }
1143 }
1144 else if ( isShutdownOptimizationEnabled && this.getBytesFree() > 0 )
1145 {
1146 optimizeFile();
1147 }
1148
1149 saveKeys();
1150
1151 try
1152 {
1153 if ( log.isDebugEnabled() )
1154 {
1155 log.debug( logCacheName + "Closing files, base filename: " + fileName );
1156 }
1157 dataFile.close();
1158 dataFile = null;
1159 keyFile.close();
1160 keyFile = null;
1161 }
1162 catch ( IOException e )
1163 {
1164 log.error( logCacheName + "Failure closing files in dispose, filename: " + fileName, e );
1165 }
1166
1167 if ( log.isInfoEnabled() )
1168 {
1169 log.info( logCacheName + "Shutdown complete." );
1170 }
1171 }
1172
1173 /**
1174 * Add descriptor to recycle bin if it is not null. Adds the length of the item to the bytes
1175 * free.
1176 * <p>
1177 * This is called in three places: (1) When an item is removed. All item removals funnel down to
1178 * the removeSingleItem method. (2) When an item on disk is updated with a value that will not
1179 * fit in the previous slot. (3) When the max key size is reached, the freed slot will be added.
1180 * <p>
1181 * The recylebin is not a set. If a slot it added twice, it will result in the wrong data being
1182 * returned.
1183 * <p>
1184 * @param ded
1185 */
1186 protected void addToRecycleBin( IndexedDiskElementDescriptor ded )
1187 {
1188 // reuse the spot
1189 if ( ded != null )
1190 {
1191 this.adjustBytesFree( ded, true );
1192
1193 if ( doRecycle )
1194 {
1195 recycle.add( ded );
1196 if ( log.isDebugEnabled() )
1197 {
1198 log.debug( logCacheName + "recycled ded" + ded );
1199 }
1200
1201 }
1202 }
1203 }
1204
1205 /**
1206 * Performs the check for optimization, and if it is required, do it.
1207 */
1208 protected void doOptimizeRealTime()
1209 {
1210 if ( isRealTimeOptimizationEnabled && !isOptimizing && ( removeCount++ >= cattr.getOptimizeAtRemoveCount() ) )
1211 {
1212 isOptimizing = true;
1213
1214 if ( log.isInfoEnabled() )
1215 {
1216 log.info( logCacheName + "Optimizing file. removeCount [" + removeCount + "] OptimizeAtRemoveCount ["
1217 + cattr.getOptimizeAtRemoveCount() + "]" );
1218 }
1219
1220 if ( currentOptimizationThread == null )
1221 {
1222 storageLock.writeLock().lock();
1223
1224 try
1225 {
1226 if ( currentOptimizationThread == null )
1227 {
1228 currentOptimizationThread = new Thread( new Runnable()
1229 {
1230 public void run()
1231 {
1232 optimizeFile();
1233
1234 currentOptimizationThread = null;
1235 }
1236 }, "IndexedDiskCache-OptimizationThread" );
1237 }
1238 }
1239 finally
1240 {
1241 storageLock.writeLock().unlock();
1242 }
1243
1244 if ( currentOptimizationThread != null )
1245 {
1246 currentOptimizationThread.start();
1247 }
1248 }
1249 }
1250 }
1251
1252 /**
1253 * File optimization is handled by this method. It works as follows:
1254 * <ol>
1255 * <li>Shutdown recycling and turn on queuing of puts. </li> <li>Take a snapshot of the current
1256 * descriptors. If there are any removes, ignore them, as they will be compacted during the next
1257 * optimization.</li> <li>Optimize the snapshot. For each descriptor:
1258 * <ol>
1259 * <li>Obtain the write-lock.</li> <li>Shift the element on the disk, in order to compact out
1260 * the free space. </li> <li>Release the write-lock. This allows elements to still be accessible
1261 * during optimization.</li>
1262 * </ol>
1263 * </li> <li>Obtain the write-lock.</li> <li>All queued puts are made at the end of the file.
1264 * Optimize these under a single write-lock.</li> <li>Truncate the file.</li> <li>Release the
1265 * write-lock. </li> <li>Restore system to standard operation.</li>
1266 * </ol>
1267 */
1268 protected void optimizeFile()
1269 {
1270 ElapsedTimer timer = new ElapsedTimer();
1271 timesOptimized++;
1272 if ( log.isInfoEnabled() )
1273 {
1274 log.info( logCacheName + "Beginning Optimization #" + timesOptimized );
1275 }
1276
1277 // CREATE SNAPSHOT
1278 IndexedDiskElementDescriptor[] defragList = null;
1279
1280 storageLock.writeLock().lock();
1281
1282 try
1283 {
1284 queueInput = true;
1285 // shut off recycle while we're optimizing,
1286 doRecycle = false;
1287 defragList = createPositionSortedDescriptorList();
1288 }
1289 finally
1290 {
1291 // Release if I acquired.
1292 storageLock.writeLock().unlock();
1293 }
1294
1295 // Defrag the file outside of the write lock. This allows a move to be made,
1296 // and yet have the element still accessible for reading or writing.
1297 long expectedNextPos = defragFile( defragList, 0 );
1298
1299 // ADD THE QUEUED ITEMS to the end and then truncate
1300 storageLock.writeLock().lock();
1301
1302 try
1303 {
1304 try
1305 {
1306 if ( !queuedPutList.isEmpty() )
1307 {
1308 // This is perhaps unnecessary, but the list might not be as sorted as we think.
1309 defragList = new IndexedDiskElementDescriptor[queuedPutList.size()];
1310 queuedPutList.toArray( defragList );
1311 Arrays.sort( defragList, new PositionComparator() );
1312
1313 // pack them at the end
1314 expectedNextPos = defragFile( defragList, expectedNextPos );
1315 }
1316 // TRUNCATE THE FILE
1317 dataFile.truncate( expectedNextPos );
1318 }
1319 catch ( Exception e )
1320 {
1321 log.error( logCacheName + "Error optimizing queued puts.", e );
1322 }
1323
1324 // RESTORE NORMAL OPERATION
1325 removeCount = 0;
1326 resetBytesFree();
1327 initializeRecycleBin();
1328 queuedPutList.clear();
1329 queueInput = false;
1330 // turn recycle back on.
1331 doRecycle = true;
1332 isOptimizing = false;
1333 }
1334 finally
1335 {
1336 storageLock.writeLock().unlock();
1337 }
1338
1339 if ( log.isInfoEnabled() )
1340 {
1341 log.info( logCacheName + "Finished #" + timesOptimized + " Optimization took "
1342 + timer.getElapsedTimeString() );
1343 }
1344 }
1345
1346 /**
1347 * Defragments the file in place by compacting out the free space (i.e., moving records
1348 * forward). If there were no gaps the resulting file would be the same size as the previous
1349 * file. This must be supplied an ordered defragList.
1350 * <p>
1351 * @param defragList sorted list of descriptors for optimization
1352 * @param startingPos the start position in the file
1353 * @return this is the potential new file end
1354 */
1355 private long defragFile( IndexedDiskElementDescriptor[] defragList, long startingPos )
1356 {
1357 ElapsedTimer timer = new ElapsedTimer();
1358 long preFileSize = 0;
1359 long postFileSize = 0;
1360 long expectedNextPos = 0;
1361 try
1362 {
1363 preFileSize = this.dataFile.length();
1364 // find the first gap in the disk and start defragging.
1365 expectedNextPos = startingPos;
1366 for ( int i = 0; i < defragList.length; i++ )
1367 {
1368 storageLock.writeLock().lock();
1369 try
1370 {
1371 if ( expectedNextPos != defragList[i].pos )
1372 {
1373 dataFile.move( defragList[i], expectedNextPos );
1374 }
1375 expectedNextPos = defragList[i].pos + IndexedDisk.HEADER_SIZE_BYTES + defragList[i].len;
1376 }
1377 finally
1378 {
1379 storageLock.writeLock().unlock();
1380 }
1381 }
1382
1383 postFileSize = this.dataFile.length();
1384
1385 // this is the potential new file end
1386 return expectedNextPos;
1387 }
1388 catch ( IOException e )
1389 {
1390 log.error( logCacheName + "Error occurred during defragmentation.", e );
1391 }
1392 finally
1393 {
1394 if ( log.isInfoEnabled() )
1395 {
1396 log.info( logCacheName + "Defragmentation took " + timer.getElapsedTimeString()
1397 + ". File Size (before=" + preFileSize + ") (after=" + postFileSize + ") (truncating to "
1398 + expectedNextPos + ")" );
1399 }
1400 }
1401
1402 return 0;
1403 }
1404
1405 /**
1406 * Creates a snapshot of the IndexedDiskElementDescriptors in the keyHash and returns them
1407 * sorted by position in the dataFile.
1408 * <p>
1409 * TODO fix values() method on the LRU map.
1410 * <p>
1411 * @return IndexedDiskElementDescriptor[]
1412 */
1413 private IndexedDiskElementDescriptor[] createPositionSortedDescriptorList()
1414 {
1415 IndexedDiskElementDescriptor[] defragList = new IndexedDiskElementDescriptor[keyHash.size()];
1416 Iterator<Map.Entry<K, IndexedDiskElementDescriptor>> iterator = keyHash.entrySet().iterator();
1417 for ( int i = 0; iterator.hasNext(); i++ )
1418 {
1419 Map.Entry<K, IndexedDiskElementDescriptor> next = iterator.next();
1420 defragList[i] = next.getValue();
1421 }
1422
1423 Arrays.sort( defragList, new PositionComparator() );
1424
1425 return defragList;
1426 }
1427
1428 /**
1429 * Returns the current cache size.
1430 * <p>
1431 * @return The size value
1432 */
1433 @Override
1434 public int getSize()
1435 {
1436 return keyHash.size();
1437 }
1438
1439 /**
1440 * Returns the size of the recyclebin in number of elements.
1441 * <p>
1442 * @return The number of items in the bin.
1443 */
1444 protected int getRecyleBinSize()
1445 {
1446 return this.recycle.size();
1447 }
1448
1449 /**
1450 * Returns the number of times we have used spots from the recycle bin.
1451 * <p>
1452 * @return The number of spots used.
1453 */
1454 protected int getRecyleCount()
1455 {
1456 return this.recycleCnt;
1457 }
1458
1459 /**
1460 * Returns the number of bytes that are free. When an item is removed, its length is recorded.
1461 * When a spot is used form the recycle bin, the length of the item stored is recorded.
1462 * <p>
1463 * @return The number bytes free on the disk file.
1464 */
1465 protected synchronized long getBytesFree()
1466 {
1467 return this.bytesFree;
1468 }
1469
1470 /**
1471 * Resets the number of bytes that are free.
1472 */
1473 private synchronized void resetBytesFree()
1474 {
1475 this.bytesFree = 0;
1476 }
1477
1478 /**
1479 * To subtract you can pass in false for add..
1480 * <p>
1481 * @param ded
1482 * @param add
1483 */
1484 private synchronized void adjustBytesFree( IndexedDiskElementDescriptor ded, boolean add )
1485 {
1486 if ( ded != null )
1487 {
1488 int amount = ded.len + IndexedDisk.HEADER_SIZE_BYTES;
1489
1490 if ( add )
1491 {
1492 this.bytesFree += amount;
1493 }
1494 else
1495 {
1496 this.bytesFree -= amount;
1497 }
1498 }
1499 }
1500
1501 /**
1502 * This is for debugging and testing.
1503 * <p>
1504 * @return the length of the data file.
1505 * @throws IOException
1506 */
1507 protected long getDataFileSize()
1508 throws IOException
1509 {
1510 long size = 0;
1511
1512 storageLock.readLock().lock();
1513
1514 try
1515 {
1516 if ( dataFile != null )
1517 {
1518 size = dataFile.length();
1519 }
1520 }
1521 finally
1522 {
1523 storageLock.readLock().unlock();
1524 }
1525
1526 return size;
1527 }
1528
1529 /**
1530 * For debugging. This dumps the values by default.
1531 */
1532 public void dump()
1533 {
1534 dump( true );
1535 }
1536
1537 /**
1538 * For debugging.
1539 * <p>
1540 * @param dumpValues A boolean indicating if values should be dumped.
1541 */
1542 public void dump( boolean dumpValues )
1543 {
1544 if ( log.isDebugEnabled() )
1545 {
1546 log.debug( logCacheName + "[dump] Number of keys: " + keyHash.size() );
1547
1548 for (Map.Entry<K, IndexedDiskElementDescriptor> e : keyHash.entrySet())
1549 {
1550 K key = e.getKey();
1551 IndexedDiskElementDescriptor ded = e.getValue();
1552
1553 log.debug( logCacheName + "[dump] Disk element, key: " + key + ", pos: " + ded.pos + ", ded.len"
1554 + ded.len + ( ( dumpValues ) ? ( ", val: " + get( key ) ) : "" ) );
1555 }
1556 }
1557 }
1558
1559 /**
1560 * @return Returns the AuxiliaryCacheAttributes.
1561 */
1562 public AuxiliaryCacheAttributes getAuxiliaryCacheAttributes()
1563 {
1564 return this.cattr;
1565 }
1566
1567 /**
1568 * Gets basic stats for the disk cache.
1569 * <p>
1570 * @return String
1571 */
1572 @Override
1573 public String getStats()
1574 {
1575 return getStatistics().toString();
1576 }
1577
1578 /**
1579 * Returns info about the disk cache.
1580 * <p>
1581 * (non-Javadoc)
1582 * @see org.apache.jcs.auxiliary.AuxiliaryCache#getStatistics()
1583 */
1584 @Override
1585 public synchronized IStats getStatistics()
1586 {
1587 IStats stats = new Stats();
1588 stats.setTypeName( "Indexed Disk Cache" );
1589
1590 ArrayList<IStatElement> elems = new ArrayList<IStatElement>();
1591
1592 IStatElement se = null;
1593
1594 se = new StatElement();
1595 se.setName( "Is Alive" );
1596 se.setData( "" + alive );
1597 elems.add( se );
1598
1599 se = new StatElement();
1600 se.setName( "Key Map Size" );
1601 if ( this.keyHash != null )
1602 {
1603 se.setData( "" + this.keyHash.size() );
1604 }
1605 else
1606 {
1607 se.setData( "-1" );
1608 }
1609 elems.add( se );
1610
1611 try
1612 {
1613 se = new StatElement();
1614 se.setName( "Data File Length" );
1615 if ( this.dataFile != null )
1616 {
1617 se.setData( "" + this.dataFile.length() );
1618 }
1619 else
1620 {
1621 se.setData( "-1" );
1622 }
1623 elems.add( se );
1624 }
1625 catch ( Exception e )
1626 {
1627 log.error( e );
1628 }
1629
1630 se = new StatElement();
1631 se.setName( "Hit Count" );
1632 se.setData( "" + this.hitCount.get() );
1633 elems.add( se );
1634
1635 se = new StatElement();
1636 se.setName( "Bytes Free" );
1637 se.setData( "" + this.bytesFree );
1638 elems.add( se );
1639
1640 se = new StatElement();
1641 se.setName( "Optimize Operation Count" );
1642 se.setData( "" + this.removeCount );
1643 elems.add( se );
1644
1645 se = new StatElement();
1646 se.setName( "Times Optimized" );
1647 se.setData( "" + this.timesOptimized );
1648 elems.add( se );
1649
1650 se = new StatElement();
1651 se.setName( "Recycle Count" );
1652 se.setData( "" + this.recycleCnt );
1653 elems.add( se );
1654
1655 se = new StatElement();
1656 se.setName( "Recycle Bin Size" );
1657 se.setData( "" + this.recycle.size() );
1658 elems.add( se );
1659
1660 se = new StatElement();
1661 se.setName( "Startup Size" );
1662 se.setData( "" + this.startupSize );
1663 elems.add( se );
1664
1665 // get the stats from the super too
1666 // get as array, convert to list, add list to our outer list
1667 IStats sStats = super.getStatistics();
1668 IStatElement[] sSEs = sStats.getStatElements();
1669 List<IStatElement> sL = Arrays.asList( sSEs );
1670 elems.addAll( sL );
1671
1672 // get an array and put them in the Stats object
1673 IStatElement[] ses = elems.toArray( new StatElement[0] );
1674 stats.setStatElements( ses );
1675
1676 return stats;
1677 }
1678
1679 /**
1680 * This is exposed for testing.
1681 * <p>
1682 * @return Returns the timesOptimized.
1683 */
1684 protected int getTimesOptimized()
1685 {
1686 return timesOptimized;
1687 }
1688
1689 /**
1690 * This is used by the event logging.
1691 * <p>
1692 * @return the location of the disk, either path or ip.
1693 */
1694 @Override
1695 protected String getDiskLocation()
1696 {
1697 return dataFile.getFilePath();
1698 }
1699
1700 /**
1701 * Compares IndexedDiskElementDescriptor based on their position.
1702 * <p>
1703 */
1704 protected static final class PositionComparator
1705 implements Comparator<IndexedDiskElementDescriptor>, Serializable
1706 {
1707 /** serialVersionUID */
1708 private static final long serialVersionUID = -8387365338590814113L;
1709
1710 /**
1711 * Compares two descriptors based on position.
1712 * <p>
1713 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
1714 */
1715 public int compare( IndexedDiskElementDescriptor o1, IndexedDiskElementDescriptor o2 )
1716 {
1717 IndexedDiskElementDescriptor ded1 = o1;
1718 IndexedDiskElementDescriptor ded2 = o2;
1719
1720 if ( ded1.pos < ded2.pos )
1721 {
1722 return -1;
1723 }
1724 else if ( ded1.pos == ded2.pos )
1725 {
1726 return 0;
1727 }
1728 else
1729 {
1730 return 1;
1731 }
1732 }
1733 }
1734
1735 /**
1736 * Class for recycling and lru. This implements the LRU overflow callback, so we can add items
1737 * to the recycle bin.
1738 */
1739 public class LRUMap
1740 extends LRUMapJCS<K, IndexedDiskElementDescriptor>
1741 // implements Serializable
1742 {
1743 /** Don't change */
1744 private static final long serialVersionUID = 4955079991472142198L;
1745
1746 /**
1747 * <code>tag</code> tells us which map we are working on.
1748 */
1749 public String tag = "orig";
1750
1751 /**
1752 * Default
1753 */
1754 public LRUMap()
1755 {
1756 super();
1757 }
1758
1759 /**
1760 * @param maxKeySize
1761 */
1762 public LRUMap( int maxKeySize )
1763 {
1764 super( maxKeySize );
1765 }
1766
1767 /**
1768 * This is called when the may key size is reached. The least recently used item will be
1769 * passed here. We will store the position and size of the spot on disk in the recycle bin.
1770 * <p>
1771 * @param key
1772 * @param value
1773 */
1774 @Override
1775 protected void processRemovedLRU(K key, IndexedDiskElementDescriptor value )
1776 {
1777 addToRecycleBin( value );
1778 if ( log.isDebugEnabled() )
1779 {
1780 log.debug( logCacheName + "Removing key: [" + key + "] from key store." );
1781 log.debug( logCacheName + "Key store size: [" + this.size() + "]." );
1782 }
1783
1784 doOptimizeRealTime();
1785 }
1786 }
1787 }