001package org.apache.commons.jcs.auxiliary.disk.block; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the License is distributed on an 016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 017 * KIND, either express or implied. See the License for the 018 * specific language governing permissions and limitations 019 * under the License. 020 */ 021 022import java.io.BufferedInputStream; 023import java.io.BufferedOutputStream; 024import java.io.EOFException; 025import java.io.File; 026import java.io.FileInputStream; 027import java.io.FileOutputStream; 028import java.io.IOException; 029import java.io.ObjectInputStream; 030import java.io.ObjectOutputStream; 031import java.util.HashMap; 032import java.util.HashSet; 033import java.util.Map; 034import java.util.Map.Entry; 035import java.util.Set; 036import java.util.TreeMap; 037import java.util.concurrent.atomic.AtomicInteger; 038 039import org.apache.commons.jcs.auxiliary.disk.behavior.IDiskCacheAttributes.DiskLimitType; 040import org.apache.commons.jcs.io.ObjectInputStreamClassLoaderAware; 041import org.apache.commons.jcs.utils.struct.AbstractLRUMap; 042import org.apache.commons.jcs.utils.struct.LRUMap; 043import org.apache.commons.jcs.utils.timing.ElapsedTimer; 044import org.apache.commons.logging.Log; 045import org.apache.commons.logging.LogFactory; 046 047/** 048 * This is responsible for storing the keys. 049 * <p> 050 * 051 * @author Aaron Smuts 052 */ 053public class BlockDiskKeyStore<K> 054{ 055 /** The logger */ 056 private static final Log log = LogFactory.getLog(BlockDiskKeyStore.class); 057 058 /** Attributes governing the behavior of the block disk cache. */ 059 private final BlockDiskCacheAttributes blockDiskCacheAttributes; 060 061 /** The key to block map */ 062 private Map<K, int[]> keyHash; 063 064 /** The file where we persist the keys */ 065 private final File keyFile; 066 067 /** The name to prefix log messages with. */ 068 protected final String logCacheName; 069 070 /** Name of the file where we persist the keys */ 071 private final String fileName; 072 073 /** The maximum number of keys to store in memory */ 074 private final int maxKeySize; 075 076 /** 077 * we need this so we can communicate free blocks to the data store when 078 * keys fall off the LRU 079 */ 080 protected final BlockDiskCache<K, ?> blockDiskCache; 081 082 private DiskLimitType diskLimitType = DiskLimitType.COUNT; 083 084 private int blockSize; 085 086 /** 087 * Set the configuration options. 088 * <p> 089 * 090 * @param cacheAttributes 091 * @param blockDiskCache 092 * used for freeing 093 */ 094 public BlockDiskKeyStore(BlockDiskCacheAttributes cacheAttributes, BlockDiskCache<K, ?> blockDiskCache) 095 { 096 this.blockDiskCacheAttributes = cacheAttributes; 097 this.logCacheName = "Region [" + this.blockDiskCacheAttributes.getCacheName() + "] "; 098 this.fileName = this.blockDiskCacheAttributes.getCacheName(); 099 this.maxKeySize = cacheAttributes.getMaxKeySize(); 100 this.blockDiskCache = blockDiskCache; 101 this.diskLimitType = cacheAttributes.getDiskLimitType(); 102 this.blockSize = cacheAttributes.getBlockSizeBytes(); 103 104 File rootDirectory = cacheAttributes.getDiskPath(); 105 106 if (log.isInfoEnabled()) 107 { 108 log.info(logCacheName + "Cache file root directory [" + rootDirectory + "]"); 109 } 110 111 this.keyFile = new File(rootDirectory, fileName + ".key"); 112 113 if (log.isInfoEnabled()) 114 { 115 log.info(logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]"); 116 } 117 118 if (keyFile.length() > 0) 119 { 120 loadKeys(); 121 if (!verify()) 122 { 123 log.warn(logCacheName + "Key File is invalid. Resetting file."); 124 initKeyMap(); 125 reset(); 126 } 127 } 128 else 129 { 130 initKeyMap(); 131 } 132 } 133 134 /** 135 * Saves key file to disk. This gets the LRUMap entry set and write the 136 * entries out one by one after putting them in a wrapper. 137 */ 138 protected void saveKeys() 139 { 140 try 141 { 142 ElapsedTimer timer = new ElapsedTimer(); 143 int numKeys = keyHash.size(); 144 if (log.isInfoEnabled()) 145 { 146 log.info(logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath() + "], key count [" + numKeys + "]"); 147 } 148 149 synchronized (keyFile) 150 { 151 FileOutputStream fos = new FileOutputStream(keyFile); 152 BufferedOutputStream bos = new BufferedOutputStream(fos, 65536); 153 ObjectOutputStream oos = new ObjectOutputStream(bos); 154 try 155 { 156 if (!verify()) 157 { 158 throw new IOException("Inconsistent key file"); 159 } 160 // don't need to synchronize, since the underlying 161 // collection makes a copy 162 for (Map.Entry<K, int[]> entry : keyHash.entrySet()) 163 { 164 BlockDiskElementDescriptor<K> descriptor = new BlockDiskElementDescriptor<K>(); 165 descriptor.setKey(entry.getKey()); 166 descriptor.setBlocks(entry.getValue()); 167 // stream these out in the loop. 168 oos.writeUnshared(descriptor); 169 } 170 } 171 finally 172 { 173 oos.flush(); 174 oos.close(); 175 } 176 } 177 178 if (log.isInfoEnabled()) 179 { 180 log.info(logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString() + " to store " + numKeys 181 + " keys. Key file length [" + keyFile.length() + "]"); 182 } 183 } 184 catch (IOException e) 185 { 186 log.error(logCacheName + "Problem storing keys.", e); 187 } 188 } 189 190 /** 191 * Resets the file and creates a new key map. 192 */ 193 protected void reset() 194 { 195 synchronized (keyFile) 196 { 197 clearMemoryMap(); 198 saveKeys(); 199 } 200 } 201 202 /** 203 * This is mainly used for testing. It leave the disk in tact, and just 204 * clears memory. 205 */ 206 protected void clearMemoryMap() 207 { 208 this.keyHash.clear(); 209 } 210 211 /** 212 * Create the map for keys that contain the index position on disk. 213 */ 214 private void initKeyMap() 215 { 216 keyHash = null; 217 if (maxKeySize >= 0) 218 { 219 if (this.diskLimitType == DiskLimitType.SIZE) 220 { 221 keyHash = new LRUMapSizeLimited(maxKeySize); 222 } 223 else 224 { 225 keyHash = new LRUMapCountLimited(maxKeySize); 226 } 227 if (log.isInfoEnabled()) 228 { 229 log.info(logCacheName + "Set maxKeySize to: '" + maxKeySize + "'"); 230 } 231 } 232 else 233 { 234 // If no max size, use a plain map for memory and processing 235 // efficiency. 236 keyHash = new HashMap<K, int[]>(); 237 // keyHash = Collections.synchronizedMap( new HashMap() ); 238 if (log.isInfoEnabled()) 239 { 240 log.info(logCacheName + "Set maxKeySize to unlimited'"); 241 } 242 } 243 } 244 245 /** 246 * Loads the keys from the .key file. The keys are stored individually on 247 * disk. They are added one by one to an LRUMap.. 248 */ 249 protected void loadKeys() 250 { 251 if (log.isInfoEnabled()) 252 { 253 log.info(logCacheName + "Loading keys for " + keyFile.toString()); 254 } 255 256 try 257 { 258 // create a key map to use. 259 initKeyMap(); 260 261 HashMap<K, int[]> keys = new HashMap<K, int[]>(); 262 263 synchronized (keyFile) 264 { 265 FileInputStream fis = new FileInputStream(keyFile); 266 BufferedInputStream bis = new BufferedInputStream(fis, 65536); 267 ObjectInputStream ois = new ObjectInputStreamClassLoaderAware(bis, null); 268 try 269 { 270 while (true) 271 { 272 @SuppressWarnings("unchecked") 273 // Need to cast from Object 274 BlockDiskElementDescriptor<K> descriptor = (BlockDiskElementDescriptor<K>) ois.readObject(); 275 if (descriptor != null) 276 { 277 keys.put(descriptor.getKey(), descriptor.getBlocks()); 278 } 279 } 280 } 281 catch (EOFException eof) 282 { 283 // nothing 284 } 285 finally 286 { 287 ois.close(); 288 } 289 } 290 291 if (!keys.isEmpty()) 292 { 293 keyHash.putAll(keys); 294 295 if (log.isDebugEnabled()) 296 { 297 log.debug(logCacheName + "Found " + keys.size() + " in keys file."); 298 } 299 300 if (log.isInfoEnabled()) 301 { 302 log.info(logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size() + "; up to " 303 + maxKeySize + " will be available."); 304 } 305 } 306 } 307 catch (Exception e) 308 { 309 log.error(logCacheName + "Problem loading keys for file " + fileName, e); 310 } 311 } 312 313 /** 314 * Gets the entry set. 315 * <p> 316 * 317 * @return entry set. 318 */ 319 public Set<Map.Entry<K, int[]>> entrySet() 320 { 321 return this.keyHash.entrySet(); 322 } 323 324 /** 325 * Gets the key set. 326 * <p> 327 * 328 * @return key set. 329 */ 330 public Set<K> keySet() 331 { 332 return this.keyHash.keySet(); 333 } 334 335 /** 336 * Gets the size of the key hash. 337 * <p> 338 * 339 * @return the number of keys. 340 */ 341 public int size() 342 { 343 return this.keyHash.size(); 344 } 345 346 /** 347 * gets the object for the key. 348 * <p> 349 * 350 * @param key 351 * @return Object 352 */ 353 public int[] get(K key) 354 { 355 return this.keyHash.get(key); 356 } 357 358 /** 359 * Puts a int[] in the keyStore. 360 * <p> 361 * 362 * @param key 363 * @param value 364 */ 365 public void put(K key, int[] value) 366 { 367 this.keyHash.put(key, value); 368 } 369 370 /** 371 * Remove by key. 372 * <p> 373 * 374 * @param key 375 * @return BlockDiskElementDescriptor if it was present, else null 376 */ 377 public int[] remove(K key) 378 { 379 return this.keyHash.remove(key); 380 } 381 382 /** 383 * Verify key store integrity 384 * 385 * @return true if key store is valid 386 */ 387 private boolean verify() 388 { 389 Map<Integer, Set<K>> blockAllocationMap = new TreeMap<Integer, Set<K>>(); 390 for (Entry<K, int[]> e : keyHash.entrySet()) 391 { 392 for (int block : e.getValue()) 393 { 394 Set<K> keys = blockAllocationMap.get(block); 395 if (keys == null) 396 { 397 keys = new HashSet<K>(); 398 blockAllocationMap.put(block, keys); 399 } 400 else if (!log.isDebugEnabled()) 401 { 402 // keys are not null, and no debug - fail fast 403 return false; 404 } 405 keys.add(e.getKey()); 406 } 407 } 408 boolean ok = true; 409 if (log.isDebugEnabled()) 410 { 411 for (Entry<Integer, Set<K>> e : blockAllocationMap.entrySet()) 412 { 413 log.debug("Block " + e.getKey() + ":" + e.getValue()); 414 if (e.getValue().size() > 1) 415 { 416 ok = false; 417 } 418 } 419 return ok; 420 } 421 else 422 { 423 return ok; 424 } 425 } 426 427 /** 428 * Class for recycling and lru. This implements the LRU size overflow 429 * callback, so we can mark the blocks as free. 430 */ 431 public class LRUMapSizeLimited extends AbstractLRUMap<K, int[]> 432 { 433 /** 434 * <code>tag</code> tells us which map we are working on. 435 */ 436 public final static String TAG = "orig-lru-size"; 437 438 // size of the content in kB 439 private AtomicInteger contentSize; 440 private int maxSize; 441 442 /** 443 * Default 444 */ 445 public LRUMapSizeLimited() 446 { 447 this(-1); 448 } 449 450 /** 451 * @param maxSize 452 * maximum cache size in kB 453 */ 454 public LRUMapSizeLimited(int maxSize) 455 { 456 super(); 457 this.maxSize = maxSize; 458 this.contentSize = new AtomicInteger(0); 459 } 460 461 // keep the content size in kB, so 2^31 kB is reasonable value 462 private void subLengthFromCacheSize(int[] value) 463 { 464 contentSize.addAndGet(value.length * blockSize / -1024 - 1); 465 } 466 467 // keep the content size in kB, so 2^31 kB is reasonable value 468 private void addLengthToCacheSize(int[] value) 469 { 470 contentSize.addAndGet(value.length * blockSize / 1024 + 1); 471 } 472 473 @Override 474 public int[] put(K key, int[] value) 475 { 476 int[] oldValue = null; 477 478 try 479 { 480 oldValue = super.put(key, value); 481 } 482 finally 483 { 484 if (value != null) 485 { 486 addLengthToCacheSize(value); 487 } 488 if (oldValue != null) 489 { 490 subLengthFromCacheSize(oldValue); 491 } 492 } 493 494 return oldValue; 495 } 496 497 @Override 498 public int[] remove(Object key) 499 { 500 int[] value = null; 501 502 try 503 { 504 value = super.remove(key); 505 return value; 506 } 507 finally 508 { 509 if (value != null) 510 { 511 subLengthFromCacheSize(value); 512 } 513 } 514 } 515 516 /** 517 * This is called when the may key size is reached. The least recently 518 * used item will be passed here. We will store the position and size of 519 * the spot on disk in the recycle bin. 520 * <p> 521 * 522 * @param key 523 * @param value 524 */ 525 @Override 526 protected void processRemovedLRU(K key, int[] value) 527 { 528 blockDiskCache.freeBlocks(value); 529 if (log.isDebugEnabled()) 530 { 531 log.debug(logCacheName + "Removing key: [" + key + "] from key store."); 532 log.debug(logCacheName + "Key store size: [" + super.size() + "]."); 533 } 534 535 if (value != null) 536 { 537 subLengthFromCacheSize(value); 538 } 539 } 540 541 @Override 542 protected boolean shouldRemove() 543 { 544 return maxSize > 0 && contentSize.get() > maxSize && this.size() > 1; 545 } 546 } 547 548 /** 549 * Class for recycling and lru. This implements the LRU overflow callback, 550 * so we can mark the blocks as free. 551 */ 552 public class LRUMapCountLimited extends LRUMap<K, int[]> 553 { 554 /** 555 * <code>tag</code> tells us which map we are working on. 556 */ 557 public final static String TAG = "orig-lru-count"; 558 559 public LRUMapCountLimited(int maxKeySize) 560 { 561 super(maxKeySize); 562 } 563 564 /** 565 * This is called when the may key size is reached. The least recently 566 * used item will be passed here. We will store the position and size of 567 * the spot on disk in the recycle bin. 568 * <p> 569 * 570 * @param key 571 * @param value 572 */ 573 @Override 574 protected void processRemovedLRU(K key, int[] value) 575 { 576 blockDiskCache.freeBlocks(value); 577 if (log.isDebugEnabled()) 578 { 579 log.debug(logCacheName + "Removing key: [" + key + "] from key store."); 580 log.debug(logCacheName + "Key store size: [" + super.size() + "]."); 581 } 582 } 583 } 584}