CountingBloomFilter.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.commons.collections4.bloomfilter;
import java.util.Objects;
/**
* The interface that describes a Bloom filter that associates a count with each
* bit index rather than a bit. This allows reversal of merge operations with
* remove operations.
*
* <p>A counting Bloom filter is expected to function identically to a standard
* Bloom filter that is the merge of all the Bloom filters that have been added
* to and not later subtracted from the counting Bloom filter. The functional
* state of a CountingBloomFilter at the start and end of a series of merge and
* subsequent remove operations of the same Bloom filters, irrespective of
* remove order, is expected to be the same.</p>
*
* <p>Removal of a filter that has not previously been merged results in an
* invalid state where the cells no longer represent a sum of merged Bloom
* filters. It is impossible to validate merge and remove exactly without
* explicitly storing all filters. Consequently such an operation may go
* undetected. The CountingBloomFilter maintains a state flag that is used as a
* warning that an operation was performed that resulted in invalid cells and
* thus an invalid state. For example this may occur if a cell for an index was
* set to negative following a remove operation.</p>
*
* <p>Implementations should document the expected state of the filter after an
* operation that generates invalid cells, and any potential recovery options.
* An implementation may support a reversal of the operation to restore the
* state to that prior to the operation. In the event that invalid cells are
* adjusted to a valid range then it should be documented if there has been
* irreversible information loss.</p>
*
* <p>Implementations may choose to throw an exception during an operation that
* generates invalid cells. Implementations should document the expected state
* of the filter after such an operation. For example are the cells not updated,
* partially updated or updated entirely before the exception is raised.</p>
*
* @see CellExtractor
* @since 4.5.0-M1
*/
public interface CountingBloomFilter extends BloomFilter<CountingBloomFilter>, CellExtractor {
// Query Operations
/**
* Adds the specified CellExtractor to this Bloom filter.
*
* <p>Specifically
* all cells for the indexes identified by the {@code other} will be incremented
* by their corresponding values in the {@code other}.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param other the CellExtractor to add.
* @return {@code true} if the addition was successful and the state is valid
* @see #isValid()
* @see #subtract(CellExtractor)
*/
boolean add(CellExtractor other);
/**
* Returns the maximum allowable value for a cell count in this Counting filter.
*
* @return the maximum allowable value for a cell count in this Counting filter.
*/
int getMaxCell();
/**
* Determines the maximum number of times the BitMapExtractor could have been merged into this counting filter.
*
* @param bitMapExtractor the BitMapExtractor to provide the indices.
* @return the maximum number of times the BitMapExtractor could have been inserted.
*/
default int getMaxInsert(final BitMapExtractor bitMapExtractor) {
if (!contains(bitMapExtractor)) {
return 0;
}
final long[] bitMaps = bitMapExtractor.asBitMapArray();
final int[] max = { Integer.MAX_VALUE };
processCells((x, y) -> {
if ((bitMaps[BitMaps.getLongIndex(x)] & BitMaps.getLongBit(x)) != 0) {
max[0] = max[0] <= y ? max[0] : y;
}
return true;
});
return max[0];
}
/**
* Determines the maximum number of times the Bloom filter could have been merged into this counting filter.
*
* @param bloomFilter the Bloom filter the check for.
* @return the maximum number of times the Bloom filter could have been inserted.
*/
default int getMaxInsert(final BloomFilter<?> bloomFilter) {
return getMaxInsert((BitMapExtractor) bloomFilter);
}
/**
* Determines the maximum number of times the Cell Extractor could have been added.
*
* @param cellExtractor the extractor of cells.
* @return the maximum number of times the CellExtractor could have been inserted.
*/
int getMaxInsert(CellExtractor cellExtractor);
/**
* Determines the maximum number of times the Hasher could have been merged into this counting filter.
*
* @param hasher the Hasher to provide the indices.
* @return the maximum number of times the hasher could have been inserted.
*/
default int getMaxInsert(final Hasher hasher) {
return getMaxInsert(hasher.indices(getShape()));
}
/**
* Determines the maximum number of times the IndexExtractor could have been merged into this counting filter.
* <p>
* To determine how many times an indexExtractor could have been added create a CellExtractor from the indexExtractor and check that
* </p>
*
* @param indexExtractor the extractor to drive the count check.
* @return the maximum number of times the IndexExtractor could have been inserted.
* @see #getMaxInsert(CellExtractor)
*/
default int getMaxInsert(final IndexExtractor indexExtractor) {
return getMaxInsert(CellExtractor.from(indexExtractor.uniqueIndices()));
}
/**
* Returns {@code true} if the internal state is valid.
*
* <p>This flag is a warning that an addition or
* subtraction of cells from this filter resulted in an invalid cell for one or more
* indexes. For example this may occur if a cell for an index was
* set to negative following a subtraction operation, or overflows the value specified by {@code getMaxCell()} following an
* addition operation.</p>
*
* <p>A counting Bloom filter that has an invalid state is no longer ensured to function
* identically to a standard Bloom filter instance that is the merge of all the Bloom filters
* that have been added to and not later subtracted from this counting Bloom filter.</p>
*
* <p>Note: The change to an invalid state may or may not be reversible. Implementations
* are expected to document their policy on recovery from an addition or removal operation
* that generated an invalid state.</p>
*
* @return {@code true} if the state is valid
*/
boolean isValid();
/**
* Merges the specified BitMap extractor into this Bloom filter.
*
* <p>Specifically: all cells for the indexes identified by the {@code bitMapExtractor} will be incremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param bitMapExtractor the BitMapExtractor
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #add(CellExtractor)
*/
@Override
default boolean merge(final BitMapExtractor bitMapExtractor) {
return merge(IndexExtractor.fromBitMapExtractor(bitMapExtractor));
}
/**
* Merges the specified Bloom filter into this Bloom filter.
*
* <p>Specifically: all cells for the indexes identified by the {@code other} filter will be incremented by 1.</p>
*
* <p>Note: If the other filter is a counting Bloom filter the other filter's cells are ignored and it is treated as an
* IndexExtractor.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param other the other Bloom filter
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #add(CellExtractor)
*/
@Override
default boolean merge(final BloomFilter<?> other) {
Objects.requireNonNull(other, "other");
return merge((IndexExtractor) other);
}
/**
* Merges the specified Hasher into this Bloom filter.
*
* <p>Specifically: all cells for the unique indexes identified by the {@code hasher} will be incremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param hasher the hasher
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #add(CellExtractor)
*/
@Override
default boolean merge(final Hasher hasher) {
Objects.requireNonNull(hasher, "hasher");
return merge(hasher.indices(getShape()));
}
/**
* Merges the specified index extractor into this Bloom filter.
*
* <p>Specifically: all unique cells for the indices identified by the {@code indexExtractor} will be incremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* <p>Notes:</p>
* <ul>
* <li>If indices that are returned multiple times should be incremented multiple times convert the IndexExtractor
* to a CellExtractor and add that.</li>
* <li>Implementations should throw {@code IllegalArgumentException} and no other exception on bad input.</li>
* </ul>
* @param indexExtractor the IndexExtractor
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #add(CellExtractor)
*/
@Override
default boolean merge(final IndexExtractor indexExtractor) {
Objects.requireNonNull(indexExtractor, "indexExtractor");
try {
return add(CellExtractor.from(indexExtractor.uniqueIndices()));
} catch (final IndexOutOfBoundsException e) {
throw new IllegalArgumentException(
String.format("Filter only accepts values in the [0,%d) range", getShape().getNumberOfBits()), e);
}
}
/**
* Removes the specified BitMapExtractor from this Bloom filter.
*
* <p>Specifically all cells for the indices produced by the {@code bitMapExtractor} will be
* decremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param bitMapExtractor the BitMapExtractor to provide the indexes
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #subtract(CellExtractor)
*/
default boolean remove(final BitMapExtractor bitMapExtractor) {
return remove(IndexExtractor.fromBitMapExtractor(bitMapExtractor));
}
/**
* Removes the specified Bloom filter from this Bloom filter.
*
* <p>Specifically: all cells for the indexes identified by the {@code other} filter will be decremented by 1.</p>
*
* <p>Note: If the other filter is a counting Bloom filter the other filter's cells are ignored and it is treated as an
* IndexExtractor.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param other the other Bloom filter
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #subtract(CellExtractor)
*/
default boolean remove(final BloomFilter<?> other) {
return remove((IndexExtractor) other);
}
/**
* Removes the unique values from the specified hasher from this Bloom filter.
*
* <p>Specifically all cells for the unique indices produced by the {@code hasher} will be
* decremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* @param hasher the hasher to provide the indexes
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #subtract(CellExtractor)
*/
default boolean remove(final Hasher hasher) {
Objects.requireNonNull(hasher, "hasher");
return remove(hasher.indices(getShape()));
}
/**
* Removes the values from the specified IndexExtractor from the Bloom filter from this Bloom filter.
*
* <p>Specifically all cells for the unique indices produced by the {@code hasher} will be
* decremented by 1.</p>
*
* <p>This method will return {@code true} if the filter is valid after the operation.</p>
*
* <p>Note: If indices that are returned multiple times should be decremented multiple times convert the IndexExtractor
* to a CellExtractor and subtract that.</p>
*
* @param indexExtractor the IndexExtractor to provide the indexes
* @return {@code true} if the removal was successful and the state is valid
* @see #isValid()
* @see #subtract(CellExtractor)
*/
default boolean remove(final IndexExtractor indexExtractor) {
Objects.requireNonNull(indexExtractor, "indexExtractor");
try {
return subtract(CellExtractor.from(indexExtractor.uniqueIndices()));
} catch (final IndexOutOfBoundsException e) {
throw new IllegalArgumentException(
String.format("Filter only accepts values in the [0,%d) range", getShape().getNumberOfBits()));
}
}
/**
* Adds the specified CellExtractor to this Bloom filter.
*
* <p>Specifically
* all cells for the indexes identified by the {@code other} will be decremented
* by their corresponding values in the {@code other}.</p>
*
* <p>This method will return true if the filter is valid after the operation.</p>
*
* @param other the CellExtractor to subtract.
* @return {@code true} if the subtraction was successful and the state is valid
* @see #isValid()
* @see #add(CellExtractor)
*/
boolean subtract(CellExtractor other);
/**
* The default implementation is a no-op since the counting bloom filter returns an unique IndexExtractor by default.
* @return this counting Bloom filter.
*/
@Override
default IndexExtractor uniqueIndices() {
return this;
}
}