Examples with BloomFilter org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter used on opensource projects

Example 1 with BloomFilter

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter in project asterixdb by apache.

the class ExternalRTree method activate.

@Override
public synchronized void activate() throws HyracksDataException {
    if (isActive) {
        throw new HyracksDataException("Failed to activate the index since it is already activated.");
    }
    if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
        //First time activation
        List<LSMComponentFileReferences> validFileReferences;
        validFileReferences = fileManager.cleanupAndGetValidFiles();
        for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
            LSMRTreeDiskComponent component;
            component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), lsmComonentFileReference.getDeleteIndexFileReference(), lsmComonentFileReference.getBloomFilterFileReference(), false);
            diskComponents.add(component);
            secondDiskComponents.add(component);
        }
        getLsmHarness().indexFirstTimeActivated();
    } else {
        // components. It should also maintain the version pointer
        for (ILSMComponent c : diskComponents) {
            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
            RTree rtree = component.getRTree();
            BTree btree = component.getBTree();
            BloomFilter bloomFilter = component.getBloomFilter();
            rtree.activate();
            btree.activate();
            bloomFilter.activate();
        }
        for (ILSMComponent c : secondDiskComponents) {
            // Only activate non shared components
            if (!diskComponents.contains(c)) {
                LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
                RTree rtree = component.getRTree();
                BTree btree = component.getBTree();
                BloomFilter bloomFilter = component.getBloomFilter();
                rtree.activate();
                btree.activate();
                bloomFilter.activate();
            }
        }
    }
    isActive = true;
}

Example 2 with BloomFilter

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter in project asterixdb by apache.

the class LSMRTree method deactivateDiskComponent.

@Override
protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
    LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
    RTree rtree = component.getRTree();
    BTree btree = component.getBTree();
    BloomFilter bloomFilter = component.getBloomFilter();
    rtree.deactivateCloseHandle();
    btree.deactivateCloseHandle();
    bloomFilter.deactivate();
}

Example 3 with BloomFilter

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter in project asterixdb by apache.

the class BloomFilterTest method singleFieldTest.

@Test
public void singleFieldTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING BLOOM FILTER");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    int numElements = 100;
    int[] keyFields = { 0 };
    BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(), keyFields);
    double acceptanleFalsePositiveRate = 0.1;
    int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
    BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate);
    bf.create();
    bf.activate();
    IIndexBulkLoader builder = bf.createBuilder(numElements, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
    int fieldCount = 2;
    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    // generate keys
    int maxKey = 1000;
    TreeSet<Integer> uniqueKeys = new TreeSet<>();
    ArrayList<Integer> keys = new ArrayList<>();
    while (uniqueKeys.size() < numElements) {
        int key = rnd.nextInt() % maxKey;
        uniqueKeys.add(key);
    }
    for (Integer i : uniqueKeys) {
        keys.add(i);
    }
    // Insert tuples in the bloom filter
    for (int i = 0; i < keys.size(); ++i) {
        TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
        builder.add(tuple);
    }
    builder.end();
    // Check all the inserted tuples can be found.
    long[] hashes = new long[2];
    for (int i = 0; i < keys.size(); ++i) {
        TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
        Assert.assertTrue(bf.contains(tuple, hashes));
    }
    bf.deactivate();
    bf.destroy();
}

Example 4 with BloomFilter

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter in project asterixdb by apache.

the class BloomFilterTest method multiFieldTest.

@Test
public void multiFieldTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING BLOOM FILTER");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    int numElements = 10000;
    int[] keyFields = { 2, 4, 1 };
    BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(), keyFields);
    double acceptanleFalsePositiveRate = 0.1;
    int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
    BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate);
    bf.create();
    bf.activate();
    IIndexBulkLoader builder = bf.createBuilder(numElements, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
    int fieldCount = 5;
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    int maxLength = 20;
    ArrayList<String> s1 = new ArrayList<>();
    ArrayList<String> s2 = new ArrayList<>();
    ArrayList<String> s3 = new ArrayList<>();
    ArrayList<String> s4 = new ArrayList<>();
    for (int i = 0; i < numElements; ++i) {
        s1.add(randomString(rnd.nextInt() % maxLength, rnd));
        s2.add(randomString(rnd.nextInt() % maxLength, rnd));
        s3.add(randomString(rnd.nextInt() % maxLength, rnd));
        s4.add(randomString(rnd.nextInt() % maxLength, rnd));
    }
    for (int i = 0; i < numElements; ++i) {
        TupleUtils.createTuple(tupleBuilder, tuple, fieldSerdes, s1.get(i), s2.get(i), i, s3.get(i), s4.get(i));
        builder.add(tuple);
    }
    builder.end();
    long[] hashes = new long[2];
    for (int i = 0; i < numElements; ++i) {
        TupleUtils.createTuple(tupleBuilder, tuple, fieldSerdes, s1.get(i), s2.get(i), i, s3.get(i), s4.get(i));
        Assert.assertTrue(bf.contains(tuple, hashes));
    }
    bf.deactivate();
    bf.destroy();
}

Example 5 with BloomFilter

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter in project asterixdb by apache.

the class ExternalBTree method deactivate.

@Override
public void deactivate(boolean flushOnExit) throws HyracksDataException {
    if (!isActive) {
        throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
    }
    if (flushOnExit) {
        BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
        cb.afterFinalize(LSMOperationType.FLUSH, null);
    }
    for (ILSMComponent c : diskComponents) {
        LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
        BTree btree = component.getBTree();
        BloomFilter bloomFilter = component.getBloomFilter();
        btree.deactivateCloseHandle();
        bloomFilter.deactivate();
    }
    for (ILSMComponent c : secondDiskComponents) {
        // Only deactivate non shared components (So components are not de-activated twice)
        if (!diskComponents.contains(c)) {
            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
            BTree btree = component.getBTree();
            BloomFilter bloomFilter = component.getBloomFilter();
            btree.deactivateCloseHandle();
            bloomFilter.deactivate();
        }
    }
    isActive = false;
}