Examples with LSMComponentFileReferences org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences used on opensource projects

Example 11 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTreeWithBuddy method activate.

@Override
public void activate() throws HyracksDataException {
    if (isActive) {
        throw new HyracksDataException("Failed to activate the index since it is already activated.");
    }
    if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
        //First time activation
        List<LSMComponentFileReferences> validFileReferences;
        validFileReferences = fileManager.cleanupAndGetValidFiles();
        for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
            LSMBTreeWithBuddyDiskComponent component;
            component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), lsmComonentFileReference.getDeleteIndexFileReference(), lsmComonentFileReference.getBloomFilterFileReference(), false);
            diskComponents.add(component);
            secondDiskComponents.add(component);
        }
        ((ExternalIndexHarness) getLsmHarness()).indexFirstTimeActivated();
    } else {
        // components. It should also maintain the version pointer
        for (ILSMComponent c : diskComponents) {
            LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
            BTree btree = component.getBTree();
            BTree buddyBtree = component.getBuddyBTree();
            BloomFilter bloomFilter = component.getBloomFilter();
            btree.activate();
            buddyBtree.activate();
            bloomFilter.activate();
        }
        for (ILSMComponent c : secondDiskComponents) {
            // Only activate non shared components
            if (!diskComponents.contains(c)) {
                LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
                BTree btree = component.getBTree();
                BTree buddyBtree = component.getBuddyBTree();
                BloomFilter bloomFilter = component.getBloomFilter();
                btree.activate();
                buddyBtree.activate();
                bloomFilter.activate();
            }
        }
    }
    isActive = true;
}

Example 12 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalRTree method commitTransaction.

@Override
public void commitTransaction() throws HyracksDataException {
    LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
    LSMRTreeDiskComponent component = null;
    if (componentFileRefrences != null) {
        component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false);
    }
    getLsmHarness().addTransactionComponents(component);
}

Example 13 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMBTreeWithBuddyFileManager method cleanupAndGetValidFiles.

@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
    List<LSMComponentFileReferences> validFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
    // Create transaction file filter
    FilenameFilter transactionFilefilter = getTransactionFileFilter(false);
    // Gather files.
    cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory, allBTreeFiles);
    HashSet<String> btreeFilesSet = new HashSet<>();
    for (ComparableFileName cmpFileName : allBTreeFiles) {
        int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
        btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
    }
    validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter), buddyBtreeFactory);
    validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(bloomFilterFilter, transactionFilefilter), null);
    // Sanity check.
    if (allBTreeFiles.size() != allBuddyBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) {
        throw new HyracksDataException("Unequal number of valid BTree, Buddy BTree, and Bloom Filter files found. Aborting cleanup.");
    }
    // Trivial cases.
    if (allBTreeFiles.isEmpty() || allBuddyBTreeFiles.isEmpty() || allBloomFilterFiles.isEmpty()) {
        return validFiles;
    }
    if (allBTreeFiles.size() == 1 && allBuddyBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) {
        validFiles.add(new LSMComponentFileReferences(allBTreeFiles.get(0).fileRef, allBuddyBTreeFiles.get(0).fileRef, allBloomFilterFiles.get(0).fileRef));
        return validFiles;
    }
    // Sorts files names from earliest to latest timestamp.
    Collections.sort(allBTreeFiles);
    Collections.sort(allBuddyBTreeFiles);
    Collections.sort(allBloomFilterFiles);
    List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
    ComparableFileName lastBTree = allBTreeFiles.get(0);
    validComparableBTreeFiles.add(lastBTree);
    List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<>();
    ComparableFileName lastBuddyBTree = allBuddyBTreeFiles.get(0);
    validComparableBuddyBTreeFiles.add(lastBuddyBTree);
    List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
    ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
    validComparableBloomFilterFiles.add(lastBloomFilter);
    for (int i = 1; i < allBTreeFiles.size(); i++) {
        ComparableFileName currentBTree = allBTreeFiles.get(i);
        ComparableFileName currentBuddyBTree = allBuddyBTreeFiles.get(i);
        ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i);
        // Current start timestamp is greater than last stop timestamp.
        if (currentBTree.interval[0].compareTo(lastBTree.interval[1]) > 0 && currentBuddyBTree.interval[0].compareTo(lastBuddyBTree.interval[1]) > 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) {
            validComparableBTreeFiles.add(currentBTree);
            validComparableBuddyBTreeFiles.add(currentBuddyBTree);
            validComparableBloomFilterFiles.add(currentBloomFilter);
            lastBTree = currentBTree;
            lastBuddyBTree = currentBuddyBTree;
            lastBloomFilter = currentBloomFilter;
        } else if (currentBTree.interval[0].compareTo(lastBTree.interval[0]) >= 0 && currentBTree.interval[1].compareTo(lastBTree.interval[1]) <= 0 && currentBuddyBTree.interval[0].compareTo(lastBuddyBTree.interval[0]) >= 0 && currentBuddyBTree.interval[1].compareTo(lastBuddyBTree.interval[1]) <= 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
            // Invalid files are completely contained in last interval.
            File invalidBTreeFile = new File(currentBTree.fullPath);
            invalidBTreeFile.delete();
            File invalidBuddyBTreeFile = new File(currentBuddyBTree.fullPath);
            invalidBuddyBTreeFile.delete();
            File invalidBloomFilterFile = new File(currentBloomFilter.fullPath);
            invalidBloomFilterFile.delete();
        } else {
            // This scenario should not be possible.
            throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
        }
    }
    // Sort valid files in reverse lexicographical order, such that newer
    // files come first.
    Collections.sort(validComparableBTreeFiles, recencyCmp);
    Collections.sort(validComparableBuddyBTreeFiles, recencyCmp);
    Collections.sort(validComparableBloomFilterFiles, recencyCmp);
    Iterator<ComparableFileName> btreeFileIter = validComparableBTreeFiles.iterator();
    Iterator<ComparableFileName> buddyBtreeFileIter = validComparableBuddyBTreeFiles.iterator();
    Iterator<ComparableFileName> bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
    while (btreeFileIter.hasNext() && buddyBtreeFileIter.hasNext()) {
        ComparableFileName cmpBTreeFileName = btreeFileIter.next();
        ComparableFileName cmpBuddyBTreeFileName = buddyBtreeFileIter.next();
        ComparableFileName cmpBloomFilterFileName = bloomFilterFileIter.next();
        validFiles.add(new LSMComponentFileReferences(cmpBTreeFileName.fileRef, cmpBuddyBTreeFileName.fileRef, cmpBloomFilterFileName.fileRef));
    }
    return validFiles;
}

Example 14 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTree method activate.

@Override
public synchronized void activate() throws HyracksDataException {
    if (isActive) {
        throw new HyracksDataException("Failed to activate the index since it is already activated.");
    }
    if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
        //First time activation
        List<LSMComponentFileReferences> validFileReferences;
        validFileReferences = fileManager.cleanupAndGetValidFiles();
        for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
            LSMBTreeDiskComponent component;
            component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), lsmComonentFileReference.getBloomFilterFileReference(), false);
            diskComponents.add(component);
            secondDiskComponents.add(component);
        }
        getLsmHarness().indexFirstTimeActivated();
    } else {
        // This index has been opened before
        for (ILSMComponent c : diskComponents) {
            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
            BTree btree = component.getBTree();
            BloomFilter bloomFilter = component.getBloomFilter();
            btree.activate();
            bloomFilter.activate();
        }
        for (ILSMComponent c : secondDiskComponents) {
            // Only activate non shared components
            if (!diskComponents.contains(c)) {
                LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
                BTree btree = component.getBTree();
                BloomFilter bloomFilter = component.getBloomFilter();
                btree.activate();
                bloomFilter.activate();
            }
        }
    }
    isActive = true;
}

Example 15 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTree method commitTransaction.

@Override
public void commitTransaction() throws HyracksDataException {
    LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
    LSMBTreeDiskComponent component = null;
    if (componentFileRefrences != null) {
        component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false);
    }
    getLsmHarness().addTransactionComponents(component);
}