Examples with LSMComponentFileReferences org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences used on opensource projects

Example 16 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTreeWithBuddy method commitTransaction.

@Override
public void commitTransaction() throws HyracksDataException {
    LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
    LSMBTreeWithBuddyDiskComponent component = null;
    if (componentFileRefrences != null) {
        component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false);
    }
    ((ExternalIndexHarness) getLsmHarness()).addTransactionComponents(component);
}

Example 17 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTreeWithBuddy method scheduleMerge.

@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexOperationContext bctx = createOpContext(NoOpOperationCallback.INSTANCE, 0);
    bctx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ITreeIndexCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields);
    LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
    ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), bctx, opCtx -> new LSMBTreeWithBuddySearchCursor(opCtx, buddyBTreeFields));
    // Since we have two lists of components, to tell whether we need to
    // keep deleted tuples, we need to know
    // which list to check against and we need to synchronize for this
    boolean keepDeleteTuples = false;
    if (version == 0) {
        keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1);
    } else {
        keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents.get(secondDiskComponents.size() - 1);
    }
    ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir(), keepDeleteTuples));
}

Example 18 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMBTreeFileManager method cleanupAndGetValidFiles.

@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
    List<LSMComponentFileReferences> validFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
    // create transaction filter <to hide transaction files>
    FilenameFilter transactionFilter = getTransactionFileFilter(false);
    // Gather files
    // List of valid BTree files.
    cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
    HashSet<String> btreeFilesSet = new HashSet<>();
    for (ComparableFileName cmpFileName : allBTreeFiles) {
        int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
        btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
    }
    if (hasBloomFilter) {
        validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), null);
        // Sanity check.
        if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
            throw new HyracksDataException("Unequal number of valid BTree and bloom filter files found. Aborting cleanup.");
        }
    }
    // Trivial cases.
    if (allBTreeFiles.isEmpty() || hasBloomFilter && allBloomFilterFiles.isEmpty()) {
        return validFiles;
    }
    // Special case: sorting is not required
    if (allBTreeFiles.size() == 1 && (!hasBloomFilter || allBloomFilterFiles.size() == 1)) {
        validFiles.add(new LSMComponentFileReferences(allBTreeFiles.get(0).fileRef, null, hasBloomFilter ? allBloomFilterFiles.get(0).fileRef : null));
        return validFiles;
    }
    // Sorts files names from earliest to latest timestamp.
    Collections.sort(allBTreeFiles);
    if (hasBloomFilter) {
        Collections.sort(allBloomFilterFiles);
    }
    List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
    ComparableFileName lastBTree = allBTreeFiles.get(0);
    validComparableBTreeFiles.add(lastBTree);
    List<ComparableFileName> validComparableBloomFilterFiles = null;
    ComparableFileName lastBloomFilter = null;
    if (hasBloomFilter) {
        validComparableBloomFilterFiles = new ArrayList<>();
        lastBloomFilter = allBloomFilterFiles.get(0);
        validComparableBloomFilterFiles.add(lastBloomFilter);
    }
    ComparableFileName currentBTree = null;
    ComparableFileName currentBloomFilter = null;
    for (int i = 1; i < allBTreeFiles.size(); i++) {
        currentBTree = allBTreeFiles.get(i);
        if (hasBloomFilter) {
            currentBloomFilter = allBloomFilterFiles.get(i);
        }
        // Current start timestamp is greater than last stop timestamp.
        if (currentBTree.interval[0].compareTo(lastBTree.interval[1]) > 0 && (!hasBloomFilter || currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0)) {
            validComparableBTreeFiles.add(currentBTree);
            lastBTree = currentBTree;
            if (hasBloomFilter) {
                validComparableBloomFilterFiles.add(currentBloomFilter);
                lastBloomFilter = currentBloomFilter;
            }
        } else if (currentBTree.interval[0].compareTo(lastBTree.interval[0]) >= 0 && currentBTree.interval[1].compareTo(lastBTree.interval[1]) <= 0 && (!hasBloomFilter || (currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) {
            // Invalid files are completely contained in last interval.
            File invalidBTreeFile = new File(currentBTree.fullPath);
            invalidBTreeFile.delete();
            if (hasBloomFilter) {
                File invalidBloomFilterFile = new File(currentBloomFilter.fullPath);
                invalidBloomFilterFile.delete();
            }
        } else {
            // This scenario should not be possible.
            throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
        }
    }
    // Sort valid files in reverse lexicographical order, such that newer
    // files come first.
    Collections.sort(validComparableBTreeFiles, recencyCmp);
    Iterator<ComparableFileName> btreeFileIter = validComparableBTreeFiles.iterator();
    Iterator<ComparableFileName> bloomFilterFileIter = null;
    if (hasBloomFilter) {
        Collections.sort(validComparableBloomFilterFiles, recencyCmp);
        bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
    }
    ComparableFileName cmpBTreeFileName = null;
    ComparableFileName cmpBloomFilterFileName = null;
    while (btreeFileIter.hasNext() && (hasBloomFilter ? bloomFilterFileIter.hasNext() : true)) {
        cmpBTreeFileName = btreeFileIter.next();
        if (hasBloomFilter) {
            cmpBloomFilterFileName = bloomFilterFileIter.next();
        }
        validFiles.add(new LSMComponentFileReferences(cmpBTreeFileName.fileRef, null, hasBloomFilter ? cmpBloomFilterFileName.fileRef : null));
    }
    return validFiles;
}

Example 19 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMBTree method createDiskComponent.

protected LSMBTreeDiskComponent createDiskComponent(LSMBTreeDiskComponentFactory factory, FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean createComponent) throws HyracksDataException {
    // Create new BTree instance.
    LSMBTreeDiskComponent component = factory.createComponent(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
    // BTree will be closed during cleanup of merge().
    if (createComponent) {
        component.getBTree().create();
    }
    component.getBTree().activate();
    if (hasBloomFilter) {
        if (createComponent) {
            component.getBloomFilter().create();
        }
        component.getBloomFilter().activate();
    }
    if (component.getLSMComponentFilter() != null && !createComponent) {
        getFilterManager().readFilter(component.getLSMComponentFilter(), component.getBTree());
    }
    return component;
}