Examples with LSMComponentFileReferences org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences used on opensource projects

Example 1 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalRTree method activate.

@Override
public synchronized void activate() throws HyracksDataException {
    if (isActive) {
        throw new HyracksDataException("Failed to activate the index since it is already activated.");
    }
    if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
        //First time activation
        List<LSMComponentFileReferences> validFileReferences;
        validFileReferences = fileManager.cleanupAndGetValidFiles();
        for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
            LSMRTreeDiskComponent component;
            component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(), lsmComonentFileReference.getDeleteIndexFileReference(), lsmComonentFileReference.getBloomFilterFileReference(), false);
            diskComponents.add(component);
            secondDiskComponents.add(component);
        }
        getLsmHarness().indexFirstTimeActivated();
    } else {
        // components. It should also maintain the version pointer
        for (ILSMComponent c : diskComponents) {
            LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
            RTree rtree = component.getRTree();
            BTree btree = component.getBTree();
            BloomFilter bloomFilter = component.getBloomFilter();
            rtree.activate();
            btree.activate();
            bloomFilter.activate();
        }
        for (ILSMComponent c : secondDiskComponents) {
            // Only activate non shared components
            if (!diskComponents.contains(c)) {
                LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
                RTree rtree = component.getRTree();
                BTree btree = component.getBTree();
                BloomFilter bloomFilter = component.getBloomFilter();
                rtree.activate();
                btree.activate();
                bloomFilter.activate();
            }
        }
    }
    isActive = true;
}

Example 2 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMRTreeFileManager method getTransactionFileReferenceForCommit.

@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
    FilenameFilter transactionFilter;
    File dir = new File(baseDir);
    String[] files = dir.list(transactionFileNameFilter);
    if (files.length == 0) {
        return null;
    }
    if (files.length != 1) {
        throw new HyracksDataException("More than one transaction lock found:" + files.length);
    } else {
        transactionFilter = getTransactionFileFilter(true);
        String txnFileName = dir.getPath() + File.separator + files[0];
        // get the actual transaction files
        files = dir.list(transactionFilter);
        if (files.length < 3) {
            throw new HyracksDataException("LSM Rtree transaction has less than 3 files :" + files.length);
        }
        try {
            Files.delete(Paths.get(txnFileName));
        } catch (IOException e) {
            throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
        }
    }
    File rTreeFile = null;
    File bTreeFile = null;
    File bloomFilterFile = null;
    for (String fileName : files) {
        if (fileName.endsWith(BTREE_STRING)) {
            bTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(RTREE_STRING)) {
            rTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
            bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
        } else {
            throw new HyracksDataException("unrecognized file found = " + fileName);
        }
    }
    FileReference rTreeFileRef = ioManager.resolveAbsolutePath(rTreeFile.getAbsolutePath());
    FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
    FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
    return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef, bloomFilterFileRef);
}

Example 3 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMRTreeFileManager method cleanupAndGetValidFiles.

@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
    List<LSMComponentFileReferences> validFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allRTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
    // Create a transaction filter <- to hide transaction components->
    FilenameFilter transactionFilter = getTransactionFileFilter(false);
    // Gather files.
    cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
    HashSet<String> btreeFilesSet = new HashSet<>();
    for (ComparableFileName cmpFileName : allBTreeFiles) {
        int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
        btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
    }
    validateFiles(btreeFilesSet, allRTreeFiles, getCompoundFilter(transactionFilter, rtreeFilter), rtreeFactory);
    validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(transactionFilter, bloomFilterFilter), null);
    // Sanity check.
    if (allRTreeFiles.size() != allBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) {
        throw new HyracksDataException("Unequal number of valid RTree, BTree, and Bloom Filter files found. Aborting cleanup.");
    }
    // Trivial cases.
    if (allRTreeFiles.isEmpty() || allBTreeFiles.isEmpty() || allBloomFilterFiles.isEmpty()) {
        return validFiles;
    }
    if (allRTreeFiles.size() == 1 && allBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) {
        validFiles.add(new LSMComponentFileReferences(allRTreeFiles.get(0).fileRef, allBTreeFiles.get(0).fileRef, allBloomFilterFiles.get(0).fileRef));
        return validFiles;
    }
    // Sorts files names from earliest to latest timestamp.
    Collections.sort(allRTreeFiles);
    Collections.sort(allBTreeFiles);
    Collections.sort(allBloomFilterFiles);
    List<ComparableFileName> validComparableRTreeFiles = new ArrayList<>();
    ComparableFileName lastRTree = allRTreeFiles.get(0);
    validComparableRTreeFiles.add(lastRTree);
    List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
    ComparableFileName lastBTree = allBTreeFiles.get(0);
    validComparableBTreeFiles.add(lastBTree);
    List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
    ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
    validComparableBloomFilterFiles.add(lastBloomFilter);
    for (int i = 1; i < allRTreeFiles.size(); i++) {
        ComparableFileName currentRTree = allRTreeFiles.get(i);
        ComparableFileName currentBTree = allBTreeFiles.get(i);
        ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i);
        // Current start timestamp is greater than last stop timestamp.
        if (currentRTree.interval[0].compareTo(lastRTree.interval[1]) > 0 && currentBTree.interval[0].compareTo(lastBTree.interval[1]) > 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) {
            validComparableRTreeFiles.add(currentRTree);
            validComparableBTreeFiles.add(currentBTree);
            validComparableBloomFilterFiles.add(currentBloomFilter);
            lastRTree = currentRTree;
            lastBTree = currentBTree;
            lastBloomFilter = currentBloomFilter;
        } else if (currentRTree.interval[0].compareTo(lastRTree.interval[0]) >= 0 && currentRTree.interval[1].compareTo(lastRTree.interval[1]) <= 0 && currentBTree.interval[0].compareTo(lastBTree.interval[0]) >= 0 && currentBTree.interval[1].compareTo(lastBTree.interval[1]) <= 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
            // Invalid files are completely contained in last interval.
            File invalidRTreeFile = new File(currentRTree.fullPath);
            invalidRTreeFile.delete();
            File invalidBTreeFile = new File(currentBTree.fullPath);
            invalidBTreeFile.delete();
            File invalidBloomFilterFile = new File(currentBloomFilter.fullPath);
            invalidBloomFilterFile.delete();
        } else {
            // This scenario should not be possible.
            throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
        }
    }
    // Sort valid files in reverse lexicographical order, such that newer
    // files come first.
    Collections.sort(validComparableRTreeFiles, recencyCmp);
    Collections.sort(validComparableBTreeFiles, recencyCmp);
    Collections.sort(validComparableBloomFilterFiles, recencyCmp);
    Iterator<ComparableFileName> rtreeFileIter = validComparableRTreeFiles.iterator();
    Iterator<ComparableFileName> btreeFileIter = validComparableBTreeFiles.iterator();
    Iterator<ComparableFileName> bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
    while (rtreeFileIter.hasNext() && btreeFileIter.hasNext()) {
        ComparableFileName cmpRTreeFileName = rtreeFileIter.next();
        ComparableFileName cmpBTreeFileName = btreeFileIter.next();
        ComparableFileName cmpBloomFilterFileName = bloomFilterFileIter.next();
        validFiles.add(new LSMComponentFileReferences(cmpRTreeFileName.fileRef, cmpBTreeFileName.fileRef, cmpBloomFilterFileName.fileRef));
    }
    return validFiles;
}

Example 4 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMInvertedIndexFileManager method cleanupAndGetValidFiles.

@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
    List<LSMComponentFileReferences> validFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allDictBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allInvListsFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allDeletedKeysBTreeFiles = new ArrayList<>();
    ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
    // Gather files.
    cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles);
    HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>();
    for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) {
        int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
        deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index));
    }
    // TODO: do we really need to validate the inverted lists files or is validating the dict. BTrees is enough?
    validateFiles(deletedKeysBTreeFilesSet, allInvListsFiles, invListFilter, null);
    validateFiles(deletedKeysBTreeFilesSet, allDictBTreeFiles, dictBTreeFilter, btreeFactory);
    validateFiles(deletedKeysBTreeFilesSet, allBloomFilterFiles, bloomFilterFilter, null);
    // Sanity check.
    if (allDictBTreeFiles.size() != allInvListsFiles.size() || allDictBTreeFiles.size() != allDeletedKeysBTreeFiles.size() || allDictBTreeFiles.size() != allBloomFilterFiles.size()) {
        throw new HyracksDataException("Unequal number of valid Dictionary BTree, Inverted Lists, Deleted BTree, and Bloom Filter files found. Aborting cleanup.");
    }
    // Trivial cases.
    if (allDictBTreeFiles.isEmpty() || allInvListsFiles.isEmpty() || allDeletedKeysBTreeFiles.isEmpty() || allBloomFilterFiles.isEmpty()) {
        return validFiles;
    }
    if (allDictBTreeFiles.size() == 1 && allInvListsFiles.size() == 1 && allDeletedKeysBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) {
        validFiles.add(new LSMComponentFileReferences(allDictBTreeFiles.get(0).fileRef, allDeletedKeysBTreeFiles.get(0).fileRef, allBloomFilterFiles.get(0).fileRef));
        return validFiles;
    }
    // Sorts files names from earliest to latest timestamp.
    Collections.sort(allDeletedKeysBTreeFiles);
    Collections.sort(allDictBTreeFiles);
    Collections.sort(allBloomFilterFiles);
    List<ComparableFileName> validComparableDictBTreeFiles = new ArrayList<>();
    ComparableFileName lastDictBTree = allDictBTreeFiles.get(0);
    validComparableDictBTreeFiles.add(lastDictBTree);
    List<ComparableFileName> validComparableDeletedKeysBTreeFiles = new ArrayList<>();
    ComparableFileName lastDeletedKeysBTree = allDeletedKeysBTreeFiles.get(0);
    validComparableDeletedKeysBTreeFiles.add(lastDeletedKeysBTree);
    List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
    ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
    validComparableBloomFilterFiles.add(lastBloomFilter);
    for (int i = 1; i < allDictBTreeFiles.size(); i++) {
        ComparableFileName currentDeletedKeysBTree = allDeletedKeysBTreeFiles.get(i);
        ComparableFileName CurrentDictBTree = allDictBTreeFiles.get(i);
        ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i);
        // Current start timestamp is greater than last stop timestamp.
        if (currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[1]) > 0 && CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[1]) > 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) {
            validComparableDictBTreeFiles.add(CurrentDictBTree);
            validComparableDeletedKeysBTreeFiles.add(currentDeletedKeysBTree);
            validComparableBloomFilterFiles.add(currentBloomFilter);
            lastDictBTree = CurrentDictBTree;
            lastDeletedKeysBTree = currentDeletedKeysBTree;
            lastBloomFilter = currentBloomFilter;
        } else if (currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[0]) >= 0 && currentDeletedKeysBTree.interval[1].compareTo(lastDeletedKeysBTree.interval[1]) <= 0 && CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0 && CurrentDictBTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
            // Invalid files are completely contained in last interval.
            File invalidDeletedBTreeFile = new File(currentDeletedKeysBTree.fullPath);
            invalidDeletedBTreeFile.delete();
            File invalidDictBTreeFile = new File(CurrentDictBTree.fullPath);
            invalidDictBTreeFile.delete();
            File invalidBloomFilterFile = new File(currentBloomFilter.fullPath);
            invalidBloomFilterFile.delete();
        } else {
            // This scenario should not be possible.
            throw new HyracksDataException("Found LSM files with overlapping but not contained timetamp intervals.");
        }
    }
    // Sort valid files in reverse lexicographical order, such that newer
    // files come first.
    Collections.sort(validComparableDictBTreeFiles, recencyCmp);
    Collections.sort(validComparableDeletedKeysBTreeFiles, recencyCmp);
    Collections.sort(validComparableBloomFilterFiles, recencyCmp);
    Iterator<ComparableFileName> dictBTreeFileIter = validComparableDictBTreeFiles.iterator();
    Iterator<ComparableFileName> deletedKeysBTreeIter = validComparableDeletedKeysBTreeFiles.iterator();
    Iterator<ComparableFileName> bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
    while (dictBTreeFileIter.hasNext() && deletedKeysBTreeIter.hasNext()) {
        ComparableFileName cmpDictBTreeFile = dictBTreeFileIter.next();
        ComparableFileName cmpDeletedKeysBTreeFile = deletedKeysBTreeIter.next();
        ComparableFileName cmpBloomFilterFileName = bloomFilterFileIter.next();
        validFiles.add(new LSMComponentFileReferences(cmpDictBTreeFile.fileRef, cmpDeletedKeysBTreeFile.fileRef, cmpBloomFilterFileName.fileRef));
    }
    return validFiles;
}

Example 5 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalRTree method scheduleMerge.

// The only change the the schedule merge is the method used to create the
// opCtx. first line <- in schedule merge, we->
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
    rctx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
    LSMComponentFileReferences relMergeFileRefs = getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0), (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1));
    ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), rctx, buddyBTreeFields);
    // create the merge operation.
    LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
    ioScheduler.scheduleOperation(mergeOp);
}