Examples with LSMComponentFileReferences org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences used on opensource projects

Example 6 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMBTreeWithBuddyFileManager method getTransactionFileReferenceForCommit.

@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
    FilenameFilter transactionFilter;
    File dir = new File(baseDir);
    String[] files = dir.list(transactionFileNameFilter);
    if (files.length == 0) {
        return null;
    }
    if (files.length != 1) {
        throw new HyracksDataException("More than one transaction lock found:" + files.length);
    } else {
        transactionFilter = getTransactionFileFilter(true);
        String txnFileName = dir.getPath() + File.separator + files[0];
        // get the actual transaction files
        files = dir.list(transactionFilter);
        if (files.length < 3) {
            throw new HyracksDataException("LSM Btree with buddy transaction has less than 3 files :" + files.length);
        }
        try {
            Files.delete(Paths.get(txnFileName));
        } catch (IOException e) {
            throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
        }
    }
    File bTreeFile = null;
    File buddyBTreeFile = null;
    File bloomFilterFile = null;
    for (String fileName : files) {
        if (fileName.endsWith(BTREE_STRING)) {
            bTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BUDDY_BTREE_STRING)) {
            buddyBTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
            bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
        } else {
            throw new HyracksDataException("unrecognized file found = " + fileName);
        }
    }
    FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
    FileReference buddyBTreeFileRef = ioManager.resolveAbsolutePath(buddyBTreeFile.getAbsolutePath());
    FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
    return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef);
}

Example 7 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMIndexFileManagerTest method cleanInvalidFilesTest.

public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
    String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
    File f = new File(dirPath);
    f.mkdirs();
    FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
    ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
    fileManager.createDirs();
    List<FileReference> flushFiles = new ArrayList<>();
    List<FileReference> allFiles = new ArrayList<>();
    int numFileNames = 100;
    long sleepTime = 5;
    // Generate a bunch of flush files.
    for (int i = 0; i < numFileNames; i++) {
        LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
        flushFiles.add(relFlushFileRefs.getInsertIndexFileReference());
        Thread.sleep(sleepTime);
    }
    allFiles.addAll(flushFiles);
    // Simulate merging some of the flush files.
    // Merge range 0 to 4.
    FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
    allFiles.add(mergeFile1);
    // Merge range 5 to 9.
    FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
    allFiles.add(mergeFile2);
    // Merge range 10 to 19.
    FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
    allFiles.add(mergeFile3);
    // Merge range 20 to 29.
    FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
    allFiles.add(mergeFile4);
    // Merge range 50 to 79.
    FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
    allFiles.add(mergeFile5);
    // Simulate merging of merge files.
    FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
    allFiles.add(mergeFile6);
    FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
    allFiles.add(mergeFile7);
    // Create all files and set delete on exit for all files.
    for (FileReference fileRef : allFiles) {
        fileRef.getFile().createNewFile();
        fileRef.getFile().deleteOnExit();
    }
    // Populate expected valid flush files.
    List<String> expectedValidFiles = new ArrayList<>();
    for (int i = 30; i < 50; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    for (int i = 80; i < 100; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    // Populate expected valid merge files.
    expectedValidFiles.add(mergeFile5.getFile().getName());
    expectedValidFiles.add(mergeFile6.getFile().getName());
    expectedValidFiles.add(mergeFile7.getFile().getName());
    // Sort expected files.
    Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
    // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
    List<LSMComponentFileReferences> lsmComonentFileReference = fileManager.cleanupAndGetValidFiles();
    // Check actual files against expected files.
    assertEquals(expectedValidFiles.size(), lsmComonentFileReference.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), lsmComonentFileReference.get(i).getInsertIndexFileReference().getFile().getName());
    }
    // Make sure invalid files were removed from the IODevices.
    ArrayList<String> remainingFiles = new ArrayList<>();
    File dir = new File(dirPath);
    FilenameFilter filter = new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return !name.startsWith(".");
        }
    };
    String[] files = dir.list(filter);
    for (String aFilePath : files) {
        File aFile = new File(aFilePath);
        remainingFiles.add(aFile.getName());
    }
    Collections.sort(remainingFiles, fileManager.getFileNameComparator());
    // Check actual files in directory against expected files.
    assertEquals(expectedValidFiles.size(), remainingFiles.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
    }
}

Example 8 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class LSMBTreeFileManager method getTransactionFileReferenceForCommit.

@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
    FilenameFilter transactionFilter;
    File dir = new File(baseDir);
    String[] files = dir.list(transactionFileNameFilter);
    if (files.length == 0) {
        return null;
    }
    if (files.length != 1) {
        throw new HyracksDataException("More than one transaction lock found:" + files.length);
    } else {
        transactionFilter = getTransactionFileFilter(true);
        String txnFileName = dir.getPath() + File.separator + files[0];
        // get the actual transaction files
        files = dir.list(transactionFilter);
        if (files.length < 2) {
            throw new HyracksDataException("LSM Btree transaction has less than 2 files :" + files.length);
        }
        try {
            Files.delete(Paths.get(txnFileName));
        } catch (IOException e) {
            throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName);
        }
    }
    File bTreeFile = null;
    File bloomFilterFile = null;
    for (String fileName : files) {
        if (fileName.endsWith(BTREE_STRING)) {
            bTreeFile = new File(dir.getPath() + File.separator + fileName);
        } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
            bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
        } else {
            throw new HyracksDataException("unrecognized file found = " + fileName);
        }
    }
    FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
    FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
    return new LSMComponentFileReferences(bTreeFileRef, null, bloomFilterFileRef);
}

Example 9 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTree method scheduleMerge.

// The only reason to override the following method is that it uses a different context object
// in addition, determining whether or not to keep deleted tuples is different here
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    ExternalBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
    opCtx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    boolean returnDeletedTuples = false;
    if (version == 0) {
        if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
            returnDeletedTuples = true;
        }
    } else {
        if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != secondDiskComponents.get(secondDiskComponents.size() - 1)) {
            returnDeletedTuples = true;
        }
    }
    ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
    BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
    BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
    FileReference firstFile = firstBTree.getFileReference();
    FileReference lastFile = lastBTree.getFileReference();
    LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
    ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
    ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
}

Example 10 with LSMComponentFileReferences

use of org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences in project asterixdb by apache.

the class ExternalBTreeWithBuddy method getMergeTargetFileName.

protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents) throws HyracksDataException {
    BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getBTree();
    BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getBTree();
    FileReference firstFile = firstTree.getFileReference();
    FileReference lastFile = lastTree.getFileReference();
    LSMComponentFileReferences fileRefs = fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
    return fileRefs;
}