Examples with FileReference org.apache.hyracks.api.io.FileReference used on opensource projects

Example 21 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class AbstractLSMRTree method getMergeFileReferences.

@Override
protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, ILSMDiskComponent lastComponent) throws HyracksDataException {
    RTree firstTree = ((LSMRTreeDiskComponent) firstComponent).getRTree();
    RTree lastTree = ((LSMRTreeDiskComponent) lastComponent).getRTree();
    FileReference firstFile = firstTree.getFileReference();
    FileReference lastFile = lastTree.getFileReference();
    return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
}

Example 22 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class BufferCacheTest method simpleOpenPinCloseTest.

@Test
public void simpleOpenPinCloseTest() throws HyracksException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    String fileName = getFileName();
    FileReference file = ioManager.resolve(fileName);
    bufferCache.createFile(file);
    int fileId = fmp.lookupFileId(file);
    int num = 10;
    int testPageId = 0;
    bufferCache.openFile(fileId);
    ICachedPage page = null;
    // tryPin should fail
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNull(page);
    // pin page should succeed
    page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), true);
    page.acquireWriteLatch();
    try {
        for (int i = 0; i < num; i++) {
            page.getBuffer().putInt(i * 4, i);
        }
        // try pin should succeed
        ICachedPage page2 = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        Assert.assertNotNull(page2);
        bufferCache.unpin(page2);
    } finally {
        page.releaseWriteLatch(true);
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    // This code is commented because the method pinSanityCheck in the BufferCache is commented.
    /*boolean exceptionThrown = false;

        // tryPin should fail since file is not open
        try {
            page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);

        // pin should fail since file is not open
        exceptionThrown = false;
        try {
            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), false);
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);*/
    // open file again
    bufferCache.openFile(fileId);
    // tryPin should succeed because page should still be cached
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNotNull(page);
    page.acquireReadLatch();
    try {
        // verify contents of page
        for (int i = 0; i < num; i++) {
            Assert.assertEquals(page.getBuffer().getInt(i * 4), i);
        }
    } finally {
        page.releaseReadLatch();
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    bufferCache.close();
}

Example 23 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class IOManagerPathTest method testPrefixNames.

@Test
public void testPrefixNames() throws HyracksDataException {
    IODeviceHandle shorter = new IODeviceHandle(new File("/tmp/tst/1"), "storage");
    IODeviceHandle longer = new IODeviceHandle(new File("/tmp/tst/11"), "storage");
    IOManager ioManager = new IOManager(Arrays.asList(new IODeviceHandle[] { shorter, longer }), new DefaultDeviceResolver());
    FileReference f = ioManager.resolveAbsolutePath("/tmp/tst/11/storage/Foo_idx_foo/my_btree");
    Assert.assertEquals("/tmp/tst/11/storage/Foo_idx_foo/my_btree", f.getAbsolutePath());
}

Example 24 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class OnDiskInvertedIndexLifecycleTest method setup.

@Override
public void setup() throws Exception {
    harness.setUp();
    ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
    IBinaryComparatorFactory[] tokenCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) };
    ITypeTraits[] invListTypeTraits = new ITypeTraits[] { IntegerPointable.TYPE_TRAITS };
    IBinaryComparatorFactory[] invListCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
    IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
    FileReference btreeFile = harness.getIOManager().resolveAbsolutePath(harness.getInvListsFileRef().getFile().getAbsolutePath() + "_btree");
    index = new OnDiskInvertedIndex(harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, harness.getInvListsFileRef(), btreeFile, harness.getMetadataPageManagerFactory());
}

Example 25 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class LSMIndexFileManagerTest method cleanInvalidFilesTest.

public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
    String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
    File f = new File(dirPath);
    f.mkdirs();
    FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
    ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
    fileManager.createDirs();
    List<FileReference> flushFiles = new ArrayList<>();
    List<FileReference> allFiles = new ArrayList<>();
    int numFileNames = 100;
    long sleepTime = 5;
    // Generate a bunch of flush files.
    for (int i = 0; i < numFileNames; i++) {
        LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
        flushFiles.add(relFlushFileRefs.getInsertIndexFileReference());
        Thread.sleep(sleepTime);
    }
    allFiles.addAll(flushFiles);
    // Simulate merging some of the flush files.
    // Merge range 0 to 4.
    FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
    allFiles.add(mergeFile1);
    // Merge range 5 to 9.
    FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
    allFiles.add(mergeFile2);
    // Merge range 10 to 19.
    FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
    allFiles.add(mergeFile3);
    // Merge range 20 to 29.
    FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
    allFiles.add(mergeFile4);
    // Merge range 50 to 79.
    FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
    allFiles.add(mergeFile5);
    // Simulate merging of merge files.
    FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
    allFiles.add(mergeFile6);
    FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
    allFiles.add(mergeFile7);
    // Create all files and set delete on exit for all files.
    for (FileReference fileRef : allFiles) {
        fileRef.getFile().createNewFile();
        fileRef.getFile().deleteOnExit();
    }
    // Populate expected valid flush files.
    List<String> expectedValidFiles = new ArrayList<>();
    for (int i = 30; i < 50; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    for (int i = 80; i < 100; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    // Populate expected valid merge files.
    expectedValidFiles.add(mergeFile5.getFile().getName());
    expectedValidFiles.add(mergeFile6.getFile().getName());
    expectedValidFiles.add(mergeFile7.getFile().getName());
    // Sort expected files.
    Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
    // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
    List<LSMComponentFileReferences> lsmComonentFileReference = fileManager.cleanupAndGetValidFiles();
    // Check actual files against expected files.
    assertEquals(expectedValidFiles.size(), lsmComonentFileReference.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), lsmComonentFileReference.get(i).getInsertIndexFileReference().getFile().getName());
    }
    // Make sure invalid files were removed from the IODevices.
    ArrayList<String> remainingFiles = new ArrayList<>();
    File dir = new File(dirPath);
    FilenameFilter filter = new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return !name.startsWith(".");
        }
    };
    String[] files = dir.list(filter);
    for (String aFilePath : files) {
        File aFile = new File(aFilePath);
        remainingFiles.add(aFile.getName());
    }
    Collections.sort(remainingFiles, fileManager.getFileNameComparator());
    // Check actual files in directory against expected files.
    assertEquals(expectedValidFiles.size(), remainingFiles.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
    }
}