Examples with IFileMapProvider org.apache.hyracks.storage.common.file.IFileMapProvider used on opensource projects

Example 1 with IFileMapProvider

use of org.apache.hyracks.storage.common.file.IFileMapProvider in project asterixdb by apache.

the class BTreeRunner method init.

@Override
protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException {
    IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
    TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
    bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    btree = BTreeUtils.createBTree(bufferCache, fmp, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, file, freePageManager);
}

Example 2 with IFileMapProvider

use of org.apache.hyracks.storage.common.file.IFileMapProvider in project asterixdb by apache.

the class TestStorageManagerComponentHolder method getBufferCache.

public static synchronized IBufferCache getBufferCache(INCServiceContext ctx) {
    if (bufferCache == null) {
        ICacheMemoryAllocator allocator = new HeapBufferAllocator();
        IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, pageSize, numPages);
        IFileMapProvider fileMapProvider = getFileMapProvider();
        bufferCache = new BufferCache(ctx.getIoManager(), prs, new DelayPageCleanerPolicy(1000), (IFileMapManager) fileMapProvider, maxOpenFiles, threadFactory);
    }
    return bufferCache;
}

Example 3 with IFileMapProvider

use of org.apache.hyracks.storage.common.file.IFileMapProvider in project asterixdb by apache.

the class LSMInvertedIndexLocalResource method createInstance.

@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
    IIOManager ioManager = serviceCtx.getIoManager();
    FileReference file = ioManager.resolve(path);
    List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
    IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx);
    IFileMapProvider fileMapManager = storageManager.getFileMapProvider(serviceCtx);
    ILSMMergePolicy mergePolicy = mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx);
    ILSMIOOperationScheduler ioScheduler = ioSchedulerProvider.getIoScheduler(serviceCtx);
    if (isPartitioned) {
        return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory);
    } else {
        return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, fileMapManager, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx), ioScheduler, ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory);
    }
}

Example 4 with IFileMapProvider

use of org.apache.hyracks.storage.common.file.IFileMapProvider in project asterixdb by apache.

the class TreeIndexStatsOperatorNodePushable method initialize.

@Override
public void initialize() throws HyracksDataException {
    treeIndexHelper.open();
    ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndexInstance();
    try {
        writer.open();
        IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext());
        IFileMapProvider fileMapProvider = storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext());
        LocalResource resource = treeIndexHelper.getResource();
        IIOManager ioManager = ctx.getIoManager();
        FileReference fileRef = ioManager.resolve(resource.getPath());
        int indexFileId = fileMapProvider.lookupFileId(fileRef);
        TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(), indexFileId, treeIndex.getRootPageId());
        TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex.getInteriorFrameFactory().createFrame(), treeIndex.getPageManager().createMetadataFrame());
        // Write the stats output as a single string field.
        FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
        ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
        DataOutput dos = tb.getDataOutput();
        tb.reset();
        utf8SerDer.serialize(stats.toString(), dos);
        tb.addFieldEndOffset();
        if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
            throw new HyracksDataException("Record size (" + tb.getSize() + ") larger than frame size (" + appender.getBuffer().capacity() + ")");
        }
        appender.write(writer, false);
    } catch (Exception e) {
        writer.fail();
        throw new HyracksDataException(e);
    } finally {
        try {
            writer.close();
        } finally {
            treeIndexHelper.close();
        }
    }
}

Example 5 with IFileMapProvider

use of org.apache.hyracks.storage.common.file.IFileMapProvider in project asterixdb by apache.

the class BufferCacheTest method simpleOpenPinCloseTest.

@Test
public void simpleOpenPinCloseTest() throws HyracksException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    String fileName = getFileName();
    FileReference file = ioManager.resolve(fileName);
    bufferCache.createFile(file);
    int fileId = fmp.lookupFileId(file);
    int num = 10;
    int testPageId = 0;
    bufferCache.openFile(fileId);
    ICachedPage page = null;
    // tryPin should fail
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNull(page);
    // pin page should succeed
    page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), true);
    page.acquireWriteLatch();
    try {
        for (int i = 0; i < num; i++) {
            page.getBuffer().putInt(i * 4, i);
        }
        // try pin should succeed
        ICachedPage page2 = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        Assert.assertNotNull(page2);
        bufferCache.unpin(page2);
    } finally {
        page.releaseWriteLatch(true);
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    // This code is commented because the method pinSanityCheck in the BufferCache is commented.
    /*boolean exceptionThrown = false;

        // tryPin should fail since file is not open
        try {
            page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);

        // pin should fail since file is not open
        exceptionThrown = false;
        try {
            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), false);
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);*/
    // open file again
    bufferCache.openFile(fileId);
    // tryPin should succeed because page should still be cached
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNotNull(page);
    page.acquireReadLatch();
    try {
        // verify contents of page
        for (int i = 0; i < num; i++) {
            Assert.assertEquals(page.getBuffer().getInt(i * 4), i);
        }
    } finally {
        page.releaseReadLatch();
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    bufferCache.close();
}