Examples with IBufferCache org.apache.hyracks.storage.common.buffercache.IBufferCache used on opensource projects

Example 11 with IBufferCache

use of org.apache.hyracks.storage.common.buffercache.IBufferCache in project asterixdb by apache.

the class BufferCacheTest method simpleOpenPinCloseTest.

@Test
public void simpleOpenPinCloseTest() throws HyracksException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    String fileName = getFileName();
    FileReference file = ioManager.resolve(fileName);
    bufferCache.createFile(file);
    int fileId = fmp.lookupFileId(file);
    int num = 10;
    int testPageId = 0;
    bufferCache.openFile(fileId);
    ICachedPage page = null;
    // tryPin should fail
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNull(page);
    // pin page should succeed
    page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), true);
    page.acquireWriteLatch();
    try {
        for (int i = 0; i < num; i++) {
            page.getBuffer().putInt(i * 4, i);
        }
        // try pin should succeed
        ICachedPage page2 = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        Assert.assertNotNull(page2);
        bufferCache.unpin(page2);
    } finally {
        page.releaseWriteLatch(true);
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    // This code is commented because the method pinSanityCheck in the BufferCache is commented.
    /*boolean exceptionThrown = false;

        // tryPin should fail since file is not open
        try {
            page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);

        // pin should fail since file is not open
        exceptionThrown = false;
        try {
            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), false);
        } catch (HyracksDataException e) {
            exceptionThrown = true;
        }
        Assert.assertTrue(exceptionThrown);*/
    // open file again
    bufferCache.openFile(fileId);
    // tryPin should succeed because page should still be cached
    page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, testPageId));
    Assert.assertNotNull(page);
    page.acquireReadLatch();
    try {
        // verify contents of page
        for (int i = 0; i < num; i++) {
            Assert.assertEquals(page.getBuffer().getInt(i * 4), i);
        }
    } finally {
        page.releaseReadLatch();
        bufferCache.unpin(page);
    }
    bufferCache.closeFile(fileId);
    bufferCache.close();
}

Example 12 with IBufferCache

use of org.apache.hyracks.storage.common.buffercache.IBufferCache in project asterixdb by apache.

the class TreeIndexStatsOperatorNodePushable method initialize.

@Override
public void initialize() throws HyracksDataException {
    treeIndexHelper.open();
    ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndexInstance();
    try {
        writer.open();
        IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext());
        IFileMapProvider fileMapProvider = storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext());
        LocalResource resource = treeIndexHelper.getResource();
        IIOManager ioManager = ctx.getIoManager();
        FileReference fileRef = ioManager.resolve(resource.getPath());
        int indexFileId = fileMapProvider.lookupFileId(fileRef);
        TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(), indexFileId, treeIndex.getRootPageId());
        TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex.getInteriorFrameFactory().createFrame(), treeIndex.getPageManager().createMetadataFrame());
        // Write the stats output as a single string field.
        FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
        ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
        DataOutput dos = tb.getDataOutput();
        tb.reset();
        utf8SerDer.serialize(stats.toString(), dos);
        tb.addFieldEndOffset();
        if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
            throw new HyracksDataException("Record size (" + tb.getSize() + ") larger than frame size (" + appender.getBuffer().capacity() + ")");
        }
        appender.write(writer, false);
    } catch (Exception e) {
        writer.fail();
        throw new HyracksDataException(e);
    } finally {
        try {
            writer.close();
        } finally {
            treeIndexHelper.close();
        }
    }
}

Example 13 with IBufferCache

use of org.apache.hyracks.storage.common.buffercache.IBufferCache in project asterixdb by apache.

the class LSMBTree method getMemoryComponentSize.

@Override
protected long getMemoryComponentSize(ILSMMemoryComponent c) {
    LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
    IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
    return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize();
}

Example 14 with IBufferCache

use of org.apache.hyracks.storage.common.buffercache.IBufferCache in project asterixdb by apache.

the class BTreeResource method createInstance.

@Override
public IIndex createInstance(INCServiceContext ctx) throws HyracksDataException {
    IBufferCache bufferCache = storageManager.getBufferCache(ctx);
    IIOManager ioManager = ctx.getIoManager();
    FileReference resourceRef = ioManager.resolve(path);
    return BTreeUtils.createBTree(bufferCache, storageManager.getFileMapProvider(ctx), typeTraits, comparatorFactories, BTreeLeafFrameType.REGULAR_NSM, resourceRef, pageManagerFactory.createPageManager(bufferCache));
}

Example 15 with IBufferCache

use of org.apache.hyracks.storage.common.buffercache.IBufferCache in project asterixdb by apache.

the class RTreeSearchCursorTest method rangeSearchTest.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void rangeSearchTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING RANGE SEARCH CURSOR FOR RTREE");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    // Declare fields.
    int fieldCount = 5;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    typeTraits[3] = IntegerPointable.TYPE_TRAITS;
    typeTraits[4] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare keys.
    int keyFieldCount = 4;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // create value providers
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(cmpFactories.length, IntegerPointable.FACTORY);
    RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
    ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
    IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
    IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference(), false);
    rtree.create();
    rtree.activate();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    int numInserts = 10000;
    ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<>();
    for (int i = 0; i < numInserts; i++) {
        int p1x = rnd.nextInt();
        int p1y = rnd.nextInt();
        int p2x = rnd.nextInt();
        int p2y = rnd.nextInt();
        int pk = rnd.nextInt();
        ;
        TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk);
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
        RTreeCheckTuple checkTuple = new RTreeCheckTuple(fieldCount, keyFieldCount);
        checkTuple.appendField(Math.min(p1x, p2x));
        checkTuple.appendField(Math.min(p1y, p2y));
        checkTuple.appendField(Math.max(p1x, p2x));
        checkTuple.appendField(Math.max(p1y, p2y));
        checkTuple.appendField(pk);
        checkTuples.add(checkTuple);
    }
    // Build key.
    ArrayTupleBuilder keyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference key = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
    MultiComparator cmp = MultiComparator.create(cmpFactories);
    ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
    SearchPredicate searchPredicate = new SearchPredicate(key, cmp);
    RTreeCheckTuple keyCheck = (RTreeCheckTuple) rTreeTestUtils.createCheckTupleFromTuple(key, fieldSerdes, keyFieldCount);
    HashMultiSet<RTreeCheckTuple> expectedResult = rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
    rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
    indexAccessor.search(searchCursor, searchPredicate);
    rTreeTestUtils.checkExpectedResults(searchCursor, expectedResult, fieldSerdes, keyFieldCount, null);
    rtree.deactivate();
    rtree.destroy();
}