Examples with VSizeFrame org.apache.hyracks.api.comm.VSizeFrame used on opensource projects

Example 36 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class LSMInsertDeleteOperatorNodePushable method open.

@Override
public void open() throws HyracksDataException {
    accessor = new FrameTupleAccessor(inputRecDesc);
    writeBuffer = new VSizeFrame(ctx);
    appender = new FrameTupleAppender(writeBuffer);
    indexHelper.open();
    lsmIndex = (AbstractLSMIndex) indexHelper.getIndexInstance();
    try {
        if (isPrimary && ctx.getSharedObject() != null) {
            PrimaryIndexLogMarkerCallback callback = new PrimaryIndexLogMarkerCallback(lsmIndex);
            TaskUtil.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
        }
        writer.open();
        modCallback = modOpCallbackFactory.createModificationOperationCallback(indexHelper.getResource(), ctx, this);
        indexAccessor = lsmIndex.createAccessor(modCallback, NoOpOperationCallback.INSTANCE);
        if (tupleFilterFactory != null) {
            tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
            frameTuple = new FrameTupleReference();
        }
        INcApplicationContext runtimeCtx = (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
        LSMIndexUtil.checkAndSetFirstLSN(lsmIndex, runtimeCtx.getTransactionSubsystem().getLogManager());
    } catch (Throwable th) {
        throw new HyracksDataException(th);
    }
}

Example 37 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class IndexSearchOperatorNodePushable method open.

@Override
public void open() throws HyracksDataException {
    writer.open();
    indexHelper.open();
    index = indexHelper.getIndexInstance();
    accessor = new FrameTupleAccessor(inputRecDesc);
    if (retainMissing) {
        int fieldCount = getFieldCount();
        nonMatchTupleBuild = new ArrayTupleBuilder(fieldCount);
        buildMissingTuple(fieldCount, nonMatchTupleBuild, nonMatchWriter);
    } else {
        nonMatchTupleBuild = null;
    }
    if (appendIndexFilter) {
        int numIndexFilterFields = index.getNumOfFilterFields();
        nonFilterTupleBuild = new ArrayTupleBuilder(numIndexFilterFields);
        buildMissingTuple(numIndexFilterFields, nonFilterTupleBuild, nonMatchWriter);
    }
    try {
        searchPred = createSearchPredicate();
        tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
        dos = tb.getDataOutput();
        appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
        ISearchOperationCallback searchCallback = searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
        indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
        cursor = createCursor();
        if (retainInput) {
            frameTuple = new FrameTupleReference();
        }
    } catch (Exception e) {
        throw new HyracksDataException(e);
    }
}

Example 38 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameWithSpillWithDiscard.

/*
     * Spill = true;
     * Discard = true
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameWithSpillWithDiscard() {
    try {
        int numberOfMemoryFrames = 50;
        int numberOfSpillFrames = 50;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        for (int i = 0; i < numberOfMemoryFrames; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Now we need to verify that the frame pool memory has been consumed!
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals(numberOfMemoryFrames, handler.getTotal());
        Assert.assertEquals(0, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        for (int i = 0; i < numberOfSpillFrames; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals(numberOfMemoryFrames + numberOfSpillFrames, handler.getTotal());
        Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        // We can only discard one frame
        double numDiscarded = 0;
        boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        while (nextShouldDiscard) {
            handler.nextFrame(frame.getBuffer());
            numDiscarded++;
            nextShouldDiscard = (numDiscarded + 1.0) / (handler.getTotal() + 1.0) <= fpa.getMaxFractionDiscard();
        }
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals((int) (numberOfMemoryFrames + numberOfSpillFrames + numDiscarded), handler.getTotal());
        Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
        // Next Call should block since we're exceeding the discard allowance
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        if (result.isDone()) {
            Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
        } else {
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
        }
        // consume memory frames
        writer.unfreeze();
        result.get();
        handler.close();
        Assert.assertTrue(result.isDone());
        Assert.assertEquals(writer.nextFrameCount(), numberOfMemoryFrames + numberOfSpillFrames + 1);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 39 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FeedSpillerUnitTest method testWriteReadFixedSizeSpill.

/*
     * 2. Test writer and reader
     * Write 1047 and Read 1042 frames.
     * Check only 1 file exists.
     * Check switchToMemory() returns false.
     * Check remaining() returns 5.
     * Read the remaining 5 frames.
     * Check switchToMemory() returns true.
     * Check that the reader returns null
     * Close the spiller.
     * Check files were deleted.
     */
@org.junit.Test
public void testWriteReadFixedSizeSpill() {
    try {
        removeSpillFiles();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        FrameSpiller spiller = new FrameSpiller(ctx, TEST_DATAVERSE + "_" + TEST_FEED + "_" + TEST_DATASET, new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE));
        spiller.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        for (int i = 0; i < 1047; i++) {
            spiller.spill(frame.getBuffer());
        }
        for (int i = 0; i < 1042; i++) {
            spiller.next();
        }
        Assert.assertEquals(5, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        Assert.assertEquals(false, spiller.switchToMemory());
        for (int i = 0; i < 4; i++) {
            spiller.next();
        }
        Assert.assertEquals(false, spiller.next() == null);
        Assert.assertEquals(true, spiller.next() == null);
        Assert.assertEquals(true, spiller.switchToMemory());
        Assert.assertEquals(1, countSpillFiles());
        spiller.close();
        Assert.assertEquals(0, spiller.remaining());
        Assert.assertEquals(0, countSpillFiles());
        Assert.assertEquals(true, spiller.switchToMemory());
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail(th.getMessage());
    }
}

Example 40 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputHandlerTest method testZeroMemoryFixedSizeFrameWithDiskNoDiscard.

@Test
public void testZeroMemoryFixedSizeFrameWithDiskNoDiscard() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        handler.nextFrame(frame.getBuffer());
        Assert.assertEquals(0, handler.getNumProcessedInMemory());
        Assert.assertEquals(1, handler.getNumSpilled());
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    } finally {
        Assert.assertNull(cause);
    }
}