Examples with IHyracksTaskContext org.apache.hyracks.api.context.IHyracksTaskContext used on opensource projects

Example 36 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class BTreeRunner method init.

@Override
protected void init(int pageSize, int numPages, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException {
    IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
    TestStorageManagerComponentHolder.init(pageSize, numPages, MAX_OPEN_FILES);
    bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    btree = BTreeUtils.createBTree(bufferCache, fmp, typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, file, freePageManager);
}

Example 37 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class FeedSpillerUnitTest method testWriteFixedSizeSpill.

/*
     * Spiller spills each 1024 frames to a file
     */
/*
     * The following tests:
     * 1. Test writer only.
     * Write 1023 frames.
     * Check only 1 file exist.
     * Write 1 more frame
     * Check two files exist.
     * Insert 1023 more frames.
     * Check that we still have 2 files.
     * Write 1 more frame.
     * Check that we have 3 files.
     * Check that we have 2048 frames to read.
     * Close the spiller
     * Check files were deleted.
     */
@org.junit.Test
public void testWriteFixedSizeSpill() {
    try {
        removeSpillFiles();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        FrameSpiller spiller = new FrameSpiller(ctx, TEST_DATAVERSE + "_" + TEST_FEED + "_" + TEST_DATASET, new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE));
        spiller.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        for (int i = 0; i < 1022; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(1023, spiller.remaining());
        Assert.assertEquals(1, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(1024, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        for (int i = 0; i < 1023; i++) {
            spiller.spill(frame.getBuffer());
        }
        Assert.assertEquals(2047, spiller.remaining());
        Assert.assertEquals(2, countSpillFiles());
        spiller.spill(frame.getBuffer());
        Assert.assertEquals(2048, spiller.remaining());
        Assert.assertEquals(3, countSpillFiles());
        spiller.close();
        Assert.assertEquals(0, spiller.remaining());
        Assert.assertEquals(0, countSpillFiles());
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail(th.getMessage());
    }
}

Example 38 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryVarSizeFrameNoDiskNoDiscard.

/*
     * Spill = false
     * Discard = false
     * VarSizeFrame
     */
@Test
public void testMemoryVarSizeFrameNoDiskNoDiscard() {
    try {
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        int multiplier = 1;
        // add NUM_FRAMES times
        while ((multiplier <= framePool.remaining())) {
            handler.nextFrame(buffer);
            multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
        }
        // we can't satisfy the next request
        // Next call should block we will do it in a different thread
        Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
        // Check that the nextFrame didn't return
        if (result.isDone()) {
            Assert.fail();
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that number of stalled is not greater than 1
        Assert.assertTrue(handler.getNumStalled() <= 1);
        writer.unfreeze();
        handler.close();
        result.get();
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 39 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Slow next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        writer.setNextDuration(1);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that nextFrame was called
        writer.validate(false);
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 40 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testZeroMemoryVarSizeFrameWithDiskNoDiscard.

@org.junit.Test
public void testZeroMemoryVarSizeFrameWithDiskNoDiscard() {
    try {
        int numRounds = 5;
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        handler.nextFrame(buffer);
        Assert.assertEquals(0, handler.getNumProcessedInMemory());
        Assert.assertEquals(1, handler.getNumSpilled());
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            int multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            handler.nextFrame(buffer);
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    } finally {
        Assert.assertNull(cause);
    }
}