Examples with FeedPolicyAccessor org.apache.asterix.external.feed.policy.FeedPolicyAccessor used on opensource projects

Example 11 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameWithSpillWithDiscard.

/*
     * Spill = true;
     * Discard = true
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameWithSpillWithDiscard() {
    try {
        int numberOfMemoryFrames = 50;
        int numberOfSpillFrames = 50;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        for (int i = 0; i < numberOfMemoryFrames; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Now we need to verify that the frame pool memory has been consumed!
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals(numberOfMemoryFrames, handler.getTotal());
        Assert.assertEquals(0, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        for (int i = 0; i < numberOfSpillFrames; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals(numberOfMemoryFrames + numberOfSpillFrames, handler.getTotal());
        Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        // We can only discard one frame
        double numDiscarded = 0;
        boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        while (nextShouldDiscard) {
            handler.nextFrame(frame.getBuffer());
            numDiscarded++;
            nextShouldDiscard = (numDiscarded + 1.0) / (handler.getTotal() + 1.0) <= fpa.getMaxFractionDiscard();
        }
        Assert.assertEquals(0, framePool.remaining());
        Assert.assertEquals((int) (numberOfMemoryFrames + numberOfSpillFrames + numDiscarded), handler.getTotal());
        Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
        // Next Call should block since we're exceeding the discard allowance
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        if (result.isDone()) {
            Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
        } else {
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
        }
        // consume memory frames
        writer.unfreeze();
        result.get();
        handler.close();
        Assert.assertTrue(result.isDone());
        Assert.assertEquals(writer.nextFrameCount(), numberOfMemoryFrames + numberOfSpillFrames + 1);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 12 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testZeroMemoryFixedSizeFrameWithDiskNoDiscard.

@Test
public void testZeroMemoryFixedSizeFrameWithDiskNoDiscard() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        handler.nextFrame(frame.getBuffer());
        Assert.assertEquals(0, handler.getNumProcessedInMemory());
        Assert.assertEquals(1, handler.getNumSpilled());
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    } finally {
        Assert.assertNull(cause);
    }
}

Example 13 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscard.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscard() {
    try {
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Next call should block we will do it in a different thread
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        // Check that the nextFrame didn't return
        if (result.isDone()) {
            Assert.fail();
        } else {
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
            // Check that no records were discarded
            // Check that the inputHandler subscribed to the framePool
            // Check that number of stalled is not greater than 1
            Assert.assertTrue(handler.getNumStalled() <= 1);
            writer.kick();
        }
        result.get();
        writer.unfreeze();
        handler.close();
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 14 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoSpillWithDiscard.

/*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameNoSpillWithDiscard() {
    try {
        int discardTestFrames = 100;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, true, DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, discardTestFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < discardTestFrames; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Next 5 calls call should NOT block but should discard.
        double numDiscarded = 0.0;
        boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        while (nextShouldDiscard) {
            handler.nextFrame(frame.getBuffer());
            numDiscarded++;
            nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        }
        // Next Call should block since we're exceeding the discard allowance
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        if (result.isDone()) {
            Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
        } else {
            // Check that no records were discarded
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
            // Check that one frame is spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
        }
        // consume memory frames
        writer.unfreeze();
        result.get();
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), discardTestFrames + 1);
    // exit
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 15 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method createFeedPolicyAccessor.

/*
     * Testing the following scenarios
     * 01. Positive Frames memory budget with fixed size frames, no spill, no discard.
     * 02. Positive Frames memory budget with variable size frames, no spill, no discard.
     * 03. Positive Frames memory budget with fixed size frames, with spill, no discard.
     * 04. Positive Frames memory budget with variable size frames, with spill, no discard.
     * 05. Positive Frames memory budget with fixed size frames, no spill, with discard.
     * 06. Positive Frames memory budget with variable size frames, no spill, with discard.
     * 07. Positive Frames memory budget with fixed size frames, with spill, with discard.
     * 08. Positive Frames memory budget with variable size frames, with spill, with discard.
     * 09. 0 Frames memory budget with fixed size frames, with spill, no discard.
     * 10. 0 Frames memory budget with variable size frames, with spill, no discard.
     * 11. TODO 0 Frames memory budget with fixed size frames, with spill, with discard.
     * 12. TODO 0 Frames memory budget with variable size frames, with spill, with discard.
     * 13. TODO Test exception handling with Open, NextFrame,Flush,Close,Fail exception throwing FrameWriter
     * 14. TODO Test exception while waiting for subscription
     */
private static FeedPolicyAccessor createFeedPolicyAccessor(boolean spill, boolean discard, long spillBudget, float discardFraction) {
    FeedPolicyAccessor fpa = Mockito.mock(FeedPolicyAccessor.class);
    Mockito.when(fpa.flowControlEnabled()).thenReturn(true);
    Mockito.when(fpa.spillToDiskOnCongestion()).thenReturn(spill);
    Mockito.when(fpa.getMaxSpillOnDisk()).thenReturn(spillBudget);
    Mockito.when(fpa.discardOnCongestion()).thenReturn(discard);
    Mockito.when(fpa.getMaxFractionDiscard()).thenReturn(discardFraction);
    return fpa;
}