Examples with FeedPolicyAccessor org.apache.asterix.external.feed.policy.FeedPolicyAccessor used on opensource projects

Example 1 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class FeedOperations method buildStartFeedJob.

public static Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> buildStartFeedJob(SessionOutput sessionOutput, MetadataProvider metadataProvider, Feed feed, List<FeedConnection> feedConnections, ILangCompilationProvider compilationProvider, IStorageComponentProvider storageComponentProvider, DefaultStatementExecutorFactory qtFactory, IHyracksClientConnection hcc) throws Exception {
    FeedPolicyAccessor fpa = new FeedPolicyAccessor(new HashMap<>());
    // TODO: Change the default Datasource to use all possible partitions
    Pair<JobSpecification, IAdapterFactory> intakeInfo = buildFeedIntakeJobSpec(feed, metadataProvider, fpa);
    //TODO: Add feed policy accessor
    List<JobSpecification> jobsList = new ArrayList<>();
    // Construct the ingestion Job
    JobSpecification intakeJob = intakeInfo.getLeft();
    IAdapterFactory ingestionAdaptorFactory = intakeInfo.getRight();
    String[] ingestionLocations = ingestionAdaptorFactory.getPartitionConstraint().getLocations();
    // Add connection job
    for (FeedConnection feedConnection : feedConnections) {
        JobSpecification connectionJob = getConnectionJob(sessionOutput, metadataProvider, feedConnection, ingestionLocations, compilationProvider, storageComponentProvider, qtFactory, hcc);
        jobsList.add(connectionJob);
    }
    return Pair.of(combineIntakeCollectJobs(metadataProvider, feed, intakeJob, jobsList, feedConnections, ingestionLocations), intakeInfo.getRight().getPartitionConstraint());
}

Example 2 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryVarSizeFrameNoDiskNoDiscard.

/*
     * Spill = false
     * Discard = false
     * VarSizeFrame
     */
@Test
public void testMemoryVarSizeFrameNoDiskNoDiscard() {
    try {
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        int multiplier = 1;
        // add NUM_FRAMES times
        while ((multiplier <= framePool.remaining())) {
            handler.nextFrame(buffer);
            multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
        }
        // we can't satisfy the next request
        // Next call should block we will do it in a different thread
        Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
        // Check that the nextFrame didn't return
        if (result.isDone()) {
            Assert.fail();
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that number of stalled is not greater than 1
        Assert.assertTrue(handler.getNumStalled() <= 1);
        writer.unfreeze();
        handler.close();
        result.get();
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 3 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Slow next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        writer.setNextDuration(1);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that nextFrame was called
        writer.validate(false);
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 4 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testZeroMemoryVarSizeFrameWithDiskNoDiscard.

@org.junit.Test
public void testZeroMemoryVarSizeFrameWithDiskNoDiscard() {
    try {
        int numRounds = 5;
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        handler.nextFrame(buffer);
        Assert.assertEquals(0, handler.getNumProcessedInMemory());
        Assert.assertEquals(1, handler.getNumSpilled());
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            int multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            handler.nextFrame(buffer);
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    } finally {
        Assert.assertNull(cause);
    }
}

Example 5 with FeedPolicyAccessor

use of org.apache.asterix.external.feed.policy.FeedPolicyAccessor in project asterixdb by apache.

the class InputHandlerTest method testMemoryVarSizeFrameWithSpillWithDiscard.

/*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
@Test
public void testMemoryVarSizeFrameWithSpillWithDiscard() {
    try {
        int numberOfMemoryFrames = 50;
        int numberOfSpillFrames = 50;
        int notDiscarded = 0;
        int totalMinFrames = 0;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer1 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        ByteBuffer buffer2 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 2);
        ByteBuffer buffer3 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 3);
        ByteBuffer buffer4 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 4);
        ByteBuffer buffer5 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 5);
        while (true) {
            if (totalMinFrames + 1 < numberOfMemoryFrames) {
                handler.nextFrame(buffer1);
                notDiscarded++;
                totalMinFrames++;
            } else {
                break;
            }
            if (totalMinFrames + 2 < numberOfMemoryFrames) {
                notDiscarded++;
                totalMinFrames += 2;
                handler.nextFrame(buffer2);
            } else {
                break;
            }
            if (totalMinFrames + 3 < numberOfMemoryFrames) {
                notDiscarded++;
                totalMinFrames += 3;
                handler.nextFrame(buffer3);
            } else {
                break;
            }
        }
        // Now we need to verify that the frame pool memory has been consumed!
        Assert.assertTrue(framePool.remaining() < 3);
        Assert.assertEquals(0, handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        while (true) {
            if (handler.getNumSpilled() < numberOfSpillFrames) {
                notDiscarded++;
                handler.nextFrame(buffer3);
            } else {
                break;
            }
            if (handler.getNumSpilled() < numberOfSpillFrames) {
                notDiscarded++;
                handler.nextFrame(buffer4);
            } else {
                break;
            }
            if (handler.getNumSpilled() < numberOfSpillFrames) {
                notDiscarded++;
                handler.nextFrame(buffer5);
            } else {
                break;
            }
        }
        Assert.assertTrue(framePool.remaining() < 3);
        Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
        Assert.assertEquals(handler.framesOnDisk(), numberOfSpillFrames);
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals(0, handler.getNumDiscarded());
        // We can only discard one frame
        double numDiscarded = 0;
        boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        while (nextShouldDiscard) {
            handler.nextFrame(buffer5);
            numDiscarded++;
            nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        }
        Assert.assertTrue(framePool.remaining() < 3);
        Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
        Assert.assertEquals(0, handler.getNumStalled());
        Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
        // Next Call should block since we're exceeding the discard allowance
        Future<?> result = EXECUTOR.submit(new Pusher(buffer5, handler));
        if (result.isDone()) {
            Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
        }
        // consume memory frames
        writer.unfreeze();
        result.get();
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), notDiscarded + 1);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}