Examples with IHyracksTaskContext org.apache.hyracks.api.context.IHyracksTaskContext used on opensource projects

Example 21 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameWithSpillNoDiscard.

/*
     * Spill = true;
     * Discard = false;
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameWithSpillNoDiscard() {
    try {
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Next call should NOT block. we will do it in a different thread
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        result.get();
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that one frame is spilled
        Assert.assertEquals(handler.getNumSpilled(), 1);
        // consume memory frames
        writer.unfreeze();
        handler.close();
        Assert.assertEquals(handler.framesOnDisk(), 0);
    // exit
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 22 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Very fast next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 23 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryVariableSizeFrameNoSpillWithDiscard.

/*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
@Test
public void testMemoryVariableSizeFrameNoSpillWithDiscard() {
    try {
        int discardTestFrames = 100;
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, true, DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, discardTestFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        // add NUM_FRAMES times
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        int multiplier = 1;
        int numFrames = 0;
        // add NUM_FRAMES times
        while ((multiplier <= framePool.remaining())) {
            numFrames++;
            handler.nextFrame(buffer);
            multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
        }
        // Next call should NOT block but should discard.
        double numDiscarded = 0.0;
        boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        while (nextShouldDiscard) {
            handler.nextFrame(buffer);
            numDiscarded++;
            nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa.getMaxFractionDiscard();
        }
        Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
        if (result.isDone()) {
            Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
        } else {
            // Check that no records were discarded
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
            // Check that one frame is spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
        }
        // consume memory frames
        writer.unfreeze();
        result.get();
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), numFrames + 1);
    // exit
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 24 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class InputHandlerTest method testMemoryVarSizeFrameWithSpillNoDiscard.

/*
     * Spill = true;
     * Discard = false;
     * Variable size frames
     */
@Test
public void testMemoryVarSizeFrameWithSpillNoDiscard() {
    for (int k = 0; k < 1000; k++) {
        try {
            Random random = new Random();
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
            int multiplier = 1;
            int numOfBuffersInMemory = 0;
            // add NUM_FRAMES times
            while ((multiplier <= framePool.remaining())) {
                numOfBuffersInMemory++;
                handler.nextFrame(buffer);
                multiplier = random.nextInt(10) + 1;
                buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            }
            // Next call should Not block. we will do it in a different thread
            Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
            result.get();
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that one frame is spilled
            Assert.assertEquals(handler.getNumSpilled(), 1);
            // consume memory frames
            while (numOfBuffersInMemory > 1) {
                writer.kick();
                numOfBuffersInMemory--;
            }
            // There should be 1 frame on disk
            Assert.assertEquals(1, handler.framesOnDisk());
            writer.unfreeze();
            handler.close();
            Assert.assertEquals(0, handler.framesOnDisk());
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
    }
    Assert.assertNull(cause);
}

Example 25 with IHyracksTaskContext

use of org.apache.hyracks.api.context.IHyracksTaskContext in project asterixdb by apache.

the class MillisecondsFromDayTimeDurationDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr0 = new VoidPointable();

                private IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInt64> int64Serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);

                AMutableInt64 aInt64 = new AMutableInt64(0);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval0.evaluate(tuple, argPtr0);
                    byte[] bytes = argPtr0.getByteArray();
                    int offset = argPtr0.getStartOffset();
                    if (bytes[offset] != ATypeTag.SERIALIZED_DAY_TIME_DURATION_TYPE_TAG) {
                        throw new TypeMismatchException(getIdentifier(), 0, bytes[offset], ATypeTag.SERIALIZED_DAY_TIME_DURATION_TYPE_TAG);
                    }
                    aInt64.setValue(ADayTimeDurationSerializerDeserializer.getDayTime(bytes, offset + 1));
                    int64Serde.serialize(aInt64, out);
                    result.set(resultStorage);
                }
            };
        }
    };
}