Examples with TestFrameWriter org.apache.hyracks.api.test.TestFrameWriter used on opensource projects

Example 1 with TestFrameWriter

use of org.apache.hyracks.api.test.TestFrameWriter in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Slow next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        writer.setNextDuration(1);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        // Check that nextFrame was called
        writer.validate(false);
        handler.close();
        Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 2 with TestFrameWriter

use of org.apache.hyracks.api.test.TestFrameWriter in project asterixdb by apache.

the class InputHandlerTest method testZeroMemoryVarSizeFrameWithDiskNoDiscard.

@org.junit.Test
public void testZeroMemoryVarSizeFrameWithDiskNoDiscard() {
    try {
        int numRounds = 5;
        Random random = new Random();
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
        handler.nextFrame(buffer);
        Assert.assertEquals(0, handler.getNumProcessedInMemory());
        Assert.assertEquals(1, handler.getNumSpilled());
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            int multiplier = random.nextInt(10) + 1;
            buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            handler.nextFrame(buffer);
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    } finally {
        Assert.assertNull(cause);
    }
}

Example 3 with TestFrameWriter

use of org.apache.hyracks.api.test.TestFrameWriter in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Very fast next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 4 with TestFrameWriter

use of org.apache.hyracks.api.test.TestFrameWriter in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testEmptyFrames.

@Test
public void testEmptyFrames() throws Exception {
    try {
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        List<TestFrameWriter> recipients = new ArrayList<>();
        try {
            partitioner.open();
            FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
            for (IFrameWriter writer : partitionWriterFactory.getWriters().values()) {
                recipients.add((TestFrameWriter) writer);
            }
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 1);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 2);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 3);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
        } catch (Throwable t) {
            partitioner.fail();
            throw t;
        } finally {
            partitioner.close();
        }
        for (TestFrameWriter writer : recipients) {
            Assert.assertEquals(writer.nextFrameCount(), 4);
            Assert.assertEquals(writer.closeCount(), 1);
        }
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}

Example 5 with TestFrameWriter

use of org.apache.hyracks.api.test.TestFrameWriter in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testMessageFitsWithTuples.

@Test
public void testMessageFitsWithTuples() throws Exception {
    try {
        // Routing will be round robin
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        FieldType[] types = { FieldType.Integer64, FieldType.Double, FieldType.Boolean, FieldType.String };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        PartitionWithMessageDataWriter partitioner = (PartitionWithMessageDataWriter) connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        partitioner.open();
        FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
        List<TestFrameWriter> recipients = new ArrayList<>();
        for (int i = 0; i < partitionWriterFactory.getWriters().values().size(); i++) {
            recipients.add(partitionWriterFactory.getWriters().get(i));
        }
        TestTupleGenerator ttg = new TestTupleGenerator(types, STRING_FIELD_SIZES, true);
        VSizeFrame frame = new VSizeFrame(ctx);
        FrameTupleAppender appender = new FrameTupleAppender(frame);
        for (int count = 0; count < NUMBER_OF_CONSUMERS; count++) {
            ITupleReference tuple = ttg.next();
            appender.append(tuple);
        }
        partitioner.nextFrame(frame.getBuffer());
        partitioner.flush();
        Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(1).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(2).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(3).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(4).nextFrameCount(), 1);
        for (TestFrameWriter writer : recipients) {
            fta.reset(writer.getLastFrame());
            Assert.assertEquals(fta.getTupleCount(), 2);
            FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
            Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
        }
        partitioner.close();
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}