Examples with IFrameWriter org.apache.hyracks.api.comm.IFrameWriter used on opensource projects

Example 1 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testEmptyFrames.

@Test
public void testEmptyFrames() throws Exception {
    try {
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        List<TestFrameWriter> recipients = new ArrayList<>();
        try {
            partitioner.open();
            FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
            for (IFrameWriter writer : partitionWriterFactory.getWriters().values()) {
                recipients.add((TestFrameWriter) writer);
            }
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 1);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 2);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 3);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
        } catch (Throwable t) {
            partitioner.fail();
            throw t;
        } finally {
            partitioner.close();
        }
        for (TestFrameWriter writer : recipients) {
            Assert.assertEquals(writer.nextFrameCount(), 4);
            Assert.assertEquals(writer.closeCount(), 1);
        }
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}

Example 2 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class AbstractExternalDatasetIndexesOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            try {
                // perform operation on btrees
                for (int i = 0; i < treeIndexesDataflowHelperFactories.size(); i++) {
                    IIndexDataflowHelper indexHelper = treeIndexesDataflowHelperFactories.get(i).create(ctx, partition);
                    performOpOnIndex(indexHelper, ctx);
                }
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}

Example 3 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class ProfilingPartitionWriterFactory method createFrameWriter.

@Override
public IFrameWriter createFrameWriter(final int receiverIndex) throws HyracksDataException {
    final IFrameWriter writer = new ConnectorSenderProfilingFrameWriter(ctx, delegate.createFrameWriter(receiverIndex), cd.getConnectorId(), senderIndex, receiverIndex);
    return new IFrameWriter() {

        private long openTime;

        private long closeTime;

        MultiResolutionEventProfiler mrep = new MultiResolutionEventProfiler(N_SAMPLES);

        @Override
        public void open() throws HyracksDataException {
            openTime = System.currentTimeMillis();
            writer.open();
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            mrep.reportEvent();
            writer.nextFrame(buffer);
        }

        @Override
        public void fail() throws HyracksDataException {
            writer.fail();
        }

        @Override
        public void close() throws HyracksDataException {
            closeTime = System.currentTimeMillis();
            try {
                ((Task) ctx).setPartitionSendProfile(new PartitionProfile(new PartitionId(ctx.getJobletContext().getJobId(), cd.getConnectorId(), senderIndex, receiverIndex), openTime, closeTime, mrep));
            } finally {
                writer.close();
            }
        }

        @Override
        public void flush() throws HyracksDataException {
            writer.flush();
        }
    };
}

Example 4 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class IntersectOperatorDescriptorTest method executeAndVerifyResult.

private void executeAndVerifyResult(List<IFrame>[] inputFrames, List<Object[]> answer) throws Exception {
    IntersectOperatorDescriptor.IntersectOperatorNodePushable pushable = new IntersectOperatorDescriptor.IntersectOperatorNodePushable(ctx, nInputs, inputRecordDescriptor, compareFields, null, comparatorFactory);
    assertEquals(nInputs, pushable.getInputArity());
    IFrameWriter[] writers = new IFrameWriter[nInputs];
    for (int i = 0; i < nInputs; i++) {
        writers[i] = pushable.getInputFrameWriter(i);
    }
    IFrameWriter resultVerifier = new OutputFrameVerifier(outRecordDescriptor, answer);
    pushable.setOutputFrameWriter(0, resultVerifier, outRecordDescriptor);
    multiThreadTaskEmulator.runInParallel(writers, inputFrames);
}

Example 5 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class FramewriterTest method mockAppenders.

public static FrameTupleAppender[] mockAppenders() throws HyracksDataException {
    FrameTupleAppender[] appenders = new FrameTupleAppender[2];
    appenders[0] = Mockito.mock(FrameTupleAppender.class);
    Mockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            IFrameWriter writer = (IFrameWriter) args[0];
            writer.nextFrame(EMPTY_BUFFER);
            return null;
        }
    }).when(appenders[0]).write(Matchers.any(IFrameWriter.class), Matchers.anyBoolean());
    appenders[1] = Mockito.mock(FrameTupleAppender.class);
    Mockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            throw new HyracksDataException("couldn't flush frame");
        }
    }).when(appenders[1]).write(Matchers.any(IFrameWriter.class), Matchers.anyBoolean());
    return appenders;
}