Examples with FrameTupleAppender org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender used on opensource projects

Example 21 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class IndexSearchOperatorNodePushable method open.

@Override
public void open() throws HyracksDataException {
    writer.open();
    indexHelper.open();
    index = indexHelper.getIndexInstance();
    accessor = new FrameTupleAccessor(inputRecDesc);
    if (retainMissing) {
        int fieldCount = getFieldCount();
        nonMatchTupleBuild = new ArrayTupleBuilder(fieldCount);
        buildMissingTuple(fieldCount, nonMatchTupleBuild, nonMatchWriter);
    } else {
        nonMatchTupleBuild = null;
    }
    if (appendIndexFilter) {
        int numIndexFilterFields = index.getNumOfFilterFields();
        nonFilterTupleBuild = new ArrayTupleBuilder(numIndexFilterFields);
        buildMissingTuple(numIndexFilterFields, nonFilterTupleBuild, nonMatchWriter);
    }
    try {
        searchPred = createSearchPredicate();
        tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
        dos = tb.getDataOutput();
        appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
        ISearchOperationCallback searchCallback = searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
        indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
        cursor = createCursor();
        if (retainInput) {
            frameTuple = new FrameTupleReference();
        }
    } catch (Exception e) {
        throw new HyracksDataException(e);
    }
}

Example 22 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class FramewriterTest method setUp.

@Before
public void setUp() throws Exception {
    // Mock static methods
    PowerMockito.mockStatic(BTreeUtils.class);
    PowerMockito.when(BTreeUtils.getSearchMultiComparator(Matchers.any(), Matchers.any())).thenReturn(mockMultiComparator());
    PowerMockito.mockStatic(FrameUtils.class);
    // Custom implementation for FrameUtils that push to next frame immediately
    PowerMockito.when(FrameUtils.appendToWriter(Matchers.any(IFrameWriter.class), Matchers.any(IFrameTupleAppender.class), Matchers.any(IFrameTupleAccessor.class), Matchers.anyInt(), Matchers.anyInt())).thenAnswer(new Answer<Integer>() {

        @Override
        public Integer answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            IFrameWriter writer = (IFrameWriter) args[0];
            writer.nextFrame(EMPTY_BUFFER);
            return BUFFER_SIZE;
        }
    });
    // create global mock for FrameTupleAccessor, ArrayTupleBuilder
    FrameTupleAccessor frameAccessor = Mockito.mock(FrameTupleAccessor.class);
    Mockito.when(frameAccessor.getTupleCount()).thenReturn(RECORDS_PER_FRAME);
    // Global custom implementations for FrameTupleAppender
    // since we have two appenders, then we need to test each test twice
    FrameTupleAppender[] appenders = mockAppenders();
    // Mock all instances of a class <Note that you need to prepare the class calling this constructor as well>
    PowerMockito.whenNew(FrameTupleAccessor.class).withAnyArguments().thenReturn(frameAccessor);
    PowerMockito.whenNew(FrameTupleAppender.class).withAnyArguments().thenAnswer(new Answer<FrameTupleAppender>() {

        @Override
        public FrameTupleAppender answer(InvocationOnMock invocation) throws Throwable {
            counter++;
            if (counter % 2 == 1) {
                return appenders[0];
            }
            return appenders[1];
        }
    });
}

Example 23 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class AbstractTupleMemoryManagerTest method prepareFixedSizeTuples.

protected Map<Integer, Integer> prepareFixedSizeTuples(int tuplePerFrame, int extraMetaBytePerFrame, int extraMetaBytePerRecord) throws HyracksDataException {
    Map<Integer, Integer> dataSet = new HashMap<>();
    ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
    FixedSizeFrame frame = new FixedSizeFrame(buffer);
    FrameTupleAppender appender = new FrameTupleAppender();
    appender.reset(frame, true);
    int sizePerTuple = (Common.MIN_FRAME_SIZE - 1 - tuplePerFrame * 4 - 4 - extraMetaBytePerFrame) / tuplePerFrame;
    int sizeChar = //2byte to write str length
    sizePerTuple - extraMetaBytePerRecord - fieldsSerDer.length * 4 - 4 - 2;
    assert (sizeChar > 0);
    for (int i = 0; i < Common.NUM_MIN_FRAME * tuplePerFrame; i++) {
        tupleBuilder.reset();
        tupleBuilder.addField(fieldsSerDer[0], i);
        tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', sizeChar));
        if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
            assert false;
        }
        dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
    }
    inFTA.reset(buffer);
    return dataSet;
}

Example 24 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class LSMPrimaryUpsertOperatorNodePushable method open.

// we have the permutation which has [pk locations, record location, optional:filter-location]
// the index -> we don't need anymore data?
// we need to use the primary index opTracker and secondary indexes callbacks for insert/delete since the lock would
// have been obtained through searchForUpsert operation
@Override
public void open() throws HyracksDataException {
    accessor = new FrameTupleAccessor(inputRecDesc);
    writeBuffer = new VSizeFrame(ctx);
    writer.open();
    indexHelper.open();
    index = indexHelper.getIndexInstance();
    try {
        if (ctx.getSharedObject() != null) {
            PrimaryIndexLogMarkerCallback callback = new PrimaryIndexLogMarkerCallback((AbstractLSMIndex) index);
            TaskUtil.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
        }
        missingTupleBuilder = new ArrayTupleBuilder(1);
        DataOutput out = missingTupleBuilder.getDataOutput();
        try {
            missingWriter.writeMissing(out);
        } catch (IOException e) {
            throw new HyracksDataException(e);
        }
        missingTupleBuilder.addFieldEndOffset();
        searchPred = createSearchPredicate();
        tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
        dos = tb.getDataOutput();
        appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
        modCallback = modOpCallbackFactory.createModificationOperationCallback(indexHelper.getResource(), ctx, this);
        abstractModCallback = (AbstractIndexModificationOperationCallback) modCallback;
        searchCallback = (LockThenSearchOperationCallback) searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, this);
        indexAccessor = index.createAccessor(abstractModCallback, searchCallback);
        lsmAccessor = (LSMTreeIndexAccessor) indexAccessor;
        cursor = indexAccessor.createSearchCursor(false);
        frameTuple = new FrameTupleReference();
        INcApplicationContext appCtx = (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
        LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) index, appCtx.getTransactionSubsystem().getLogManager());
        frameOpCallback = new IFrameOperationCallback() {

            IFrameOperationCallback callback = frameOpCallbackFactory.createFrameOperationCallback(ctx, (ILSMIndexAccessor) indexAccessor);

            @Override
            public void frameCompleted() throws HyracksDataException {
                callback.frameCompleted();
                appender.write(writer, true);
            }
        };
    } catch (Exception e) {
        indexHelper.close();
        throw new HyracksDataException(e);
    }
}

Example 25 with FrameTupleAppender

use of org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender in project asterixdb by apache.

the class EmptyTupleSourceRuntimeFactory method createPushRuntime.

@Override
public IPushRuntime createPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    return new AbstractOneInputSourcePushRuntime() {

        private final ArrayTupleBuilder tb = new ArrayTupleBuilder(0);

        private final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));

        @Override
        public void open() throws HyracksDataException {
            writer.open();
            if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                throw new IllegalStateException();
            }
            appender.write(writer, true);
        }

        @Override
        public void close() throws HyracksDataException {
            writer.close();
        }

        @Override
        public void flush() throws HyracksDataException {
            appender.flush(writer);
        }
    };
}