Examples with IFrame org.apache.hyracks.api.comm.IFrame used on opensource projects

Example 1 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class CommitRuntime method nextFrame.

@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
    tAccess.reset(buffer);
    int nTuple = tAccess.getTupleCount();
    for (int t = 0; t < nTuple; t++) {
        if (isTemporaryDatasetWriteJob) {
            /**
                 * This "if branch" is for writes over temporary datasets. A temporary dataset does not require any lock
                 * and does not generate any write-ahead update and commit log but generates flush log and job commit
                 * log. However, a temporary dataset still MUST guarantee no-steal policy so that this notification call
                 * should be delivered to PrimaryIndexOptracker and used correctly in order to decrement number of
                 * active operation count of PrimaryIndexOptracker. By maintaining the count correctly and only allowing
                 * flushing when the count is 0, it can guarantee the no-steal policy for temporary datasets, too.
                 */
            // TODO: Fix this for upserts. an upsert tuple right now expect to notify the opTracker twice (one for
            // delete and one for insert)
            transactionContext.notifyOptracker(false);
        } else {
            tRef.reset(tAccess, t);
            try {
                formLogRecord(buffer, t);
                logMgr.log(logRecord);
                if (!isSink) {
                    appendTupleToFrame(t);
                }
            } catch (ACIDException e) {
                throw new HyracksDataException(e);
            }
        }
    }
    IFrame message = TaskUtil.get(HyracksConstants.KEY_MESSAGE, ctx);
    if (message != null && MessagingFrameTupleAppender.getMessageType(message) == MessagingFrameTupleAppender.MARKER_MESSAGE) {
        try {
            formMarkerLogRecords(message.getBuffer());
            logMgr.log(logRecord);
        } catch (ACIDException e) {
            throw new HyracksDataException(e);
        }
        message.reset();
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
    }
}

Example 2 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class TopKRunGeneratorTest method testInMemoryOnly.

private void testInMemoryOnly(IHyracksTaskContext ctx, int topK, ORDER order, AbstractSortRunGenerator sorter) throws HyracksDataException {
    Map<Integer, String> keyValuePair = null;
    switch(order) {
        case INORDER:
            keyValuePair = new TreeMap<>();
            break;
        case REVERSE:
            keyValuePair = new TreeMap<>(Collections.reverseOrder());
            break;
    }
    List<IFrame> frameList = new ArrayList<>();
    int minDataSize = PAGE_SIZE * NUM_PAGES * 4 / 5;
    int minRecordSize = 16;
    int maxRecordSize = 64;
    AbstractRunGeneratorTest.prepareData(ctx, frameList, minDataSize, minRecordSize, maxRecordSize, null, keyValuePair);
    assert topK > 0;
    ByteBuffer buffer = prepareSortedData(keyValuePair);
    Map<Integer, String> topKAnswer = getTopKAnswer(keyValuePair, topK);
    doSort(sorter, buffer);
    assertEquals(0, sorter.getRuns().size());
    validateResult(sorter, topKAnswer);
}

Example 3 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class InputFrameGenerator method generateDataFrame.

public List<IFrame> generateDataFrame(RecordDescriptor recordDescriptor, List<Object[]> listOfObject) throws HyracksDataException {
    List<IFrame> listFrame = new ArrayList<>();
    VSizeFrame frame = new VSizeFrame(manager);
    FrameFixedFieldAppender appender = new FrameFixedFieldAppender(recordDescriptor.getFieldCount());
    appender.reset(frame, true);
    ByteArrayOutputStream baos = new ByteArrayOutputStream(manager.getInitialFrameSize());
    DataOutputStream ds = new DataOutputStream(baos);
    for (Object[] objs : listOfObject) {
        for (int i = 0; i < recordDescriptor.getFieldCount(); i++) {
            baos.reset();
            recordDescriptor.getFields()[i].serialize(objs[i], ds);
            if (!appender.appendField(baos.toByteArray(), 0, baos.size())) {
                listFrame.add(frame);
                frame = new VSizeFrame(manager);
                appender.resetWithLeftOverData(frame);
                if (!appender.appendField(baos.toByteArray(), 0, baos.size())) {
                    throw new HyracksDataException("Should never happen!");
                }
            }
        }
    }
    listFrame.add(frame);
    return listFrame;
}

Example 4 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testOnlyOneRunShouldMerge.

@Test
public void testOnlyOneRunShouldMerge() throws HyracksDataException {
    int pageSize = 128;
    int numRuns = 1;
    int numFramesPerRun = 1;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 8;
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    List<Map<Integer, String>> keyValueMapList = new ArrayList<>(numRuns);
    List<TestFrameReader> readerList = new ArrayList<>(numRuns);
    List<IFrame> frameList = new ArrayList<>(numRuns);
    prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMapList);
    RunMergingFrameReader reader = new RunMergingFrameReader(ctx, readerList, frameList, SortFields, Comparators, null, RecordDesc);
    testMergeSucceed(ctx, reader, keyValueMapList);
}

Example 5 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testMergeSucceedInner.

private int testMergeSucceedInner(IHyracksTaskContext ctx, RunMergingFrameReader reader, List<Map<Integer, String>> keyValueMapList) throws HyracksDataException {
    IFrame frame = new VSizeFrame(ctx);
    reader.open();
    int count = 0;
    for (int i = 0; i < keyValueMapList.size(); i++) {
        keyValueMapList.set(i, new TreeMap<>(keyValueMapList.get(i)));
        count += keyValueMapList.get(i).size();
    }
    while (reader.nextFrame(frame)) {
        assertFrameIsSorted(frame, keyValueMapList);
    }
    return count;
}