Examples with VSizeFrame org.apache.hyracks.api.comm.VSizeFrame used on opensource projects

Example 21 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class LimitOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {

        private FrameTupleAccessor fta;

        private int currentSize;

        private boolean finished;

        @Override
        public void open() throws HyracksDataException {
            fta = new FrameTupleAccessor(outRecDescs[0]);
            currentSize = 0;
            finished = false;
            writer.open();
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            if (!finished) {
                fta.reset(buffer);
                int count = fta.getTupleCount();
                if ((currentSize + count) > outputLimit) {
                    FrameTupleAppender partialAppender = new FrameTupleAppender(new VSizeFrame(ctx));
                    int copyCount = outputLimit - currentSize;
                    for (int i = 0; i < copyCount; i++) {
                        FrameUtils.appendToWriter(writer, partialAppender, fta, i);
                        currentSize++;
                    }
                    partialAppender.write(writer, false);
                    finished = true;
                } else {
                    FrameUtils.flushFrame(buffer, writer);
                    currentSize += count;
                }
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            writer.fail();
        }

        @Override
        public void close() throws HyracksDataException {
            writer.close();
        }

        @Override
        public void flush() throws HyracksDataException {
            writer.flush();
        }
    };
}

Example 22 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class PartitionDataWriter method allocateFrames.

protected void allocateFrames(int i) throws HyracksDataException {
    appenders[i].reset(new VSizeFrame(ctx), true);
    allocatedFrames[i] = true;
}

Example 23 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class DelimitedDataTupleParserFactory method createTupleParser.

@Override
public ITupleParser createTupleParser(final IHyracksTaskContext ctx) {
    return new ITupleParser() {

        @Override
        public void parse(InputStream in, IFrameWriter writer) throws HyracksDataException {
            try {
                IValueParser[] valueParsers = new IValueParser[valueParserFactories.length];
                for (int i = 0; i < valueParserFactories.length; ++i) {
                    valueParsers[i] = valueParserFactories[i].createValueParser();
                }
                IFrame frame = new VSizeFrame(ctx);
                FrameTupleAppender appender = new FrameTupleAppender();
                appender.reset(frame, true);
                ArrayTupleBuilder tb = new ArrayTupleBuilder(valueParsers.length);
                DataOutput dos = tb.getDataOutput();
                FieldCursorForDelimitedDataParser cursor = new FieldCursorForDelimitedDataParser(new InputStreamReader(in), fieldDelimiter, quote);
                while (cursor.nextRecord()) {
                    tb.reset();
                    for (int i = 0; i < valueParsers.length; ++i) {
                        if (!cursor.nextField()) {
                            break;
                        }
                        // Eliminate double quotes in the field that we are going to parse
                        if (cursor.isDoubleQuoteIncludedInThisField) {
                            cursor.eliminateDoubleQuote(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart);
                            cursor.fEnd -= cursor.doubleQuoteCount;
                            cursor.isDoubleQuoteIncludedInThisField = false;
                        }
                        valueParsers[i].parse(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart, dos);
                        tb.addFieldEndOffset();
                    }
                    FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
                }
                appender.write(writer, true);
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 24 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class ExternalGroupWriteOperatorNodePushable method buildGroup.

private int[] buildGroup(RunFileReader reader, ISpillableTable table, RunFileWriter[] runFileWriters) throws HyracksDataException {
    ExternalHashGroupBy groupBy = new ExternalHashGroupBy(this, table, runFileWriters, partialAggRecordDesc);
    reader.open();
    try {
        VSizeFrame frame = new VSizeFrame(ctx);
        while (reader.nextFrame(frame)) {
            groupBy.insert(frame.getBuffer());
        }
        groupBy.flushSpilledPartitions();
    } finally {
        reader.close();
    }
    return groupBy.getSpilledNumTuples();
}

Example 25 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class TreeIndexDiskOrderScanOperatorNodePushable method initialize.

@Override
public void initialize() throws HyracksDataException {
    treeIndexHelper.open();
    ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndexInstance();
    try {
        ITreeIndexFrame cursorFrame = treeIndex.getLeafFrameFactory().createFrame();
        ITreeIndexCursor cursor = new TreeIndexDiskOrderScanCursor(cursorFrame);
        LocalResource resource = treeIndexHelper.getResource();
        ISearchOperationCallback searchCallback = searchCallbackFactory.createSearchOperationCallback(resource.getId(), ctx, null);
        ITreeIndexAccessor indexAccessor = (ITreeIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
        try {
            writer.open();
            indexAccessor.diskOrderScan(cursor);
            int fieldCount = treeIndex.getFieldCount();
            FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
            ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
            DataOutput dos = tb.getDataOutput();
            while (cursor.hasNext()) {
                tb.reset();
                cursor.next();
                ITupleReference frameTuple = cursor.getTuple();
                for (int i = 0; i < frameTuple.getFieldCount(); i++) {
                    dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
                    tb.addFieldEndOffset();
                }
                FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
            }
            appender.write(writer, true);
        } catch (Throwable th) {
            writer.fail();
            throw new HyracksDataException(th);
        } finally {
            try {
                cursor.close();
            } catch (Exception cursorCloseException) {
                throw new IllegalStateException(cursorCloseException);
            } finally {
                writer.close();
            }
        }
    } catch (Throwable th) {
        treeIndexHelper.close();
        throw new HyracksDataException(th);
    }
}