Examples with IFrame org.apache.hyracks.api.comm.IFrame used on opensource projects

Example 11 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class ResultWriterOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) throws HyracksDataException {
    final IDatasetPartitionManager dpm = ctx.getDatasetPartitionManager();
    final IFrame frame = new VSizeFrame(ctx);
    final FrameOutputStream frameOutputStream = new FrameOutputStream(ctx.getInitialFrameSize());
    frameOutputStream.reset(frame, true);
    PrintStream printStream = new PrintStream(frameOutputStream);
    final RecordDescriptor outRecordDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
    final IResultSerializer resultSerializer = resultSerializerFactory.createResultSerializer(outRecordDesc, printStream);
    final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(outRecordDesc);
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private IFrameWriter datasetPartitionWriter;

        private boolean failed = false;

        @Override
        public void open() throws HyracksDataException {
            try {
                datasetPartitionWriter = dpm.createDatasetPartitionWriter(ctx, rsId, ordered, asyncMode, partition, nPartitions);
                datasetPartitionWriter.open();
                resultSerializer.init();
            } catch (HyracksException e) {
                throw HyracksDataException.create(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            frameTupleAccessor.reset(buffer);
            for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                resultSerializer.appendTuple(frameTupleAccessor, tIndex);
                if (!frameOutputStream.appendTuple()) {
                    frameOutputStream.flush(datasetPartitionWriter);
                    resultSerializer.appendTuple(frameTupleAccessor, tIndex);
                    frameOutputStream.appendTuple();
                }
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            failed = true;
            datasetPartitionWriter.fail();
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                if (!failed && frameOutputStream.getTupleCount() > 0) {
                    frameOutputStream.flush(datasetPartitionWriter);
                }
            } catch (Exception e) {
                datasetPartitionWriter.fail();
                throw e;
            } finally {
                datasetPartitionWriter.close();
            }
        }

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder();
            sb.append("{ ");
            sb.append("\"rsId\": \"").append(rsId).append("\", ");
            sb.append("\"ordered\": ").append(ordered).append(", ");
            sb.append("\"asyncMode\": ").append(asyncMode).append(" }");
            return sb.toString();
        }
    };
}

Example 12 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class DelimitedDataTupleParserFactory method createTupleParser.

@Override
public ITupleParser createTupleParser(final IHyracksTaskContext ctx) {
    return new ITupleParser() {

        @Override
        public void parse(InputStream in, IFrameWriter writer) throws HyracksDataException {
            try {
                IValueParser[] valueParsers = new IValueParser[valueParserFactories.length];
                for (int i = 0; i < valueParserFactories.length; ++i) {
                    valueParsers[i] = valueParserFactories[i].createValueParser();
                }
                IFrame frame = new VSizeFrame(ctx);
                FrameTupleAppender appender = new FrameTupleAppender();
                appender.reset(frame, true);
                ArrayTupleBuilder tb = new ArrayTupleBuilder(valueParsers.length);
                DataOutput dos = tb.getDataOutput();
                FieldCursorForDelimitedDataParser cursor = new FieldCursorForDelimitedDataParser(new InputStreamReader(in), fieldDelimiter, quote);
                while (cursor.nextRecord()) {
                    tb.reset();
                    for (int i = 0; i < valueParsers.length; ++i) {
                        if (!cursor.nextField()) {
                            break;
                        }
                        // Eliminate double quotes in the field that we are going to parse
                        if (cursor.isDoubleQuoteIncludedInThisField) {
                            cursor.eliminateDoubleQuote(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart);
                            cursor.fEnd -= cursor.doubleQuoteCount;
                            cursor.isDoubleQuoteIncludedInThisField = false;
                        }
                        valueParsers[i].parse(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart, dos);
                        tb.addFieldEndOffset();
                    }
                    FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
                }
                appender.write(writer, true);
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 13 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class ResultPrinter method print.

public void print(ResultReader resultReader) throws HyracksDataException {
    printPrefix();
    final IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
    final IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
    while (resultReader.read(frame) > 0) {
        final ByteBuffer frameBuffer = frame.getBuffer();
        final byte[] frameBytes = frameBuffer.array();
        fta.reset(frameBuffer);
        final int last = fta.getTupleCount();
        for (int tIndex = 0; tIndex < last; tIndex++) {
            final int start = fta.getTupleStartOffset(tIndex);
            int length = fta.getTupleEndOffset(tIndex) - start;
            if (conf.fmt() == SessionConfig.OutputFormat.CSV && ((length > 0) && (frameBytes[start + length - 1] == '\n'))) {
                length--;
            }
            String result = new String(frameBytes, start, length, UTF_8);
            if (wrapArray && notFirst) {
                output.out().print(", ");
            }
            notFirst = true;
            displayRecord(result);
        }
        frameBuffer.clear();
    }
    printPostfix();
}

Example 14 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method testNormalIncomingFrames.

@Test
public void testNormalIncomingFrames() throws HyracksDataException {
    HashMap<Integer, Integer> tupleSet = prepareTuples();
    for (IFrame frame : frameList) {
        assertTrue(framesMemoryManager.insertFrame(frame.getBuffer()) >= 0);
    }
    assertEquals(NUM_MIN_FRAME, framesMemoryManager.getNumFrames());
    assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
}

Example 15 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method testRandomTuplesAreAllStoredInBuffer.

@Test
public void testRandomTuplesAreAllStoredInBuffer() throws HyracksDataException {
    Map<Integer, Integer> tupleSet = prepareRandomTuples();
    for (IFrame frame : frameList) {
        if (framesMemoryManager.insertFrame(frame.getBuffer()) < 0) {
            fta.reset(frame.getBuffer());
            for (int i = 0; i < fta.getTupleCount(); ++i) {
                int id = parseTuple(fta.getBuffer(), fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
                tupleSet.remove(id);
            //                    System.out.println(
            //                            "can't appended id:" + id + ",frameSize:" + frame.getInitialFrameSize());
            }
        }
    }
    assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
    framesMemoryManager.reset();
}