Examples with IFrameWriter org.apache.hyracks.api.comm.IFrameWriter used on opensource projects

Example 26 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class WordTupleParserFactory method createTupleParser.

@Override
public ITupleParser createTupleParser(final IHyracksTaskContext ctx) {
    return new ITupleParser() {

        @Override
        public void parse(InputStream in, IFrameWriter writer) throws HyracksDataException {
            try {
                FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
                ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
                DataOutput dos = tb.getDataOutput();
                IValueParser utf8StringParser = UTF8StringParserFactory.INSTANCE.createValueParser();
                WordCursor cursor = new WordCursor(new InputStreamReader(in));
                while (cursor.nextWord()) {
                    tb.reset();
                    utf8StringParser.parse(cursor.buffer, cursor.fStart, cursor.fEnd - cursor.fStart, dos);
                    tb.addFieldEndOffset();
                    FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
                }
                appender.write(writer, true);
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 27 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class TextKeyValueParserFactory method createKeyValueParser.

@Override
public IKeyValueParser<LongWritable, Text> createKeyValueParser(final IHyracksTaskContext ctx) throws HyracksDataException {
    final ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
    final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
    return new IKeyValueParser<LongWritable, Text>() {

        @Override
        public void open(IFrameWriter writer) {
        }

        @Override
        public void parse(LongWritable key, Text value, IFrameWriter writer, String fileString) throws HyracksDataException {
            tb.reset();
            tb.addField(value.getBytes(), 0, value.getLength());
            FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
        }

        @Override
        public void close(IFrameWriter writer) throws HyracksDataException {
            appender.write(writer, false);
        }
    };
}

Example 28 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class ExternalFilesIndexModificationOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            final IIndexDataflowHelper indexHelper = dataflowHelperFactory.create(ctx, partition);
            FileIndexTupleTranslator filesTupleTranslator = new FileIndexTupleTranslator();
            // Open and get
            indexHelper.open();
            IIndex index = indexHelper.getIndexInstance();
            LSMTwoPCBTreeBulkLoader bulkLoader = null;
            try {
                bulkLoader = (LSMTwoPCBTreeBulkLoader) ((ExternalBTree) index).createTransactionBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size());
                // The files must be ordered according to their numbers
                for (ExternalFile file : files) {
                    switch(file.getPendingOp()) {
                        case ADD_OP:
                        case APPEND_OP:
                            bulkLoader.add(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case DROP_OP:
                            bulkLoader.delete(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case NO_OP:
                            break;
                        default:
                            throw HyracksDataException.create(ErrorCode.UNKNOWN_EXTERNAL_FILE_PENDING_OP, file.getPendingOp());
                    }
                }
                bulkLoader.end();
            } catch (Exception e) {
                if (bulkLoader != null) {
                    bulkLoader.abort();
                }
                throw HyracksDataException.create(e);
            } finally {
                indexHelper.close();
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}

Example 29 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class ExternalFilesIndexCreateOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition);
            IIndexDataflowHelper indexHelper = dataflowHelperFactory.create(ctx, partition);
            FileIndexTupleTranslator filesTupleTranslator = new FileIndexTupleTranslator();
            // Build the index
            indexBuilder.build();
            // Open the index
            indexHelper.open();
            try {
                IIndex index = indexHelper.getIndexInstance();
                // Create bulk loader
                IIndexBulkLoader bulkLoader = index.createBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size(), false);
                // Load files
                for (ExternalFile file : files) {
                    bulkLoader.add(filesTupleTranslator.getTupleFromFile(file));
                }
                bulkLoader.end();
            } finally {
                indexHelper.close();
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}

Example 30 with IFrameWriter

use of org.apache.hyracks.api.comm.IFrameWriter in project asterixdb by apache.

the class FrameFixedFieldTupleAppenderTest method testProcess.

private void testProcess(IFrameTupleAccessor accessor) throws HyracksDataException {
    IFrameWriter writer = prepareValidator(accessor);
    writer.open();
    for (int tid = 0; tid < accessor.getTupleCount(); tid++) {
        for (int fid = 0; fid < fields.length; fid++) {
            if (!appender.appendField(accessor, tid, fid)) {
                appender.write(writer, true);
                if (!appender.appendField(accessor, tid, fid)) {
                }
            }
        }
    }
    appender.write(writer, true);
    writer.close();
}