Examples with AbstractUnaryInputSinkOperatorNodePushable org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable used on opensource projects

Example 6 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class HDFSWriteOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) throws HyracksDataException {
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private FSDataOutputStream dos;

        private RecordDescriptor inputRd = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);

        ;

        private FrameTupleAccessor accessor = new FrameTupleAccessor(inputRd);

        private FrameTupleReference tuple = new FrameTupleReference();

        private ITupleWriter tupleWriter;

        private ClassLoader ctxCL;

        @Override
        public void open() throws HyracksDataException {
            ctxCL = Thread.currentThread().getContextClassLoader();
            Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
            Job conf = confFactory.getConf();
            String outputPath = FileOutputFormat.getOutputPath(conf).toString();
            String fileName = outputPath + File.separator + "part-" + partition;
            tupleWriter = tupleWriterFactory.getTupleWriter(ctx, partition, nPartitions);
            try {
                FileSystem dfs = FileSystem.get(conf.getConfiguration());
                dos = dfs.create(new Path(fileName), true);
                tupleWriter.open(dos);
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            accessor.reset(buffer);
            int tupleCount = accessor.getTupleCount();
            for (int i = 0; i < tupleCount; i++) {
                tuple.reset(accessor, i);
                tupleWriter.write(dos, tuple);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                tupleWriter.close(dos);
                dos.close();
            } catch (Exception e) {
                throw new HyracksDataException(e);
            } finally {
                Thread.currentThread().setContextClassLoader(ctxCL);
            }
        }
    };
}