Examples with AbstractUnaryInputSinkOperatorNodePushable org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable used on opensource projects

Example 1 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class HDFSWriteOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) throws HyracksDataException {
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private FSDataOutputStream dos;

        private RecordDescriptor inputRd = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);

        ;

        private FrameTupleAccessor accessor = new FrameTupleAccessor(inputRd);

        private FrameTupleReference tuple = new FrameTupleReference();

        private ITupleWriter tupleWriter;

        private ClassLoader ctxCL;

        @Override
        public void open() throws HyracksDataException {
            ctxCL = Thread.currentThread().getContextClassLoader();
            Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
            JobConf conf = confFactory.getConf();
            String outputDirPath = FileOutputFormat.getOutputPath(conf).toString();
            String fileName = outputDirPath + File.separator + "part-" + partition;
            tupleWriter = tupleWriterFactory.getTupleWriter(ctx, partition, nPartitions);
            try {
                FileSystem dfs = FileSystem.get(conf);
                dos = dfs.create(new Path(fileName), true);
                tupleWriter.open(dos);
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            accessor.reset(buffer);
            int tupleCount = accessor.getTupleCount();
            for (int i = 0; i < tupleCount; i++) {
                tuple.reset(accessor, i);
                tupleWriter.write(dos, tuple);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                tupleWriter.close(dos);
                dos.close();
            } catch (Exception e) {
                throw new HyracksDataException(e);
            } finally {
                Thread.currentThread().setContextClassLoader(ctxCL);
            }
        }
    };
}

Example 2 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class ResultWriterOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) throws HyracksDataException {
    final IDatasetPartitionManager dpm = ctx.getDatasetPartitionManager();
    final IFrame frame = new VSizeFrame(ctx);
    final FrameOutputStream frameOutputStream = new FrameOutputStream(ctx.getInitialFrameSize());
    frameOutputStream.reset(frame, true);
    PrintStream printStream = new PrintStream(frameOutputStream);
    final RecordDescriptor outRecordDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
    final IResultSerializer resultSerializer = resultSerializerFactory.createResultSerializer(outRecordDesc, printStream);
    final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(outRecordDesc);
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private IFrameWriter datasetPartitionWriter;

        private boolean failed = false;

        @Override
        public void open() throws HyracksDataException {
            try {
                datasetPartitionWriter = dpm.createDatasetPartitionWriter(ctx, rsId, ordered, asyncMode, partition, nPartitions);
                datasetPartitionWriter.open();
                resultSerializer.init();
            } catch (HyracksException e) {
                throw HyracksDataException.create(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            frameTupleAccessor.reset(buffer);
            for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                resultSerializer.appendTuple(frameTupleAccessor, tIndex);
                if (!frameOutputStream.appendTuple()) {
                    frameOutputStream.flush(datasetPartitionWriter);
                    resultSerializer.appendTuple(frameTupleAccessor, tIndex);
                    frameOutputStream.appendTuple();
                }
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            failed = true;
            datasetPartitionWriter.fail();
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                if (!failed && frameOutputStream.getTupleCount() > 0) {
                    frameOutputStream.flush(datasetPartitionWriter);
                }
            } catch (Exception e) {
                datasetPartitionWriter.fail();
                throw e;
            } finally {
                datasetPartitionWriter.close();
            }
        }

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder();
            sb.append("{ ");
            sb.append("\"rsId\": \"").append(rsId).append("\", ");
            sb.append("\"ordered\": ").append(ordered).append(", ");
            sb.append("\"asyncMode\": ").append(asyncMode).append(" }");
            return sb.toString();
        }
    };
}

Example 3 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class FrameFileWriterOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
    final FileSplit[] splits = fileSplitProvider.getFileSplits();
    final IIOManager ioManager = ctx.getIoManager();
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private OutputStream out;

        @Override
        public void open() throws HyracksDataException {
            try {
                out = new FileOutputStream(splits[partition].getFile(ioManager));
            } catch (FileNotFoundException e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            try {
                out.write(buffer.array());
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                out.close();
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 4 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class FlushDatasetOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        @Override
        public void open() throws HyracksDataException {
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
        }

        @Override
        public void fail() throws HyracksDataException {
            this.close();
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                INcApplicationContext appCtx = (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
                IDatasetLifecycleManager datasetLifeCycleManager = appCtx.getDatasetLifecycleManager();
                ILockManager lockManager = appCtx.getTransactionSubsystem().getLockManager();
                ITransactionManager txnManager = appCtx.getTransactionSubsystem().getTransactionManager();
                // get the local transaction
                ITransactionContext txnCtx = txnManager.getTransactionContext(jobId, false);
                // lock the dataset granule
                lockManager.lock(datasetId, -1, LockMode.S, txnCtx);
                // flush the dataset synchronously
                datasetLifeCycleManager.flushDataset(datasetId.getId(), false);
            } catch (ACIDException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 5 with AbstractUnaryInputSinkOperatorNodePushable

use of org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable in project asterixdb by apache.

the class PlainFileWriterOperatorDescriptor method createPushRuntime.

/*
     * (non-Javadoc)
     *
     * @see
     * org.apache.hyracks.api.dataflow.IActivityNode#createPushRuntime(edu.
     * uci.ics.hyracks.api.context.IHyracksContext,
     * org.apache.hyracks.api.job.IOperatorEnvironment,
     * org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider, int,
     * int)
     */
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) throws HyracksDataException {
    // Output files
    final FileSplit[] splits = fileSplitProvider.getFileSplits();
    IIOManager ioManager = ctx.getIoManager();
    // Frame accessor
    final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
    // Record descriptor
    final RecordDescriptor recordDescriptor = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private BufferedWriter out;

        private ByteBufferInputStream bbis;

        private DataInputStream di;

        @Override
        public void open() throws HyracksDataException {
            try {
                out = new BufferedWriter(new FileWriter(splits[partition].getFile(ioManager)));
                bbis = new ByteBufferInputStream();
                di = new DataInputStream(bbis);
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            try {
                frameTupleAccessor.reset(buffer);
                for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                    int start = frameTupleAccessor.getTupleStartOffset(tIndex) + frameTupleAccessor.getFieldSlotsLength();
                    bbis.setByteBuffer(buffer, start);
                    Object[] record = new Object[recordDescriptor.getFieldCount()];
                    for (int i = 0; i < record.length; ++i) {
                        Object instance = recordDescriptor.getFields()[i].deserialize(di);
                        if (i == 0) {
                            out.write(String.valueOf(instance));
                        } else {
                            out.write(delim + String.valueOf(instance));
                        }
                    }
                    out.write("\n");
                }
            } catch (IOException ex) {
                throw new HyracksDataException(ex);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                out.close();
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}