Examples with AbstractOneInputOneOutputOneFramePushRuntime org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime used on opensource projects

Example 1 with AbstractOneInputOneOutputOneFramePushRuntime

use of org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime in project asterixdb by apache.

the class StreamLimitRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
    final IBinaryIntegerInspector bii = binaryIntegerInspectorFactory.createBinaryIntegerInspector(ctx);
    return new AbstractOneInputOneOutputOneFramePushRuntime() {

        private final IPointable p = VoidPointable.FACTORY.createPointable();

        private IScalarEvaluator evalMaxObjects;

        private IScalarEvaluator evalOffset = null;

        // how many tuples still to write
        private int toWrite = 0;

        // how many tuples still to skip
        private int toSkip = 0;

        private boolean firstTuple = true;

        private boolean afterLastTuple = false;

        @Override
        public void open() throws HyracksDataException {
            writer.open();
            if (evalMaxObjects == null) {
                initAccessAppendRef(ctx);
                evalMaxObjects = maxObjectsEvalFactory.createScalarEvaluator(ctx);
                if (offsetEvalFactory != null) {
                    evalOffset = offsetEvalFactory.createScalarEvaluator(ctx);
                }
            }
            afterLastTuple = false;
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            if (afterLastTuple) {
                return;
            }
            tAccess.reset(buffer);
            int nTuple = tAccess.getTupleCount();
            int start = 0;
            if (nTuple <= toSkip) {
                toSkip -= nTuple;
                return;
            } else if (toSkip > 0) {
                start = toSkip;
                toSkip = 0;
            }
            for (int t = start; t < nTuple; t++) {
                if (firstTuple) {
                    firstTuple = false;
                    toWrite = evaluateInteger(evalMaxObjects, t);
                    if (evalOffset != null) {
                        toSkip = evaluateInteger(evalOffset, t);
                    }
                }
                if (toSkip > 0) {
                    toSkip--;
                } else if (toWrite > 0) {
                    toWrite--;
                    if (projectionList != null) {
                        appendProjectionToFrame(t, projectionList);
                    } else {
                        appendTupleToFrame(t);
                    }
                } else {
                    afterLastTuple = true;
                    break;
                }
            }
        }

        @Override
        public void close() throws HyracksDataException {
            // how many tuples still to write
            toWrite = 0;
            // how many tuples still to skip
            toSkip = 0;
            firstTuple = true;
            afterLastTuple = false;
            super.close();
        }

        private int evaluateInteger(IScalarEvaluator eval, int tIdx) throws HyracksDataException {
            tRef.reset(tAccess, tIdx);
            eval.evaluate(tRef, p);
            int lim = bii.getIntegerValue(p.getByteArray(), p.getStartOffset(), p.getLength());
            return lim;
        }

        @Override
        public void flush() throws HyracksDataException {
            appender.flush(writer);
        }
    };
}

Example 2 with AbstractOneInputOneOutputOneFramePushRuntime

use of org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime in project asterixdb by apache.

the class AggregateRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    return new AbstractOneInputOneOutputOneFramePushRuntime() {

        private IAggregateEvaluator[] aggregs = new IAggregateEvaluator[aggregFactories.length];

        private IPointable result = VoidPointable.FACTORY.createPointable();

        private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(aggregs.length);

        private boolean first = true;

        private boolean isOpen = false;

        @Override
        public void open() throws HyracksDataException {
            if (first) {
                first = false;
                initAccessAppendRef(ctx);
                for (int i = 0; i < aggregFactories.length; i++) {
                    aggregs[i] = aggregFactories[i].createAggregateEvaluator(ctx);
                }
            }
            for (int i = 0; i < aggregFactories.length; i++) {
                aggregs[i].init();
            }
            isOpen = true;
            writer.open();
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            tAccess.reset(buffer);
            int nTuple = tAccess.getTupleCount();
            for (int t = 0; t < nTuple; t++) {
                tRef.reset(tAccess, t);
                processTuple(tRef);
            }
        }

        @Override
        public void close() throws HyracksDataException {
            if (isOpen) {
                try {
                    computeAggregate();
                    appendToFrameFromTupleBuilder(tupleBuilder);
                } finally {
                    super.close();
                }
            }
        }

        private void computeAggregate() throws HyracksDataException {
            tupleBuilder.reset();
            for (int f = 0; f < aggregs.length; f++) {
                aggregs[f].finish(result);
                tupleBuilder.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
            }
        }

        private void processTuple(FrameTupleReference tupleRef) throws HyracksDataException {
            for (int f = 0; f < aggregs.length; f++) {
                aggregs[f].step(tupleRef);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            if (isOpen) {
                writer.fail();
            }
        }
    };
}

Example 3 with AbstractOneInputOneOutputOneFramePushRuntime

use of org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime in project asterixdb by apache.

the class StringStreamingRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    final IPrinter[] printers = new IPrinter[printerFactories.length];
    for (int i = 0; i < printerFactories.length; i++) {
        printers[i] = printerFactories[i].createPrinter();
    }
    return new AbstractOneInputOneOutputOneFramePushRuntime() {

        final class ForwardScriptOutput implements Runnable {

            private InputStream inStream;

            private ITupleParser parser;

            public ForwardScriptOutput(ITupleParser parser, InputStream inStream) {
                this.parser = parser;
                this.inStream = inStream;
            }

            @Override
            public void run() {
                try {
                    parser.parse(inStream, writer);
                } catch (HyracksDataException e) {
                    throw new RuntimeException(e);
                } finally {
                    try {
                        inStream.close();
                    } catch (Exception e) {
                    }
                }
            }
        }

        final class DumpInStreamToPrintStream implements Runnable {

            private BufferedReader reader;

            private PrintStream printStream;

            public DumpInStreamToPrintStream(InputStream inStream, PrintStream printStream) {
                this.reader = new BufferedReader(new InputStreamReader(inStream));
                this.printStream = printStream;
            }

            @Override
            public void run() {
                String s;
                try {
                    while ((s = reader.readLine()) != null) {
                        printStream.println(s);
                    }
                } catch (IOException e) {
                    throw new RuntimeException(e);
                } finally {
                    try {
                        reader.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    printStream.close();
                }
            }
        }

        private Process process;

        private PrintStream ps;

        private boolean first = true;

        private Thread outputPipe;

        private Thread dumpStderr;

        @Override
        public void open() throws HyracksDataException {
            if (first) {
                first = false;
                initAccessAppendRef(ctx);
            }
            try {
                ITupleParser parser = parserFactory.createTupleParser(ctx);
                process = Runtime.getRuntime().exec(command);
                ps = new PrintStream(process.getOutputStream());
                ForwardScriptOutput fso = new ForwardScriptOutput(parser, process.getInputStream());
                outputPipe = new Thread(fso);
                outputPipe.start();
                DumpInStreamToPrintStream disps = new DumpInStreamToPrintStream(process.getErrorStream(), System.err);
                dumpStderr = new Thread(disps);
                dumpStderr.start();
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            tAccess.reset(buffer);
            int nTuple = tAccess.getTupleCount();
            for (int t = 0; t < nTuple; t++) {
                tRef.reset(tAccess, t);
                for (int i = 0; i < printers.length; i++) {
                    printers[i].print(buffer.array(), tRef.getFieldStart(i), tRef.getFieldLength(i), ps);
                    ps.print(fieldDelimiter);
                    if (i == printers.length - 1) {
                        ps.print('\n');
                    }
                }
            }
        }

        @Override
        public void close() throws HyracksDataException {
            // first close the printer printing to the process
            ps.close();
            int ret = 0;
            try {
                ret = process.waitFor();
                outputPipe.join();
                dumpStderr.join();
            } catch (InterruptedException e) {
                throw new HyracksDataException(e);
            }
            if (ret != 0) {
                throw new HyracksDataException("Process exit value: " + ret);
            }
            // close the following operator in the chain
            super.close();
        }

        @Override
        public void flush() throws HyracksDataException {
            ps.flush();
        }
    };
}

Example 4 with AbstractOneInputOneOutputOneFramePushRuntime

use of org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime in project asterixdb by apache.

the class AssignRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    final int[] projectionToOutColumns = new int[projectionList.length];
    for (int j = 0; j < projectionList.length; j++) {
        projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
    }
    return new AbstractOneInputOneOutputOneFramePushRuntime() {

        private IPointable result = VoidPointable.FACTORY.createPointable();

        private IScalarEvaluator[] eval = new IScalarEvaluator[evalFactories.length];

        private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);

        private boolean first = true;

        private boolean isOpen = false;

        private int tupleIndex = 0;

        @Override
        public void open() throws HyracksDataException {
            if (first) {
                initAccessAppendRef(ctx);
                first = false;
                int n = evalFactories.length;
                for (int i = 0; i < n; i++) {
                    eval[i] = evalFactories[i].createScalarEvaluator(ctx);
                }
            }
            isOpen = true;
            writer.open();
        }

        @Override
        public void close() throws HyracksDataException {
            if (isOpen) {
                super.close();
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            // what if nTuple is 0?
            tAccess.reset(buffer);
            int nTuple = tAccess.getTupleCount();
            if (nTuple < 1) {
                if (nTuple < 0) {
                    throw new HyracksDataException("Negative number of tuples in the frame: " + nTuple);
                }
                appender.flush(writer);
            } else {
                if (nTuple > 1) {
                    for (; tupleIndex < nTuple - 1; tupleIndex++) {
                        tRef.reset(tAccess, tupleIndex);
                        produceTuple(tupleBuilder, tAccess, tupleIndex, tRef);
                        appendToFrameFromTupleBuilder(tupleBuilder);
                    }
                }
                if (tupleIndex < nTuple) {
                    tRef.reset(tAccess, tupleIndex);
                    produceTuple(tupleBuilder, tAccess, tupleIndex, tRef);
                    if (flushFramesRapidly) {
                        // Whenever all the tuples in the incoming frame have been consumed, the assign operator
                        // will push its frame to the next operator; i.e., it won't wait until the frame gets full.
                        appendToFrameFromTupleBuilder(tupleBuilder, true);
                    } else {
                        appendToFrameFromTupleBuilder(tupleBuilder);
                    }
                } else {
                    if (flushFramesRapidly) {
                        flushAndReset();
                    }
                }
            }
            tupleIndex = 0;
        }

        private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex, FrameTupleReference tupleRef) throws HyracksDataException {
            try {
                tb.reset();
                for (int f = 0; f < projectionList.length; f++) {
                    int k = projectionToOutColumns[f];
                    if (k >= 0) {
                        eval[k].evaluate(tupleRef, result);
                        tb.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
                    } else {
                        tb.addField(accessor, tIndex, projectionList[f]);
                    }
                }
            } catch (HyracksDataException e) {
                throw HyracksDataException.create(ErrorCode.ERROR_PROCESSING_TUPLE, e, tupleIndex);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            if (isOpen) {
                super.fail();
            }
        }

        @Override
        public void flush() throws HyracksDataException {
            appender.flush(writer);
        }
    };
}

Example 5 with AbstractOneInputOneOutputOneFramePushRuntime

use of org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime in project asterixdb by apache.

the class RunningAggregateRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    final int[] projectionToOutColumns = new int[projectionList.length];
    for (int j = 0; j < projectionList.length; j++) {
        projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
    }
    return new AbstractOneInputOneOutputOneFramePushRuntime() {

        private final IPointable p = VoidPointable.FACTORY.createPointable();

        private final IRunningAggregateEvaluator[] raggs = new IRunningAggregateEvaluator[runningAggregates.length];

        private final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);

        private boolean first = true;

        private boolean isOpen = false;

        @Override
        public void open() throws HyracksDataException {
            initAccessAppendRef(ctx);
            if (first) {
                first = false;
                int n = runningAggregates.length;
                for (int i = 0; i < n; i++) {
                    raggs[i] = runningAggregates[i].createRunningAggregateEvaluator(ctx);
                }
            }
            for (int i = 0; i < runningAggregates.length; i++) {
                raggs[i].init();
            }
            isOpen = true;
            writer.open();
        }

        @Override
        public void close() throws HyracksDataException {
            if (isOpen) {
                super.close();
            }
        }

        @Override
        public void fail() throws HyracksDataException {
            if (isOpen) {
                super.fail();
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            tAccess.reset(buffer);
            int nTuple = tAccess.getTupleCount();
            for (int t = 0; t < nTuple; t++) {
                tRef.reset(tAccess, t);
                produceTuple(tupleBuilder, tAccess, t, tRef);
                appendToFrameFromTupleBuilder(tupleBuilder);
            }
        }

        private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex, FrameTupleReference tupleRef) throws HyracksDataException {
            tb.reset();
            for (int f = 0; f < projectionList.length; f++) {
                int k = projectionToOutColumns[f];
                if (k >= 0) {
                    raggs[k].step(tupleRef, p);
                    tb.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
                } else {
                    tb.addField(accessor, tIndex, projectionList[f]);
                }
            }
        }

        @Override
        public void flush() throws HyracksDataException {
            appender.flush(writer);
        }
    };
}