Examples with ProcessState io.prestosql.operator.WorkProcessor.ProcessState used on opensource projects

Example 1 with ProcessState

use of io.prestosql.operator.WorkProcessor.ProcessState in project hetu-core by openlookeng.

the class TestWorkProcessor method testYield.

@Test(timeOut = 5000)
public void testYield() {
    SettableFuture<?> future = SettableFuture.create();
    List<ProcessState<Integer>> baseScenario = ImmutableList.of(ProcessState.ofResult(1), ProcessState.ofResult(2), ProcessState.blocked(future), ProcessState.ofResult(3), ProcessState.ofResult(4), ProcessState.finished());
    AtomicBoolean yieldSignal = new AtomicBoolean();
    WorkProcessor<Integer> processor = processorFrom(baseScenario).yielding(yieldSignal::get);
    // no yield, process normally
    assertResult(processor, 1);
    yieldSignal.set(true);
    assertYields(processor);
    // processor should progress since it yielded last time
    assertResult(processor, 2);
    // yield signal is still set
    assertYields(processor);
    // base scenario future blocks
    assertBlocks(processor);
    assertUnblocks(processor, future);
    // continue to process normally
    yieldSignal.set(false);
    assertResult(processor, 3);
    assertResult(processor, 4);
    assertFinishes(processor);
}

Example 2 with ProcessState

use of io.prestosql.operator.WorkProcessor.ProcessState in project hetu-core by openlookeng.

the class TestWorkProcessor method testFinishWheb.

@Test(timeOut = 5000)
public void testFinishWheb() {
    AtomicBoolean finished = new AtomicBoolean();
    SettableFuture<?> future = SettableFuture.create();
    List<ProcessState<Integer>> scenario = ImmutableList.of(ProcessState.ofResult(1), ProcessState.yield(), ProcessState.blocked(future), ProcessState.ofResult(2));
    WorkProcessor<Integer> processor = processorFrom(scenario).finishWhen(finished::get);
    assertResult(processor, 1);
    assertYields(processor);
    assertBlocks(processor);
    finished.set(true);
    assertBlocks(processor);
    assertUnblocks(processor, future);
    assertFinishes(processor);
}

Example 3 with ProcessState

use of io.prestosql.operator.WorkProcessor.ProcessState in project hetu-core by openlookeng.

the class WorkProcessorUtils method transform.

static <T, R> WorkProcessor<R> transform(WorkProcessor<T> processor, Transformation<T, R> transformation) {
    requireNonNull(processor, "processor is null");
    requireNonNull(transformation, "transformation is null");
    return create(new WorkProcessor.Process<R>() {

        @RestorableConfig(stateClassName = "TransformState", uncapturedFields = { "element" })
        private final RestorableConfig restorableConfig = null;

        T element;

        @Override
        public ProcessState<R> process() {
            while (true) {
                if (element == null && !processor.isFinished()) {
                    if (processor.process()) {
                        if (!processor.isFinished()) {
                            element = requireNonNull(processor.getResult(), "result is null");
                        }
                    } else if (processor.isBlocked()) {
                        return ProcessState.blocked(processor.getBlockedFuture());
                    } else {
                        return ProcessState.yield();
                    }
                }
                TransformationState<R> state = requireNonNull(transformation.process(element), "state is null");
                if (state.isNeedsMoreData()) {
                    checkState(!processor.isFinished(), "Cannot request more data when base processor is finished");
                    // set element to empty() in order to fetch a new one
                    element = null;
                }
                // pass-through transformation state if it doesn't require new data
                switch(state.getType()) {
                    case NEEDS_MORE_DATA:
                        break;
                    case BLOCKED:
                        return ProcessState.blocked(state.getBlocked());
                    case YIELD:
                        return ProcessState.yield();
                    case RESULT:
                        return ProcessState.ofResult(state.getResult());
                    case FINISHED:
                        return ProcessState.finished();
                }
            }
        }

        @Override
        public Object capture(BlockEncodingSerdeProvider serdeProvider) {
            TransformState myState = new TransformState();
            myState.transformation = transformation.capture(serdeProvider);
            myState.validElement = element != null;
            myState.processor = processor.capture(serdeProvider);
            return myState;
        }

        @Override
        public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
            TransformState myState = (TransformState) state;
            transformation.restore(myState.transformation, serdeProvider);
            processor.restore(myState.processor, serdeProvider);
            if (myState.validElement) {
                checkArgument(processor.getResult() != null);
                element = processor.getResult();
            }
        }

        @Override
        public Object captureResult(R result, BlockEncodingSerdeProvider serdeProvider) {
            return transformation.captureResult(result, serdeProvider);
        }

        @Override
        public R restoreResult(Object resultState, BlockEncodingSerdeProvider serdeProvider) {
            return transformation.restoreResult(resultState, serdeProvider);
        }
    });
}

Example 4 with ProcessState

use of io.prestosql.operator.WorkProcessor.ProcessState in project hetu-core by openlookeng.

the class WorkProcessorUtils method mergeSorted.

static <T> WorkProcessor<T> mergeSorted(List<WorkProcessor<T>> processorList, Comparator<T> comparator) {
    requireNonNull(comparator, "comparator is null");
    checkArgument(processorList.size() > 0, "There must be at least one base processor");
    PriorityQueue<ElementAndProcessor<T>> queue = new PriorityQueue<>(2, comparing(ElementAndProcessor::getElement, comparator));
    return create(new WorkProcessor.Process<T>() {

        @RestorableConfig(stateClassName = "MergeSortedState", uncapturedFields = { "val$queue" })
        private final RestorableConfig restorableConfig = null;

        int nextProcessor;

        WorkProcessor<T> processor = requireNonNull(processorList.get(nextProcessor++));

        @Override
        public ProcessState<T> process() {
            while (true) {
                if (processor.process()) {
                    if (!processor.isFinished()) {
                        queue.add(new ElementAndProcessor<>(processor.getResult(), processor));
                    }
                } else if (processor.isBlocked()) {
                    return ProcessState.blocked(processor.getBlockedFuture());
                } else {
                    return ProcessState.yield();
                }
                if (nextProcessor < processorList.size()) {
                    processor = requireNonNull(processorList.get(nextProcessor++));
                    continue;
                }
                if (queue.isEmpty()) {
                    processor = null;
                    return ProcessState.finished();
                }
                ElementAndProcessor<T> elementAndProcessor = queue.poll();
                processor = elementAndProcessor.getProcessor();
                return ProcessState.ofResult(elementAndProcessor.getElement());
            }
        }

        @Override
        public Object capture(BlockEncodingSerdeProvider serdeProvider) {
            MergeSortedState myState = new MergeSortedState();
            myState.processorList = new Object[processorList.size()];
            for (int i = 0; i < processorList.size(); i++) {
                myState.processorList[i] = processorList.get(i).capture(serdeProvider);
            }
            myState.nextProcessor = nextProcessor;
            // Record which processors are in queue
            myState.queueProcessorIndex = new ArrayList<>();
            for (ElementAndProcessor enp : queue) {
                myState.queueProcessorIndex.add(processorList.indexOf(enp.processor));
            }
            myState.processor = processorList.indexOf(processor);
            return myState;
        }

        @Override
        public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
            MergeSortedState myState = (MergeSortedState) state;
            checkArgument(myState.processorList.length == processorList.size());
            for (int i = 0; i < myState.processorList.length; i++) {
                processorList.get(i).restore(myState.processorList[i], serdeProvider);
            }
            nextProcessor = myState.nextProcessor;
            queue.clear();
            for (Integer queueProcessorIndex : myState.queueProcessorIndex) {
                checkArgument(queueProcessorIndex < processorList.size(), "Processor index exceeded processor list.");
                queue.add(new ElementAndProcessor<>(processorList.get(queueProcessorIndex).getResult(), processorList.get(queueProcessorIndex)));
            }
            this.processor = processorList.get(myState.processor);
        }

        @Override
        public Object captureResult(T result, BlockEncodingSerdeProvider serdeProvider) {
            for (int i = 0; i < processorList.size(); i++) {
                if (((ProcessWorkProcessor) processorList.get(i)).state.getType() == ProcessState.Type.RESULT && processorList.get(i).getResult() == result) {
                    return i;
                }
            }
            throw new IllegalArgumentException("Unable to capture result.");
        }

        @Override
        public T restoreResult(Object resultState, BlockEncodingSerdeProvider serdeProvider) {
            checkArgument(((int) resultState) < processorList.size());
            ProcessWorkProcessor<T> targetProcessor = (ProcessWorkProcessor) processorList.get((int) resultState);
            checkArgument(targetProcessor.state != null && targetProcessor.state.getType() == ProcessState.Type.RESULT);
            return targetProcessor.getResult();
        }
    });
}