Examples with ExecutionPhase io.crate.execution.dsl.phases.ExecutionPhase used on opensource projects

Example 1 with ExecutionPhase

use of io.crate.execution.dsl.phases.ExecutionPhase in project crate by crate.

the class JobLauncher method executeBulk.

public List<CompletableFuture<Long>> executeBulk(TransactionContext txnCtx) {
    Iterable<NodeOperation> nodeOperations = nodeOperationTrees.stream().flatMap(opTree -> opTree.nodeOperations().stream())::iterator;
    Map<String, Collection<NodeOperation>> operationByServer = NodeOperationGrouper.groupByServer(nodeOperations);
    List<ExecutionPhase> handlerPhases = new ArrayList<>(nodeOperationTrees.size());
    List<RowConsumer> handlerConsumers = new ArrayList<>(nodeOperationTrees.size());
    List<CompletableFuture<Long>> results = new ArrayList<>(nodeOperationTrees.size());
    for (NodeOperationTree nodeOperationTree : nodeOperationTrees) {
        CollectingRowConsumer<?, Long> consumer = new CollectingRowConsumer<>(Collectors.collectingAndThen(Collectors.summingLong(r -> ((long) r.get(0))), sum -> sum));
        handlerConsumers.add(consumer);
        results.add(consumer.completionFuture());
        handlerPhases.add(nodeOperationTree.leaf());
    }
    try {
        setupTasks(txnCtx, operationByServer, handlerPhases, handlerConsumers);
    } catch (Throwable throwable) {
        return Collections.singletonList(CompletableFuture.failedFuture(throwable));
    }
    return results;
}

Example 2 with ExecutionPhase

use of io.crate.execution.dsl.phases.ExecutionPhase in project crate by crate.

the class JobLauncher method getHandlerBucketReceivers.

private List<PageBucketReceiver> getHandlerBucketReceivers(RootTask rootTask, List<Tuple<ExecutionPhase, RowConsumer>> handlerPhases) {
    final List<PageBucketReceiver> pageBucketReceivers = new ArrayList<>(handlerPhases.size());
    for (Tuple<ExecutionPhase, ?> handlerPhase : handlerPhases) {
        Task ctx = rootTask.getTaskOrNull(handlerPhase.v1().phaseId());
        if (ctx instanceof DownstreamRXTask) {
            PageBucketReceiver pageBucketReceiver = ((DownstreamRXTask) ctx).getBucketReceiver((byte) 0);
            pageBucketReceivers.add(pageBucketReceiver);
        }
    }
    return pageBucketReceivers;
}

Example 3 with ExecutionPhase

use of io.crate.execution.dsl.phases.ExecutionPhase in project crate by crate.

the class JobLauncher method createHandlerPhaseAndReceivers.

private List<Tuple<ExecutionPhase, RowConsumer>> createHandlerPhaseAndReceivers(List<ExecutionPhase> handlerPhases, List<RowConsumer> handlerReceivers, InitializationTracker initializationTracker) {
    List<Tuple<ExecutionPhase, RowConsumer>> handlerPhaseAndReceiver = new ArrayList<>();
    ListIterator<RowConsumer> consumerIt = handlerReceivers.listIterator();
    for (ExecutionPhase handlerPhase : handlerPhases) {
        InterceptingRowConsumer interceptingBatchConsumer = new InterceptingRowConsumer(jobId, consumerIt.next(), initializationTracker, executor, transportKillJobsNodeAction);
        handlerPhaseAndReceiver.add(new Tuple<>(handlerPhase, interceptingBatchConsumer));
    }
    return handlerPhaseAndReceiver;
}

Example 4 with ExecutionPhase

use of io.crate.execution.dsl.phases.ExecutionPhase in project crate by crate.

the class JobSetup method registerContextPhases.

private void registerContextPhases(Iterable<? extends NodeOperation> nodeOperations, Context context) {
    for (NodeOperation nodeOperation : nodeOperations) {
        // context for nodeOperations without dependencies can be built immediately (e.g. FetchPhase)
        if (nodeOperation.downstreamExecutionPhaseId() == NodeOperation.NO_DOWNSTREAM) {
            LOGGER.trace("Building context for nodeOp without downstream: {}", nodeOperation);
            createContexts(nodeOperation.executionPhase(), context);
            context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
        }
        if (ExecutionPhases.hasDirectResponseDownstream(nodeOperation.downstreamNodes())) {
            var executionPhase = nodeOperation.executionPhase();
            CircuitBreaker breaker = breaker();
            int ramAccountingBlockSizeInBytes = BlockBasedRamAccounting.blockSizeInBytes(breaker.getLimit());
            var ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, executionPhase.label()), ramAccountingBlockSizeInBytes);
            Streamer<?>[] streamers = StreamerVisitor.streamersFromOutputs(executionPhase);
            SingleBucketBuilder bucketBuilder = new SingleBucketBuilder(streamers, ramAccounting);
            context.directResponseFutures.add(bucketBuilder.completionFuture().whenComplete((res, err) -> ramAccounting.close()));
            context.registerBatchConsumer(nodeOperation.downstreamExecutionPhaseId(), bucketBuilder);
        }
    }
}

Example 5 with ExecutionPhase

use of io.crate.execution.dsl.phases.ExecutionPhase in project crate by crate.

the class NodeOperationCtxTest method testFindLeafs.

@Test
public void testFindLeafs() {
    ExecutionPhase p1 = newPhase(0, "n1", "n2");
    ExecutionPhase p2 = newPhase(1, "n1", "n2");
    ExecutionPhase p3 = newPhase(2, "n2");
    NodeOperationCtx opCtx = new NodeOperationCtx("n1", List.of(NodeOperation.withDownstream(p1, p2, (byte) 0), NodeOperation.withDownstream(p2, p3, (byte) 0)));
    assertThat(opCtx.findLeafs(), is(IntArrayList.from(2)));
}