Examples with NodeOperation io.crate.execution.dsl.phases.NodeOperation used on opensource projects

Example 1 with NodeOperation

use of io.crate.execution.dsl.phases.NodeOperation in project crate by crate.

the class JobRequest method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeLong(jobId.getMostSignificantBits());
    out.writeLong(jobId.getLeastSignificantBits());
    out.writeString(coordinatorNodeId);
    out.writeVInt(nodeOperations.size());
    for (NodeOperation nodeOperation : nodeOperations) {
        nodeOperation.writeTo(out);
    }
    out.writeBoolean(enableProfiling);
    sessionSettings.writeTo(out);
}

Example 2 with NodeOperation

use of io.crate.execution.dsl.phases.NodeOperation in project crate by crate.

the class JobLauncher method executeBulk.

public List<CompletableFuture<Long>> executeBulk(TransactionContext txnCtx) {
    Iterable<NodeOperation> nodeOperations = nodeOperationTrees.stream().flatMap(opTree -> opTree.nodeOperations().stream())::iterator;
    Map<String, Collection<NodeOperation>> operationByServer = NodeOperationGrouper.groupByServer(nodeOperations);
    List<ExecutionPhase> handlerPhases = new ArrayList<>(nodeOperationTrees.size());
    List<RowConsumer> handlerConsumers = new ArrayList<>(nodeOperationTrees.size());
    List<CompletableFuture<Long>> results = new ArrayList<>(nodeOperationTrees.size());
    for (NodeOperationTree nodeOperationTree : nodeOperationTrees) {
        CollectingRowConsumer<?, Long> consumer = new CollectingRowConsumer<>(Collectors.collectingAndThen(Collectors.summingLong(r -> ((long) r.get(0))), sum -> sum));
        handlerConsumers.add(consumer);
        results.add(consumer.completionFuture());
        handlerPhases.add(nodeOperationTree.leaf());
    }
    try {
        setupTasks(txnCtx, operationByServer, handlerPhases, handlerConsumers);
    } catch (Throwable throwable) {
        return Collections.singletonList(CompletableFuture.failedFuture(throwable));
    }
    return results;
}

Example 3 with NodeOperation

use of io.crate.execution.dsl.phases.NodeOperation in project crate by crate.

the class JobSetup method prepareSourceOperations.

/**
 * recursively build all contexts that depend on startPhaseId (excl. startPhaseId)
 * <p>
 * {@link Context#opCtx#targetToSourceMap} will be used to traverse the nodeOperations
 */
private void prepareSourceOperations(int startPhaseId, Context context) {
    IntContainer sourcePhaseIds = context.opCtx.targetToSourceMap.get(startPhaseId);
    if (sourcePhaseIds == null) {
        return;
    }
    for (IntCursor sourcePhaseId : sourcePhaseIds) {
        NodeOperation nodeOperation = context.opCtx.nodeOperationByPhaseId.get(sourcePhaseId.value);
        createContexts(nodeOperation.executionPhase(), context);
        context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
    }
    for (IntCursor sourcePhaseId : sourcePhaseIds) {
        prepareSourceOperations(sourcePhaseId.value, context);
    }
}

Example 4 with NodeOperation

use of io.crate.execution.dsl.phases.NodeOperation in project crate by crate.

the class JobSetup method registerContextPhases.

private void registerContextPhases(Iterable<? extends NodeOperation> nodeOperations, Context context) {
    for (NodeOperation nodeOperation : nodeOperations) {
        // context for nodeOperations without dependencies can be built immediately (e.g. FetchPhase)
        if (nodeOperation.downstreamExecutionPhaseId() == NodeOperation.NO_DOWNSTREAM) {
            LOGGER.trace("Building context for nodeOp without downstream: {}", nodeOperation);
            createContexts(nodeOperation.executionPhase(), context);
            context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
        }
        if (ExecutionPhases.hasDirectResponseDownstream(nodeOperation.downstreamNodes())) {
            var executionPhase = nodeOperation.executionPhase();
            CircuitBreaker breaker = breaker();
            int ramAccountingBlockSizeInBytes = BlockBasedRamAccounting.blockSizeInBytes(breaker.getLimit());
            var ramAccounting = new BlockBasedRamAccounting(b -> breaker.addEstimateBytesAndMaybeBreak(b, executionPhase.label()), ramAccountingBlockSizeInBytes);
            Streamer<?>[] streamers = StreamerVisitor.streamersFromOutputs(executionPhase);
            SingleBucketBuilder bucketBuilder = new SingleBucketBuilder(streamers, ramAccounting);
            context.directResponseFutures.add(bucketBuilder.completionFuture().whenComplete((res, err) -> ramAccounting.close()));
            context.registerBatchConsumer(nodeOperation.downstreamExecutionPhaseId(), bucketBuilder);
        }
    }
}

Example 5 with NodeOperation

use of io.crate.execution.dsl.phases.NodeOperation in project crate by crate.

the class ExecutionPhasesRootTaskTest method testGroupByServer.

@Test
public void testGroupByServer() throws Exception {
    var routingMap = new TreeMap<String, Map<String, IntIndexedContainer>>();
    routingMap.put("node1", Map.of("t1", IntArrayList.from(1, 2)));
    routingMap.put("node2", Map.of("t1", IntArrayList.from(3, 4)));
    Routing twoNodeRouting = new Routing(routingMap);
    UUID jobId = UUID.randomUUID();
    RoutedCollectPhase c1 = new RoutedCollectPhase(jobId, 1, "c1", twoNodeRouting, RowGranularity.DOC, List.of(), List.of(), WhereClause.MATCH_ALL.queryOrFallback(), DistributionInfo.DEFAULT_BROADCAST);
    MergePhase m1 = new MergePhase(jobId, 2, "merge1", 2, 1, Set.of("node3", "node4"), List.of(), List.of(), DistributionInfo.DEFAULT_BROADCAST, null);
    MergePhase m2 = new MergePhase(jobId, 3, "merge2", 2, 1, Set.of("node1", "node3"), List.of(), List.of(), DistributionInfo.DEFAULT_BROADCAST, null);
    NodeOperation n1 = NodeOperation.withDownstream(c1, m1, (byte) 0);
    NodeOperation n2 = NodeOperation.withDownstream(m1, m2, (byte) 0);
    NodeOperation n3 = NodeOperation.withDownstream(m2, mock(ExecutionPhase.class), (byte) 0);
    Map<String, Collection<NodeOperation>> groupByServer = NodeOperationGrouper.groupByServer(List.of(n1, n2, n3));
    assertThat(groupByServer.containsKey("node1"), is(true));
    assertThat(groupByServer.get("node1"), Matchers.containsInAnyOrder(n1, n3));
    assertThat(groupByServer.containsKey("node2"), is(true));
    assertThat(groupByServer.get("node2"), Matchers.containsInAnyOrder(n1));
    assertThat(groupByServer.containsKey("node3"), is(true));
    assertThat(groupByServer.get("node3"), Matchers.containsInAnyOrder(n2, n3));
    assertThat(groupByServer.containsKey("node4"), is(true));
    assertThat(groupByServer.get("node4"), Matchers.containsInAnyOrder(n2));
}