Examples with SenderTasklet com.hazelcast.jet.impl.execution.SenderTasklet used on opensource projects

Example 1 with SenderTasklet

use of com.hazelcast.jet.impl.execution.SenderTasklet in project hazelcast by hazelcast.

the class Networking method handleFlowControlPacket.

private void handleFlowControlPacket(Address fromAddr, byte[] packet) throws IOException {
    BufferObjectDataInput input = createObjectDataInput(nodeEngine, packet);
    for (; ; ) {
        final long executionId = input.readLong();
        if (executionId == TERMINAL_EXECUTION_ID) {
            break;
        }
        final Map<SenderReceiverKey, SenderTasklet> senderMap = jobExecutionService.getSenderMap(executionId);
        for (; ; ) {
            final int vertexId = input.readInt();
            if (vertexId == TERMINAL_VERTEX_ID) {
                break;
            }
            int ordinal = input.readInt();
            int sendSeqLimitCompressed = input.readInt();
            final SenderTasklet t;
            if (senderMap != null && (t = senderMap.get(new SenderReceiverKey(vertexId, ordinal, fromAddr))) != null) {
                t.setSendSeqLimitCompressed(sendSeqLimitCompressed);
            }
        }
    }
}

Example 2 with SenderTasklet

use of com.hazelcast.jet.impl.execution.SenderTasklet in project hazelcast by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge, String jobPrefix, InternalSerializationService jobSerializationService) {
    assert !edge.isLocal() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            @SuppressWarnings("unchecked") ComparatorEx<Object> origComparator = (ComparatorEx<Object>) edge.getOrderComparator();
            ComparatorEx<ObjectWithPartitionId> adaptedComparator = origComparator == null ? null : (l, r) -> origComparator.compare(l.getItem(), r.getItem());
            final InboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, jobPrefix + "/toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr, adaptedComparator);
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, memberConnections.get(destAddr), destVertexId, edge.getConfig().getPacketSizeLimit(), executionId, edge.sourceVertex().name(), edge.sourceOrdinal(), jobSerializationService);
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}

Example 3 with SenderTasklet

use of com.hazelcast.jet.impl.execution.SenderTasklet in project hazelcast-jet by hazelcast.

the class Networking method handleFlowControlPacket.

private void handleFlowControlPacket(Address fromAddr, byte[] packet) throws IOException {
    try (BufferObjectDataInput in = createObjectDataInput(nodeEngine, packet)) {
        final int executionCtxCount = in.readInt();
        for (int j = 0; j < executionCtxCount; j++) {
            final long executionId = in.readLong();
            final Map<Integer, Map<Integer, Map<Address, SenderTasklet>>> senderMap = jobExecutionService.getSenderMap(executionId);
            if (senderMap == null) {
                logMissingExeCtx(executionId);
                continue;
            }
            final int flowCtlMsgCount = in.readInt();
            for (int k = 0; k < flowCtlMsgCount; k++) {
                int destVertexId = in.readInt();
                int destOrdinal = in.readInt();
                int sendSeqLimitCompressed = in.readInt();
                final SenderTasklet t = Optional.ofNullable(senderMap.get(destVertexId)).map(ordinalMap -> ordinalMap.get(destOrdinal)).map(addrMap -> addrMap.get(fromAddr)).orElse(null);
                if (t == null) {
                    logMissingSenderTasklet(destVertexId, destOrdinal);
                    return;
                }
                t.setSendSeqLimitCompressed(sendSeqLimitCompressed);
            }
        }
    }
}

Example 4 with SenderTasklet

use of com.hazelcast.jet.impl.execution.SenderTasklet in project hazelcast-jet by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge) {
    assert edge.isDistributed() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            final ConcurrentInboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, "sender-toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr.toString().replace('.', '-'));
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, executionId, destVertexId, edge.getConfig().getPacketSizeLimit());
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}