Examples with ExecutionContext com.hazelcast.jet.impl.execution.ExecutionContext used on opensource projects

Example 11 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class Networking method createFlowControlPacket.

private Map<Address, byte[]> createFlowControlPacket() throws IOException {
    class MemberData {

        final BufferObjectDataOutput output = createObjectDataOutput(nodeEngine, lastFlowPacketSize);

        final Connection memberConnection;

        Long startedExecutionId;

        MemberData(Address address) {
            memberConnection = getMemberConnection(nodeEngine, address);
        }
    }
    Map<Address, MemberData> res = new HashMap<>();
    for (ExecutionContext execCtx : jobExecutionService.getExecutionContexts()) {
        Map<SenderReceiverKey, ReceiverTasklet> receiverMap = execCtx.receiverMap();
        if (receiverMap == null) {
            continue;
        }
        for (Entry<SenderReceiverKey, ReceiverTasklet> en : receiverMap.entrySet()) {
            assert !en.getKey().address.equals(nodeEngine.getThisAddress());
            MemberData md = res.computeIfAbsent(en.getKey().address, address -> new MemberData(address));
            if (md.startedExecutionId == null) {
                md.startedExecutionId = execCtx.executionId();
                md.output.writeLong(md.startedExecutionId);
            }
            assert en.getKey().vertexId != TERMINAL_VERTEX_ID;
            md.output.writeInt(en.getKey().vertexId);
            md.output.writeInt(en.getKey().ordinal);
            md.output.writeInt(en.getValue().updateAndGetSendSeqLimitCompressed(md.memberConnection));
        }
        for (MemberData md : res.values()) {
            if (md.startedExecutionId != null) {
                // write a mark to terminate values for an execution
                md.output.writeInt(TERMINAL_VERTEX_ID);
                md.startedExecutionId = null;
            }
        }
    }
    for (MemberData md : res.values()) {
        assert md.output.position() > 0;
        // write a mark to terminate all executions
        // Execution IDs are generated using Flake ID generator and those are >0 normally, we
        // use MIN_VALUE as a terminator.
        md.output.writeLong(TERMINAL_EXECUTION_ID);
    }
    // finalize the packets
    int maxSize = 0;
    for (Entry<Address, MemberData> entry : res.entrySet()) {
        byte[] data = entry.getValue().output.toByteArray();
        // we break type safety to avoid creating a new map, we replace the values to a different type in place
        @SuppressWarnings({ "unchecked", "rawtypes" }) Entry<Address, byte[]> entry1 = (Entry) entry;
        entry1.setValue(data);
        if (data.length > maxSize) {
            maxSize = data.length;
        }
    }
    lastFlowPacketSize = maxSize;
    return (Map) res;
}

Example 12 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class SnapshotPhase2Operation method doRun.

@Override
protected CompletableFuture<Void> doRun() {
    JetServiceBackend service = getJetServiceBackend();
    ExecutionContext ctx = service.getJobExecutionService().assertExecutionContext(getCallerAddress(), jobId(), executionId, getClass().getSimpleName());
    assert !ctx.isLightJob() : "snapshot phase 2 started on a light job: " + idToString(executionId);
    return ctx.beginSnapshotPhase2(snapshotId, success).whenComplete((r, t) -> {
        if (t != null) {
            getLogger().warning(String.format("Snapshot %d phase 2 for %s finished with an error on member: %s", snapshotId, ctx.jobNameAndExecutionId(), t), t);
        } else {
            logFine(getLogger(), "Snapshot %s phase 2 for %s finished successfully on member", snapshotId, ctx.jobNameAndExecutionId());
        }
    });
}

Example 13 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class GetLocalJobMetricsOperation method run.

@Override
public void run() {
    JetServiceBackend service = getJetServiceBackend();
    ExecutionContext executionContext = service.getJobExecutionService().getExecutionContext(executionId);
    if (executionContext == null) {
        throw new ExecutionNotFoundException(executionId);
    }
    response = executionContext.getJobMetrics();
}

Example 14 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class HazelcastConnector_RestartTest method waitExecutionDoneOnMember.

private void waitExecutionDoneOnMember(HazelcastInstance instance, long executionId) {
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instance);
    JobExecutionService executionService = jetServiceBackend.getJobExecutionService();
    ExecutionContext execCtx = executionService.getExecutionContext(executionId);
    assertTrueEventually(() -> assertTrue(execCtx == null || execCtx.getExecutionFuture().isDone()));
}

Example 15 with ExecutionContext

use of com.hazelcast.jet.impl.execution.ExecutionContext in project hazelcast by hazelcast.

the class ExecutionLifecycleTest method when_executionCancelledBeforeStart_then_jobFutureIsCancelledOnExecute.

@Test
public void when_executionCancelledBeforeStart_then_jobFutureIsCancelledOnExecute() {
    // not applicable to light jobs - we hack around with ExecutionContext
    assumeFalse(useLightJob);
    // Given
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, MEMBER_COUNT)));
    NodeEngineImpl nodeEngineImpl = getNodeEngineImpl(instance());
    Address localAddress = nodeEngineImpl.getThisAddress();
    ClusterServiceImpl clusterService = (ClusterServiceImpl) nodeEngineImpl.getClusterService();
    MembersView membersView = clusterService.getMembershipManager().getMembersView();
    int memberListVersion = membersView.getVersion();
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instance());
    long jobId = 0;
    long executionId = 1;
    JobConfig jobConfig = new JobConfig();
    final Map<MemberInfo, ExecutionPlan> executionPlans = ExecutionPlanBuilder.createExecutionPlans(nodeEngineImpl, membersView.getMembers(), dag, jobId, executionId, jobConfig, NO_SNAPSHOT, false, null);
    ExecutionPlan executionPlan = executionPlans.get(membersView.getMember(localAddress));
    jetServiceBackend.getJobClassLoaderService().getOrCreateClassLoader(jobConfig, jobId, COORDINATOR);
    Set<MemberInfo> participants = new HashSet<>(membersView.getMembers());
    jetServiceBackend.getJobExecutionService().initExecution(jobId, executionId, localAddress, memberListVersion, participants, executionPlan);
    ExecutionContext executionContext = jetServiceBackend.getJobExecutionService().getExecutionContext(executionId);
    executionContext.terminateExecution(null);
    // When
    CompletableFuture<Void> future = executionContext.beginExecution(jetServiceBackend.getTaskletExecutionService());
    // Then
    expectedException.expect(CancellationException.class);
    future.join();
}