Examples with JetServiceBackend com.hazelcast.jet.impl.JetServiceBackend used on opensource projects

Example 21 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class ExecutionContext method initialize.

public ExecutionContext initialize(@Nonnull Address coordinator, @Nonnull Set<Address> participants, @Nonnull ExecutionPlan plan) {
    this.coordinator = coordinator;
    this.participants = participants;
    jobConfig = plan.getJobConfig();
    jobName = jobConfig.getName() == null ? jobName : jobConfig.getName();
    // Must be populated early, so all processor suppliers are
    // available to be completed in the case of init failure
    vertices = plan.getVertices();
    snapshotContext = new SnapshotContext(nodeEngine.getLogger(SnapshotContext.class), jobNameAndExecutionId(), plan.lastSnapshotId(), jobConfig.getProcessingGuarantee());
    JetServiceBackend jetServiceBackend = nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
    serializationService = jetServiceBackend.createSerializationService(jobConfig.getSerializerConfigs());
    metricsEnabled = jobConfig.isMetricsEnabled() && nodeEngine.getConfig().getMetricsConfig().isEnabled();
    plan.initialize(nodeEngine, jobId, executionId, snapshotContext, tempDirectories, serializationService);
    int numPrioritySsTasklets = plan.getStoreSnapshotTaskletCount() != 0 ? plan.getHigherPriorityVertexCount() : 0;
    snapshotContext.initTaskletCount(plan.getProcessorTaskletCount(), plan.getStoreSnapshotTaskletCount(), numPrioritySsTasklets);
    Map<SenderReceiverKey, ReceiverTasklet> receiverMapTmp = new HashMap<>();
    for (Entry<Integer, Map<Integer, Map<Address, ReceiverTasklet>>> vertexIdEntry : plan.getReceiverMap().entrySet()) {
        for (Entry<Integer, Map<Address, ReceiverTasklet>> ordinalEntry : vertexIdEntry.getValue().entrySet()) {
            for (Entry<Address, ReceiverTasklet> addressEntry : ordinalEntry.getValue().entrySet()) {
                SenderReceiverKey key = new SenderReceiverKey(vertexIdEntry.getKey(), ordinalEntry.getKey(), addressEntry.getKey());
                // the queue might already exist, if some data were received for it, or it will be created now
                Queue<byte[]> queue = receiverQueuesMap.computeIfAbsent(key, CREATE_RECEIVER_QUEUE_FN);
                ReceiverTasklet receiverTasklet = addressEntry.getValue();
                receiverTasklet.initIncomingQueue(queue);
                receiverMapTmp.put(new SenderReceiverKey(vertexIdEntry.getKey(), ordinalEntry.getKey(), addressEntry.getKey()), receiverTasklet);
            }
        }
    }
    this.receiverMap = unmodifiableMap(receiverMapTmp);
    Map<SenderReceiverKey, SenderTasklet> senderMapTmp = new HashMap<>();
    for (Entry<Integer, Map<Integer, Map<Address, SenderTasklet>>> e1 : plan.getSenderMap().entrySet()) {
        for (Entry<Integer, Map<Address, SenderTasklet>> e2 : e1.getValue().entrySet()) {
            for (Entry<Address, SenderTasklet> e3 : e2.getValue().entrySet()) {
                senderMapTmp.put(new SenderReceiverKey(e1.getKey(), e2.getKey(), e3.getKey()), e3.getValue());
            }
        }
    }
    this.senderMap = unmodifiableMap(senderMapTmp);
    tasklets = plan.getTasklets();
    return this;
}

Example 22 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class GetJobConfigOperation method run.

@Override
public void run() {
    JetServiceBackend service = getJetServiceBackend();
    response = service.getJobConfig(jobId(), isLightJob);
}

Example 23 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class InitExecutionOperation method doRun.

@Override
protected CompletableFuture<?> doRun() {
    ILogger logger = getLogger();
    if (!getNodeEngine().getLocalMember().getVersion().asVersion().equals(coordinatorVersion)) {
        // the same address.
        throw new JetException("Mismatch between coordinator and participant version");
    }
    JetServiceBackend service = getJetServiceBackend();
    Address caller = getCallerAddress();
    LoggingUtil.logFine(logger, "Initializing execution plan for %s from %s", jobIdAndExecutionId(jobId(), executionId), caller);
    ExecutionPlan plan = deserializePlan(serializedPlan);
    if (isLightJob) {
        return service.getJobExecutionService().runLightJob(jobId(), executionId, caller, coordinatorMemberListVersion, participants, plan);
    } else {
        service.getJobExecutionService().initExecution(jobId(), executionId, caller, coordinatorMemberListVersion, participants, plan);
        return CompletableFuture.completedFuture(null);
    }
}

Example 24 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class InitExecutionOperation method deserializePlan.

private ExecutionPlan deserializePlan(Data planBlob) {
    if (isLightJob) {
        return getNodeEngine().getSerializationService().toObject(planBlob);
    } else {
        JetServiceBackend service = getJetServiceBackend();
        JobConfig jobConfig = service.getJobConfig(jobId(), isLightJob);
        JobClassLoaderService jobClassloaderService = service.getJobClassLoaderService();
        ClassLoader cl = jobClassloaderService.getOrCreateClassLoader(jobConfig, jobId(), EXECUTION);
        try {
            jobClassloaderService.prepareProcessorClassLoaders(jobId());
            return deserializeWithCustomClassLoader(getNodeEngine().getSerializationService(), cl, planBlob);
        } finally {
            jobClassloaderService.clearProcessorClassLoaders();
        }
    }
}

Example 25 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class TopologyChangeTest method when_jobParticipantReceivesStaleInitOperation_then_jobRestarts.

@Test
public void when_jobParticipantReceivesStaleInitOperation_then_jobRestarts() {
    // Given
    HazelcastInstance newInstance = createHazelcastInstance(config);
    for (HazelcastInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT + 1, instance);
    }
    rejectOperationsBetween(instances[0], instances[2], JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount + 1)));
    Job job = instances[0].getJet().newJob(dag);
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instances[0]);
    assertTrueEventually(() -> assertFalse(jetServiceBackend.getJobCoordinationService().getMasterContexts().isEmpty()));
    MasterContext masterContext = jetServiceBackend.getJobCoordinationService().getMasterContext(job.getId());
    assertTrueEventually(() -> {
        assertEquals(STARTING, masterContext.jobStatus());
        assertNotEquals(0, masterContext.executionId());
    });
    // When
    long executionId = masterContext.executionId();
    assertTrueEventually(() -> {
        Arrays.stream(instances).filter(instance -> !instance.getCluster().getLocalMember().isLiteMember()).filter(instance -> instance != instances[2]).map(JetTestSupport::getJetServiceBackend).map(service -> service.getJobExecutionService().getExecutionContext(executionId)).forEach(Assert::assertNotNull);
    });
    newInstance.getLifecycleService().terminate();
    for (HazelcastInstance instance : instances) {
        assertClusterSizeEventually(NODE_COUNT, instance);
    }
    resetPacketFiltersFrom(instances[0]);
    // Then
    job.join();
    assertNotEquals(executionId, masterContext.executionId());
}