Examples with JetServiceBackend com.hazelcast.jet.impl.JetServiceBackend used on opensource projects

Example 26 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class SplitBrainTest method when_quorumIsLostOnBothSides_then_jobRestartsAfterMerge.

@Test
public void when_quorumIsLostOnBothSides_then_jobRestartsAfterMerge() {
    int firstSubClusterSize = 2;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    NoOutputSourceP.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<HazelcastInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(NoOutputSourceP::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].getJet().newJob(dag, new JobConfig().setSplitBrainProtection(true));
        assertOpenEventually(NoOutputSourceP.executionStarted);
    };
    BiConsumer<HazelcastInstance[], HazelcastInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        NoOutputSourceP.proceedLatch.countDown();
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetServiceBackend service1 = getJetServiceBackend(firstSubCluster[0]);
            JetServiceBackend service2 = getJetServiceBackend(secondSubCluster[0]);
            MasterContext masterContext = service1.getJobCoordinationService().getMasterContext(jobId);
            assertNotNull(masterContext);
            masterContext = service2.getJobCoordinationService().getMasterContext(jobId);
            assertNotNull(masterContext);
        });
        assertTrueAllTheTime(() -> {
            JetServiceBackend service1 = getJetServiceBackend(firstSubCluster[0]);
            JetServiceBackend service2 = getJetServiceBackend(secondSubCluster[0]);
            JobStatus status1 = service1.getJobCoordinationService().getJobStatus(jobId).get();
            JobStatus status2 = service2.getJobCoordinationService().getJobStatus(jobId).get();
            assertStatusNotRunningOrStarting(status1);
            assertStatusNotRunningOrStarting(status2);
        }, 20);
    };
    Consumer<HazelcastInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals(clusterSize * 2, MockPS.initCount.get());
            assertEquals(clusterSize * 2, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue("received " + t, t instanceof CancellationException));
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}

Example 27 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class SplitBrainTest method when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides.

@Test
public void when_splitBrainProtectionIsDisabled_then_jobCompletesOnBothSides() {
    int firstSubClusterSize = 2;
    int secondSubClusterSize = 2;
    int clusterSize = firstSubClusterSize + secondSubClusterSize;
    NoOutputSourceP.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    Job[] jobRef = new Job[1];
    Consumer<HazelcastInstance[]> beforeSplit = instances -> {
        MockPS processorSupplier = new MockPS(NoOutputSourceP::new, clusterSize);
        DAG dag = new DAG().vertex(new Vertex("test", processorSupplier));
        jobRef[0] = instances[0].getJet().newJob(dag);
        assertOpenEventually(NoOutputSourceP.executionStarted);
    };
    BiConsumer<HazelcastInstance[], HazelcastInstance[]> onSplit = (firstSubCluster, secondSubCluster) -> {
        NoOutputSourceP.proceedLatch.countDown();
        long jobId = jobRef[0].getId();
        assertTrueEventually(() -> {
            JetServiceBackend service1 = getJetServiceBackend(firstSubCluster[0]);
            JetServiceBackend service2 = getJetServiceBackend(secondSubCluster[0]);
            assertEquals(COMPLETED, service1.getJobCoordinationService().getJobStatus(jobId).get());
            assertEquals(COMPLETED, service2.getJobCoordinationService().getJobStatus(jobId).get());
        });
    };
    Consumer<HazelcastInstance[]> afterMerge = instances -> {
        assertTrueEventually(() -> {
            assertEquals("init count", clusterSize * 2, MockPS.initCount.get());
            assertEquals("close count", clusterSize * 2, MockPS.closeCount.get());
        });
        assertEquals(clusterSize, MockPS.receivedCloseErrors.size());
        MockPS.receivedCloseErrors.forEach(t -> assertTrue("received " + t, t instanceof CancellationException));
    };
    testSplitBrain(firstSubClusterSize, secondSubClusterSize, beforeSplit, onSplit, afterMerge);
}

Example 28 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class ExecutionLifecycleTest method getJobResult.

private JobResult getJobResult(Job job) {
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instance());
    assertNull(jetServiceBackend.getJobRepository().getJobRecord(job.getId()));
    JobResult jobResult = jetServiceBackend.getJobRepository().getJobResult(job.getId());
    assertNotNull(jobResult);
    return jobResult;
}

Example 29 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class VertexDef_HigherPrioritySourceTest method assertHigherPriorityVertices.

private void assertHigherPriorityVertices(Vertex... vertices) {
    JobConfig jobConfig = new JobConfig();
    Map<MemberInfo, ExecutionPlan> executionPlans = createExecutionPlans(nodeEngineImpl, membersView, dag, 0, 0, jobConfig, 0, false, null);
    ExecutionPlan plan = executionPlans.values().iterator().next();
    SnapshotContext ssContext = new SnapshotContext(mock(ILogger.class), "job", 0, EXACTLY_ONCE);
    // In the production code the plan#initialize is only called from places where we have already set up the
    // processor classloaders
    JetServiceBackend jetService = nodeEngineImpl.getService(JetServiceBackend.SERVICE_NAME);
    jetService.getJobClassLoaderService().getOrCreateClassLoader(jobConfig, 0, JobPhase.EXECUTION);
    try {
        jetService.getJobClassLoaderService().prepareProcessorClassLoaders(0);
        plan.initialize(nodeEngineImpl, 0, 0, ssContext, null, (InternalSerializationService) nodeEngineImpl.getSerializationService());
    } finally {
        jetService.getJobClassLoaderService().clearProcessorClassLoaders();
    }
    jetService.getJobClassLoaderService().tryRemoveClassloadersForJob(0, JobPhase.EXECUTION);
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(plan.getVertices());
    String actualHigherPriorityVertices = plan.getVertices().stream().filter(v -> higherPriorityVertices.contains(v.vertexId())).map(VertexDef::name).sorted().collect(joining("\n"));
    String expectedVertices = Arrays.stream(vertices).map(Vertex::getName).sorted().collect(joining("\n"));
    assertEquals(expectedVertices, actualHigherPriorityVertices);
}

Example 30 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class NonSmartClientTest method when_lightJobSubmittedToNonMaster_then_coordinatedByNonMaster.

@Test
public void when_lightJobSubmittedToNonMaster_then_coordinatedByNonMaster() {
    Job job = nonMasterClient.getJet().newLightJob(streamingDag());
    JetServiceBackend service = getNodeEngineImpl(nonMasterInstance).getService(JetServiceBackend.SERVICE_NAME);
    assertTrueEventually(() -> assertNotNull(service.getJobCoordinationService().getLightMasterContexts().get(job.getId())));
}