Examples with JetServiceBackend com.hazelcast.jet.impl.JetServiceBackend used on opensource projects

Example 11 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class HazelcastConnector_RestartTest method executionId.

private Long executionId(HazelcastInstance instance, Job job) {
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instance);
    JobExecutionService executionService = jetServiceBackend.getJobExecutionService();
    return executionService.getExecutionIdForJobId(job.getId());
}

Example 12 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class HazelcastConnector_RestartTest method waitExecutionDoneOnMember.

private void waitExecutionDoneOnMember(HazelcastInstance instance, long executionId) {
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instance);
    JobExecutionService executionService = jetServiceBackend.getJobExecutionService();
    ExecutionContext execCtx = executionService.getExecutionContext(executionId);
    assertTrueEventually(() -> assertTrue(execCtx == null || execCtx.getExecutionFuture().isDone()));
}

Example 13 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class TopologyChangeTest method when_nodeIsShuttingDownDuringInit_then_jobRestarts.

@Test
public void when_nodeIsShuttingDownDuringInit_then_jobRestarts() {
    // Given that newInstance will have a long shutdown process
    for (HazelcastInstance instance : instances) {
        warmUpPartitions(instance);
    }
    dropOperationsBetween(instances[2], instances[0], PartitionDataSerializerHook.F_ID, singletonList(SHUTDOWN_REQUEST));
    rejectOperationsBetween(instances[0], instances[2], JetInitDataSerializerHook.FACTORY_ID, singletonList(INIT_EXECUTION_OP));
    // When a job participant starts its shutdown after the job is submitted
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(TestProcessors.Identity::new, nodeCount - 1)));
    Job job = instances[0].getJet().newJob(dag);
    JetServiceBackend jetServiceBackend = getJetServiceBackend(instances[0]);
    assertTrueEventually(() -> assertFalse(jetServiceBackend.getJobCoordinationService().getMasterContexts().isEmpty()));
    spawn(() -> instances[2].shutdown());
    // Then, it restarts until the shutting down node is gone
    assertJobStatusEventually(job, STARTING);
    assertTrueAllTheTime(() -> assertEquals(STARTING, job.getStatus()), 5);
    resetPacketFiltersFrom(instances[2]);
    job.join();
}

Example 14 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class TopologyChangeTest method when_coordinatorLeavesDuringExecution_then_jobCompletes.

@Test
public void when_coordinatorLeavesDuringExecution_then_jobCompletes() throws Throwable {
    // Given
    DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, nodeCount)));
    // When
    Long jobId = null;
    try {
        Job job = instances[0].getJet().newJob(dag);
        Future<Void> future = job.getFuture();
        jobId = job.getId();
        NoOutputSourceP.executionStarted.await();
        instances[0].getLifecycleService().terminate();
        // Processor#close.
        for (int i = 1; i < instances.length; i++) {
            JetServiceBackend jetServiceBackend = getJetServiceBackend(instances[i]);
            jetServiceBackend.getJobExecutionService().waitAllExecutionsTerminated();
        }
        NoOutputSourceP.proceedLatch.countDown();
        future.get();
        fail();
    } catch (ExecutionException expected) {
        assertTrue(expected.getCause() instanceof HazelcastInstanceNotActiveException);
    }
    // Then
    assertNotNull(jobId);
    final long completedJobId = jobId;
    JobRepository jobRepository = getJetServiceBackend(instances[1]).getJobRepository();
    assertTrueEventually(() -> {
        JobResult jobResult = jobRepository.getJobResult(completedJobId);
        assertNotNull(jobResult);
        assertTrue(jobResult.isSuccessful());
    });
    final int count = liteMemberFlags[0] ? (2 * nodeCount) : (2 * nodeCount - 1);
    assertEquals(count, MockPS.initCount.get());
    assertTrueEventually(() -> {
        assertEquals(count, MockPS.closeCount.get());
        assertEquals(nodeCount, MockPS.receivedCloseErrors.size());
        for (Throwable error : MockPS.receivedCloseErrors) {
            assertTrue("unexpected exc: " + error, error instanceof CancellationException);
        }
    });
}

Example 15 with JetServiceBackend

use of com.hazelcast.jet.impl.JetServiceBackend in project hazelcast by hazelcast.

the class GracefulShutdownTest method when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot.

@Test
public void when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot() throws Exception {
    MapConfig mapConfig = new MapConfig(JobRepository.SNAPSHOT_DATA_MAP_PREFIX + "*");
    mapConfig.getMapStoreConfig().setClassName(BlockingMapStore.class.getName()).setEnabled(true);
    Config config = instances[0].getConfig();
    ((DynamicConfigurationAwareConfig) config).getStaticConfig().addMapConfig(mapConfig);
    BlockingMapStore.shouldBlock = false;
    BlockingMapStore.wasBlocked = false;
    DAG dag = new DAG();
    int numItems = 5000;
    Vertex source = dag.newVertex("source", throttle(() -> new EmitIntegersP(numItems), 500));
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(source, sink));
    source.localParallelism(1);
    Job job = instances[0].getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(2000));
    // wait for the first snapshot
    JetServiceBackend jetServiceBackend = getNode(instances[0]).nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
    JobRepository jobRepository = jetServiceBackend.getJobCoordinationService().jobRepository();
    assertJobStatusEventually(job, RUNNING);
    assertTrueEventually(() -> assertTrue(jobRepository.getJobExecutionRecord(job.getId()).dataMapIndex() >= 0));
    // When
    BlockingMapStore.shouldBlock = true;
    job.restart();
    assertTrueEventually(() -> assertTrue("blocking did not happen", BlockingMapStore.wasBlocked), 5);
    Future shutdownFuture = spawn(() -> instances[1].shutdown());
    logger.info("savedCounters=" + EmitIntegersP.savedCounters);
    int minCounter = EmitIntegersP.savedCounters.values().stream().mapToInt(Integer::intValue).min().getAsInt();
    BlockingMapStore.shouldBlock = false;
    shutdownFuture.get();
    // Then
    job.join();
    Map<Integer, Integer> actual = new ArrayList<>(instances[0].<Integer>getList("sink")).stream().collect(Collectors.toMap(Function.identity(), item -> 1, Integer::sum));
    Map<Integer, Integer> expected = IntStream.range(0, numItems).boxed().collect(Collectors.toMap(Function.identity(), item -> item < minCounter ? 2 : 1));
    assertEquals(expected, actual);
}