Examples with JobRepository com.hazelcast.jet.impl.JobRepository used on opensource projects

Example 6 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class SourceBuilderTest method test_nonFaultTolerantSource_processingGuaranteeOn.

@Test
public void test_nonFaultTolerantSource_processingGuaranteeOn() {
    StreamSource<Integer> source = SourceBuilder.stream("src", procCtx -> "foo").<Integer>fillBufferFn((ctx, buffer) -> {
        buffer.add(0);
        Thread.sleep(100);
    }).build();
    Pipeline p = Pipeline.create();
    IList<Integer> result = hz().getList("result-" + UuidUtil.newUnsecureUuidString());
    p.readFrom(source).withoutTimestamps().writeTo(Sinks.list(result));
    Job job = hz().getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(100));
    JobRepository jr = new JobRepository(hz());
    waitForFirstSnapshot(jr, job.getId(), 10, true);
    job.restart();
    assertJobStatusEventually(job, JobStatus.RUNNING);
    int currentSize = result.size();
    assertTrueEventually(() -> assertTrue(result.size() > currentSize), 5);
}

Example 7 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class OperationLossTest method when_snapshotOperationLost_then_retried.

@Test
public void when_snapshotOperationLost_then_retried() {
    PacketFiltersUtil.dropOperationsFrom(instance(), JetInitDataSerializerHook.FACTORY_ID, singletonList(JetInitDataSerializerHook.SNAPSHOT_PHASE1_OPERATION));
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", () -> new DummyStatefulP()).localParallelism(1);
    Vertex v2 = dag.newVertex("v2", mapP(identity())).localParallelism(1);
    dag.edge(between(v1, v2).distributed());
    Job job = instance().getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(100));
    assertJobStatusEventually(job, RUNNING);
    JobRepository jobRepository = new JobRepository(instance());
    assertTrueEventually(() -> {
        JobExecutionRecord record = jobRepository.getJobExecutionRecord(job.getId());
        assertNotNull("null JobExecutionRecord", record);
        assertEquals("ongoingSnapshotId", 0, record.ongoingSnapshotId());
    }, 20);
    sleepSeconds(1);
    // now lift the filter and check that a snapshot is done
    logger.info("Lifting the packet filter...");
    PacketFiltersUtil.resetPacketFiltersFrom(instance());
    waitForFirstSnapshot(jobRepository, job.getId(), 10, false);
    cancelAndJoin(job);
}

Example 8 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class JobTimeoutClusterTest method when_masterFails_timedOutJobIsCancelled.

@Test
public void when_masterFails_timedOutJobIsCancelled() {
    final HazelcastInstance[] instances = createHazelcastInstances(2);
    final HazelcastInstance oldMaster = instances[0];
    final HazelcastInstance newMaster = instances[1];
    assertClusterSizeEventually(2, newMaster);
    assertClusterStateEventually(ClusterState.ACTIVE, newMaster);
    final DAG dag = new DAG();
    dag.newVertex("stuck", () -> new MockP().streaming());
    final JobConfig jobConfig = new JobConfig().setTimeoutMillis(10000L).setSnapshotIntervalMillis(1L).setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    final Job job = oldMaster.getJet().newJob(dag, jobConfig);
    final long jobId = job.getId();
    // start and wait for the job to start running
    assertJobStatusEventually(job, JobStatus.RUNNING);
    final JobRepository oldJobRepository = new JobRepository(oldMaster);
    assertTrueEventually(() -> {
        final JobExecutionRecord record = oldJobRepository.getJobExecutionRecord(jobId);
        assertTrue(record.snapshotId() > 0);
    });
    // kill old master and wait for the cluster to reconfigure
    oldMaster.getLifecycleService().terminate();
    assertClusterStateEventually(ClusterState.ACTIVE, newMaster);
    assertClusterSize(1, newMaster);
    // wait for the job to be restarted and cancelled due to timeout
    final Job restartedJob = newMaster.getJet().getJob(jobId);
    assertNotNull(restartedJob);
    assertJobStatusEventually(restartedJob, JobStatus.FAILED);
}

Example 9 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class SplitBrainTest method when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated.

@Test
public void when_newMemberJoinsToCluster_then_jobQuorumSizeIsUpdated() {
    int clusterSize = 3;
    HazelcastInstance[] instances = new HazelcastInstance[clusterSize];
    for (int i = 0; i < clusterSize; i++) {
        instances[i] = createHazelcastInstance(createConfig());
    }
    NoOutputSourceP.executionStarted = new CountDownLatch(clusterSize * PARALLELISM);
    MockPS processorSupplier = new MockPS(NoOutputSourceP::new, clusterSize);
    DAG dag = new DAG().vertex(new Vertex("test", processorSupplier).localParallelism(PARALLELISM));
    Job job = instances[0].getJet().newJob(dag, new JobConfig().setSplitBrainProtection(true));
    assertOpenEventually(NoOutputSourceP.executionStarted);
    createHazelcastInstance(createConfig());
    assertTrueEventually(() -> {
        JetServiceBackend service = getJetServiceBackend(instances[0]);
        JobRepository jobRepository = service.getJobRepository();
        JobExecutionRecord record = jobRepository.getJobExecutionRecord(job.getId());
        assertEquals(3, record.getQuorumSize());
        MasterContext masterContext = service.getJobCoordinationService().getMasterContext(job.getId());
        assertEquals(3, masterContext.jobExecutionRecord().getQuorumSize());
    });
    NoOutputSourceP.proceedLatch.countDown();
}

Example 10 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class PostponedSnapshotTest method when_jobHasHigherPriorityEdge_then_noSnapshotUntilEdgeDone.

@Test
public void when_jobHasHigherPriorityEdge_then_noSnapshotUntilEdgeDone() {
    Job job = startJob(100);
    latches.set(0, 1);
    JobRepository jr = new JobRepository(instance);
    assertTrueEventually(() -> assertTrue(jr.getJobExecutionRecord(job.getId()).dataMapIndex() != NO_SNAPSHOT));
    // finish the job
    latches.set(1, 1);
    job.join();
}