Examples with JobConfig com.hazelcast.jet.config.JobConfig used on opensource projects

Example 91 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class WatermarkCoalescer_IntegrationTest method when_i1_activeNoWm_i2_aheadAndIdle_then_wmForwardedAfterADelay.

@Test
public void when_i1_activeNoWm_i2_aheadAndIdle_then_wmForwardedAfterADelay() {
    dag = createDag(mode, singletonList(entry(1, 1)), asList(wm(150), IDLE_MESSAGE));
    JobConfig config = new JobConfig().setMaxWatermarkRetainMillis(5000);
    instance.newJob(dag, config);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 3);
    assertEquals(entry(1, 1), sinkList.get(0));
    long time = System.nanoTime();
    assertTrueEventually(() -> assertEquals(2, sinkList.size()), 6);
    assertEquals("wm(150)", sinkList.get(1));
    long elapsedMs = NANOSECONDS.toMillis(System.nanoTime() - time);
    assertTrue("Too little elapsed time, WM probably emitted immediately", elapsedMs > 3000);
}

Example 92 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class StreamUtil method executeJob.

public static void executeJob(StreamContext context, DAG dag) {
    JobConfig jobConfig = context.getJobConfig() != null ? context.getJobConfig() : new JobConfig();
    Job job = context.getJetInstance().newJob(dag, jobConfig);
    try {
        job.getFuture().get();
    } catch (InterruptedException | ExecutionException e) {
        throw rethrow(e);
    }
    context.getStreamListeners().forEach(Runnable::run);
}

Example 93 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class JobRepository method completeJob.

/**
 * Puts a JobResult for the given job and deletes the JobRecord.
 * @throws JobNotFoundException if the JobRecord is not found
 * @throws IllegalStateException if the JobResult is already present
 */
void completeJob(long jobId, String coordinator, long completionTime, Throwable error) {
    JobRecord jobRecord = getJobRecord(jobId);
    if (jobRecord == null) {
        throw new JobNotFoundException(jobId);
    }
    JobConfig config = jobRecord.getConfig();
    long creationTime = jobRecord.getCreationTime();
    JobResult jobResult = new JobResult(jobId, config, coordinator, creationTime, completionTime, error);
    JobResult prev = jobResults.putIfAbsent(jobId, jobResult);
    if (prev != null) {
        throw new IllegalStateException("Job result already exists in the " + jobResults.getName() + " map:\n" + "previous record: " + prev + "\n" + "new record: " + jobResult);
    }
    deleteJob(jobId);
}

Example 94 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class JobRestartWithSnapshotTest method when_snapshotStartedBeforeExecution_then_firstSnapshotIsSuccessful.

@Test
public void when_snapshotStartedBeforeExecution_then_firstSnapshotIsSuccessful() throws Exception {
    // instance1 is always coordinator
    // delay ExecuteOperation so that snapshot is started before execution is started on the worker member
    delayOperationsFrom(hz(instance1), JetInitDataSerializerHook.FACTORY_ID, singletonList(JetInitDataSerializerHook.START_EXECUTION_OP));
    DAG dag = new DAG();
    dag.newVertex("p", FirstSnapshotProcessor::new).localParallelism(1);
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    config.setSnapshotIntervalMillis(0);
    Job job = instance1.newJob(dag, config);
    // the first snapshot should succeed
    assertTrueEventually(() -> {
        IMapJet<Long, SnapshotRecord> records = getSnapshotsMap(job);
        SnapshotRecord record = records.get(0L);
        assertNotNull("no record found for snapshot 0", record);
        assertTrue("snapshot was not successful", record.isSuccessful());
    }, 30);
}

Example 95 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast-jet by hazelcast.

the class JobRestartWithSnapshotTest method when_snapshotDoneBeforeStarted_then_snapshotSuccessful.

@Test
public void when_snapshotDoneBeforeStarted_then_snapshotSuccessful() throws Exception {
    /*
        Design of this test

        The DAG is "source -> sink". Source completes immediately on  non-coordinator (worker)
        and is infinite on coordinator. Edge between source and sink is distributed. This
        situation will cause that after the source completes on member, the sink on worker
        will only have the remote source. This will allow that we can receive the barrier
        from remote coordinator before worker even starts the snapshot. This is the very
        purpose of this test. To ensure that this happens, we postpone handling of SnapshotOperation
        on the worker.
        */
    // instance1 is always coordinator
    delayOperationsFrom(hz(instance1), JetInitDataSerializerHook.FACTORY_ID, singletonList(JetInitDataSerializerHook.SNAPSHOT_OP));
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", new NonBalancedSource(getAddress(instance2).toString()));
    Vertex sink = dag.newVertex("sink", writeListP("sink"));
    dag.edge(between(source, sink).distributed());
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    config.setSnapshotIntervalMillis(500);
    Job job = instance1.newJob(dag, config);
    assertTrueEventually(() -> assertNotNull(getSnapshotContext(job)));
    SnapshotContext ssContext = getSnapshotContext(job);
    assertTrueEventually(() -> assertTrue("numRemainingTasklets was not negative, the tested scenario did not happen", ssContext.getNumRemainingTasklets().get() < 0), 3);
}