Examples with JobRepository com.hazelcast.jet.impl.JobRepository used on opensource projects

Example 21 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class SuspendResumeTest method when_cancelSuspendedJob_then_jobCancels.

@Test
public void when_cancelSuspendedJob_then_jobCancels() throws Exception {
    Job job = instances[0].getJet().newJob(dag);
    NoOutputSourceP.executionStarted.await();
    job.suspend();
    assertJobStatusEventually(job, SUSPENDED);
    // When-Then
    cancelAndJoin(job);
    assertJobStatusEventually(job, FAILED);
    // check that job resources are deleted
    JobRepository jobRepository = new JobRepository(instances[0]);
    assertTrueEventually(() -> {
        assertNull("JobRecord", jobRepository.getJobRecord(job.getId()));
        JobResult jobResult = jobRepository.getJobResult(job.getId());
        assertContains(jobResult.getFailureText(), CancellationException.class.getName());
        assertFalse("Job result successful", jobResult.isSuccessful());
    });
}

Example 22 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class SourceBuilder_TopologyChangeTest method testTopologyChange.

private void testTopologyChange(Supplier<HazelcastInstance> secondMemberSupplier, Consumer<HazelcastInstance> changeTopologyFn, boolean assertMonotonicity) {
    stateRestored = false;
    StreamSource<Integer> source = SourceBuilder.timestampedStream("src", ctx -> new NumberGeneratorContext()).<Integer>fillBufferFn((src, buffer) -> {
        long expectedCount = NANOSECONDS.toMillis(System.nanoTime() - src.startTime);
        expectedCount = Math.min(expectedCount, src.current + 100);
        while (src.current < expectedCount) {
            buffer.add(src.current, src.current);
            src.current++;
        }
    }).createSnapshotFn(src -> {
        System.out.println("Will save " + src.current + " to snapshot");
        return src;
    }).restoreSnapshotFn((src, states) -> {
        stateRestored = true;
        assert states.size() == 1;
        src.restore(states.get(0));
        System.out.println("Restored " + src.current + " from snapshot");
    }).build();
    Config config = smallInstanceConfig();
    // restart sooner after member add
    config.getJetConfig().setScaleUpDelayMillis(1000);
    HazelcastInstance hz = createHazelcastInstance(config);
    HazelcastInstance possibleSecondNode = secondMemberSupplier.get();
    long windowSize = 100;
    IList<WindowResult<Long>> result = hz.getList("result-" + UuidUtil.newUnsecureUuidString());
    Pipeline p = Pipeline.create();
    p.readFrom(source).withNativeTimestamps(0).window(tumbling(windowSize)).aggregate(AggregateOperations.counting()).peek().writeTo(Sinks.list(result));
    Job job = hz.getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(500));
    assertTrueEventually(() -> assertFalse("result list is still empty", result.isEmpty()));
    assertJobStatusEventually(job, JobStatus.RUNNING);
    JobRepository jr = new JobRepository(hz);
    waitForFirstSnapshot(jr, job.getId(), 10, false);
    assertFalse(stateRestored);
    changeTopologyFn.accept(possibleSecondNode);
    assertTrueEventually(() -> assertTrue("restoreSnapshotFn was not called", stateRestored));
    // wait until more results are added
    int oldSize = result.size();
    assertTrueEventually(() -> assertTrue("no more results added to the list", result.size() > oldSize));
    cancelAndJoin(job);
    // results should contain sequence of results, each with count=windowSize, monotonic, if job was
    // allowed to terminate gracefully
    Iterator<WindowResult<Long>> iterator = result.iterator();
    for (int i = 0; i < result.size(); i++) {
        WindowResult<Long> next = iterator.next();
        assertEquals(windowSize, (long) next.result());
        if (assertMonotonicity) {
            assertEquals(i * windowSize, next.start());
        }
    }
}

Example 23 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class JobRestartStressTestBase method stressTest.

// has sub-classes in jet-enterprise
@SuppressWarnings("WeakerAccess")
protected void stressTest(Function<Tuple3<HazelcastInstance, DAG, Job>, Job> action) throws Exception {
    JobRepository jobRepository = new JobRepository(instance1);
    TestProcessors.reset(2);
    DAG dag = new DAG();
    dag.newVertex("dummy-stateful-p", DummyStatefulP::new).localParallelism(1);
    Job[] job = { instance1.getJet().newJob(dag, new JobConfig().setSnapshotIntervalMillis(10).setProcessingGuarantee(EXACTLY_ONCE)) };
    logger.info("waiting for 1st snapshot");
    waitForFirstSnapshot(jobRepository, job[0].getId(), 5, false);
    logger.info("first snapshot found");
    for (int i = 0; i < 10; i++) {
        job[0] = action.apply(tuple3(instance1, dag, job[0]));
        waitForNextSnapshot(jobRepository, job[0].getId(), 5, false);
    }
    cancelAndJoin(job[0]);
}

Example 24 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class JmsSourceIntegrationTestBase method when_exactlyOnceTopicDefaultConsumer_then_noGuaranteeUsed.

@Test
public void when_exactlyOnceTopicDefaultConsumer_then_noGuaranteeUsed() {
    SupplierEx<ConnectionFactory> mockSupplier = () -> {
        ConnectionFactory mockConnectionFactory = mock(ConnectionFactory.class);
        Connection mockConn = mock(Connection.class);
        Session mockSession = mock(Session.class);
        MessageConsumer mockConsumer = mock(MessageConsumer.class);
        when(mockConnectionFactory.createConnection()).thenReturn(mockConn);
        when(mockConn.createSession(anyBoolean(), anyInt())).thenReturn(mockSession);
        when(mockSession.createConsumer(any())).thenReturn(mockConsumer);
        // throw, if commit is called
        doThrow(new AssertionError("commit must not be called")).when(mockSession).commit();
        return (ConnectionFactory) mockConnectionFactory;
    };
    p.readFrom(Sources.jmsTopic(destinationName, mockSupplier)).withoutTimestamps().writeTo(Sinks.logger());
    Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(10));
    assertJobStatusEventually(job, RUNNING);
    JobRepository jr = new JobRepository(instance());
    waitForFirstSnapshot(jr, job.getId(), 5, true);
    assertTrueAllTheTime(() -> assertEquals(RUNNING, job.getStatus()), 1);
}

Example 25 with JobRepository

use of com.hazelcast.jet.impl.JobRepository in project hazelcast by hazelcast.

the class WriteFilePTest method stressTest.

private void stressTest(boolean graceful, boolean exactlyOnce) throws Exception {
    int numItems = 500;
    Pipeline p = Pipeline.create();
    p.readFrom(SourceBuilder.stream("src", procCtx -> tuple2(new int[1], procCtx.logger())).fillBufferFn((ctx, buf) -> {
        if (ctx.f0()[0] < numItems) {
            buf.add(ctx.f0()[0]++);
            sleepMillis(5);
        }
    }).createSnapshotFn(ctx -> {
        ctx.f1().fine("src vertex saved to snapshot: " + ctx.f0()[0]);
        return ctx.f0()[0];
    }).restoreSnapshotFn((ctx, state) -> {
        ctx.f0()[0] = state.get(0);
        ctx.f1().fine("src vertex restored from snapshot: " + ctx.f0()[0]);
    }).build()).withoutTimestamps().writeTo(Sinks.filesBuilder(directory.toString()).exactlyOnce(exactlyOnce).build()).setLocalParallelism(2);
    JobConfig config = new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(50);
    JobProxy job = (JobProxy) instance().getJet().newJob(p, config);
    long endTime = System.nanoTime() + SECONDS.toNanos(60);
    do {
        assertJobStatusEventually(job, RUNNING);
        sleepMillis(100);
        job.restart(graceful);
        try {
            checkFileContents(0, numItems, exactlyOnce, true, false);
            // if content matches, break the loop. Otherwise restart and try again
            break;
        } catch (AssertionError ignored) {
        }
    } while (System.nanoTime() < endTime);
    waitForNextSnapshot(new JobRepository(instance()), job.getId(), 10, true);
    ditchJob(job, instances());
    // when the job is cancelled, there should be no temporary files
    checkFileContents(0, numItems, exactlyOnce, false, false);
}