Examples with StreamSource com.hazelcast.jet.pipeline.StreamSource used on opensource projects

Example 11 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class JobExecutionMetricsTest method snapshotPipeline.

private Pipeline snapshotPipeline() {
    Pipeline p = Pipeline.create();
    StreamSource<Long> source = SourceBuilder.stream("src", procCtx -> new long[1]).<Long>fillBufferFn((ctx, buf) -> {
        buf.add(ctx[0]++);
        Thread.sleep(5);
    }).createSnapshotFn(ctx -> ctx[0]).restoreSnapshotFn((ctx, state) -> ctx[0] = state.get(0)).build();
    p.readFrom(source).withoutTimestamps().writeTo(Sinks.logger());
    return p;
}

Example 12 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class JobSnapshotMetricsTest method pipeline.

private Pipeline pipeline() {
    Pipeline p = Pipeline.create();
    StreamSource<Long> source = SourceBuilder.stream(SOURCE_VERTEX_NAME, procCtx -> new long[1]).<Long>fillBufferFn((ctx, buf) -> {
        buf.add(ctx[0]++);
        Thread.sleep(5);
    }).createSnapshotFn(ctx -> ctx[0]).restoreSnapshotFn((ctx, state) -> ctx[0] = state.get(0)).build();
    p.readFrom(source).withoutTimestamps().filter(t -> t % 2 == 0).writeTo(Sinks.noop());
    return p;
}

Example 13 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class SuspendExecutionOnFailureTest method when_jobSuspendedDueToFailure_then_canBeResumed.

@Test
public void when_jobSuspendedDueToFailure_then_canBeResumed() {
    int numItems = 100;
    int interruptItem = 50;
    StreamSource<Integer> source = SourceBuilder.stream("src", procCtx -> new int[1]).<Integer>fillBufferFn((ctx, buf) -> {
        if (ctx[0] < numItems) {
            buf.add(ctx[0]++);
            sleepMillis(5);
        }
    }).createSnapshotFn(ctx -> ctx[0]).restoreSnapshotFn((ctx, state) -> ctx[0] = state.get(0)).build();
    Pipeline p = Pipeline.create();
    p.readFrom(source).withoutTimestamps().<String, Boolean, Map.Entry<Integer, Integer>>mapUsingIMap("SuspendExecutionOnFailureTest_failureMap", item -> "key", (item, value) -> {
        if (value && item == interruptItem) {
            throw new RuntimeException("Fail deliberately");
        }
        return entry(item, item);
    }).setLocalParallelism(1).writeTo(Sinks.map("SuspendExecutionOnFailureTest_sinkMap"));
    IMap<String, Boolean> counterMap = hz().getMap("SuspendExecutionOnFailureTest_failureMap");
    counterMap.put("key", true);
    jobConfig.setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE).setSnapshotIntervalMillis(50);
    Job job = hz().getJet().newJob(p, jobConfig);
    assertJobStatusEventually(job, SUSPENDED);
    IMap<Integer, Integer> sinkMap = hz().getMap("SuspendExecutionOnFailureTest_sinkMap");
    counterMap.put("key", false);
    job.resume();
    // sinkMap is an idempotent sink, so even though we're using at-least-once, no key will be duplicated, so it
    // must contain the expected number of items.
    assertTrueEventually(() -> assertEquals(numItems, sinkMap.size()));
    assertTrueEventually(() -> assertEquals(JobStatus.RUNNING, job.getStatus()));
    cancelAndJoin(job);
}

Example 14 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class JmsSourceIntegrationTestBase method when_messageIdFn_then_used.

@Test
public void when_messageIdFn_then_used() throws JMSException {
    StreamSource<String> source = Sources.jmsQueueBuilder(getConnectionFactory()).destinationName(destinationName).messageIdFn(m -> {
        throw new RuntimeException("mock exception");
    }).build(TEXT_MESSAGE_FN);
    p.readFrom(source).withoutTimestamps().writeTo(Sinks.logger());
    Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE));
    sendMessages(true);
    try {
        job.join();
        fail("job didn't fail");
    } catch (Exception e) {
        assertContains(e.toString(), "mock exception");
    }
}