use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.
the class GracefulShutdownTest method when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot.
@Test
public void when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot() throws Exception {
MapConfig mapConfig = new MapConfig(JobRepository.SNAPSHOT_DATA_MAP_PREFIX + "*");
mapConfig.getMapStoreConfig().setClassName(BlockingMapStore.class.getName()).setEnabled(true);
Config config = instances[0].getConfig();
((DynamicConfigurationAwareConfig) config).getStaticConfig().addMapConfig(mapConfig);
BlockingMapStore.shouldBlock = false;
BlockingMapStore.wasBlocked = false;
DAG dag = new DAG();
int numItems = 5000;
Vertex source = dag.newVertex("source", throttle(() -> new EmitIntegersP(numItems), 500));
Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
dag.edge(between(source, sink));
source.localParallelism(1);
Job job = instances[0].getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(2000));
// wait for the first snapshot
JetServiceBackend jetServiceBackend = getNode(instances[0]).nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
JobRepository jobRepository = jetServiceBackend.getJobCoordinationService().jobRepository();
assertJobStatusEventually(job, RUNNING);
assertTrueEventually(() -> assertTrue(jobRepository.getJobExecutionRecord(job.getId()).dataMapIndex() >= 0));
// When
BlockingMapStore.shouldBlock = true;
job.restart();
assertTrueEventually(() -> assertTrue("blocking did not happen", BlockingMapStore.wasBlocked), 5);
Future shutdownFuture = spawn(() -> instances[1].shutdown());
logger.info("savedCounters=" + EmitIntegersP.savedCounters);
int minCounter = EmitIntegersP.savedCounters.values().stream().mapToInt(Integer::intValue).min().getAsInt();
BlockingMapStore.shouldBlock = false;
shutdownFuture.get();
// Then
job.join();
Map<Integer, Integer> actual = new ArrayList<>(instances[0].<Integer>getList("sink")).stream().collect(Collectors.toMap(Function.identity(), item -> 1, Integer::sum));
Map<Integer, Integer> expected = IntStream.range(0, numItems).boxed().collect(Collectors.toMap(Function.identity(), item -> item < minCounter ? 2 : 1));
assertEquals(expected, actual);
}
use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.
the class JobRestartWithSnapshotTest method when_nodeDown_then_jobRestartsFromSnapshot.
@SuppressWarnings("unchecked")
private void when_nodeDown_then_jobRestartsFromSnapshot(boolean twoStage) throws Exception {
/*
Design of this test:
It uses a random partitioned generator of source events. The events are
Map.Entry(partitionId, timestamp). For each partition timestamps from
0..elementsInPartition are generated.
We start the test with two nodes and localParallelism(1) and 3 partitions
for source. Source instances generate items at the same rate of 10 per
second: this causes one instance to be twice as fast as the other in terms of
timestamp. The source processor saves partition offsets similarly to how
KafkaSources.kafka() and Sources.mapJournal() do.
After some time we shut down one instance. The job restarts from the
snapshot and all partitions are restored to single source processor
instance. Partition offsets are very different, so the source is written
in a way that it emits from the most-behind partition in order to not
emit late events from more ahead partitions.
Local parallelism of InsertWatermarkP is also 1 to avoid the edge case
when different instances of InsertWatermarkP might initialize with first
event in different frame and make them start the no-gap emission from
different WM, which might cause the SlidingWindowP downstream to miss
some of the first windows.
The sink writes to an IMap which is an idempotent sink.
The resulting contents of the sink map are compared to expected value.
*/
DAG dag = new DAG();
SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(3);
AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
IMap<List<Long>, Long> result = instance1.getMap("result");
result.clear();
int numPartitions = 3;
int elementsInPartition = 250;
SupplierEx<Processor> sup = () -> new SequencesInPartitionsGeneratorP(numPartitions, elementsInPartition, true);
Vertex generator = dag.newVertex("generator", throttle(sup, 30)).localParallelism(1);
Vertex insWm = dag.newVertex("insWm", insertWatermarksP(eventTimePolicy(o -> ((Entry<Integer, Integer>) o).getValue(), limitingLag(0), wDef.frameSize(), wDef.frameOffset(), 0))).localParallelism(1);
Vertex map = dag.newVertex("map", mapP((KeyedWindowResult kwr) -> entry(asList(kwr.end(), (long) (int) kwr.key()), kwr.result())));
Vertex writeMap = dag.newVertex("writeMap", SinkProcessors.writeMapP("result"));
if (twoStage) {
Vertex aggregateStage1 = dag.newVertex("aggregateStage1", Processors.accumulateByFrameP(singletonList((FunctionEx<? super Object, ?>) t -> ((Entry<Integer, Integer>) t).getKey()), singletonList(t1 -> ((Entry<Integer, Integer>) t1).getValue()), TimestampKind.EVENT, wDef, aggrOp.withIdentityFinish()));
Vertex aggregateStage2 = dag.newVertex("aggregateStage2", combineToSlidingWindowP(wDef, aggrOp, KeyedWindowResult::new));
dag.edge(between(insWm, aggregateStage1).partitioned(entryKey())).edge(between(aggregateStage1, aggregateStage2).distributed().partitioned(entryKey())).edge(between(aggregateStage2, map));
} else {
Vertex aggregate = dag.newVertex("aggregate", Processors.aggregateToSlidingWindowP(singletonList((FunctionEx<Object, Integer>) t -> ((Entry<Integer, Integer>) t).getKey()), singletonList(t1 -> ((Entry<Integer, Integer>) t1).getValue()), TimestampKind.EVENT, wDef, 0L, aggrOp, KeyedWindowResult::new));
dag.edge(between(insWm, aggregate).distributed().partitioned(entryKey())).edge(between(aggregate, map));
}
dag.edge(between(generator, insWm)).edge(between(map, writeMap));
JobConfig config = new JobConfig();
config.setProcessingGuarantee(EXACTLY_ONCE);
config.setSnapshotIntervalMillis(1200);
Job job = instance1.getJet().newJob(dag, config);
JobRepository jobRepository = new JobRepository(instance1);
int timeout = (int) (MILLISECONDS.toSeconds(config.getSnapshotIntervalMillis() * 3) + 8);
waitForFirstSnapshot(jobRepository, job.getId(), timeout, false);
waitForNextSnapshot(jobRepository, job.getId(), timeout, false);
// wait a little more to emit something, so that it will be overwritten in the sink map
Thread.sleep(300);
instance2.getLifecycleService().terminate();
// Now the job should detect member shutdown and restart from snapshot.
// Let's wait until the next snapshot appears.
waitForNextSnapshot(jobRepository, job.getId(), (int) (MILLISECONDS.toSeconds(config.getSnapshotIntervalMillis()) + 10), false);
waitForNextSnapshot(jobRepository, job.getId(), timeout, false);
job.join();
// compute expected result
Map<List<Long>, Long> expectedMap = new HashMap<>();
for (long partition = 0; partition < numPartitions; partition++) {
long cnt = 0;
for (long value = 1; value <= elementsInPartition; value++) {
cnt++;
if (value % wDef.frameSize() == 0) {
expectedMap.put(asList(value, partition), cnt);
cnt = 0;
}
}
if (cnt > 0) {
expectedMap.put(asList(wDef.higherFrameTs(elementsInPartition - 1), partition), cnt);
}
}
// check expected result
if (!expectedMap.equals(result)) {
System.out.println("All expected entries: " + expectedMap.entrySet().stream().map(Object::toString).collect(joining(", ")));
System.out.println("All actual entries: " + result.entrySet().stream().map(Object::toString).collect(joining(", ")));
System.out.println("Non-received expected items: " + expectedMap.keySet().stream().filter(key -> !result.containsKey(key)).map(Object::toString).collect(joining(", ")));
System.out.println("Received non-expected items: " + result.entrySet().stream().filter(entry -> !expectedMap.containsKey(entry.getKey())).map(Object::toString).collect(joining(", ")));
System.out.println("Different keys: ");
for (Entry<List<Long>, Long> rEntry : result.entrySet()) {
Long expectedValue = expectedMap.get(rEntry.getKey());
if (expectedValue != null && !expectedValue.equals(rEntry.getValue())) {
System.out.println("key: " + rEntry.getKey() + ", expected value: " + expectedValue + ", actual value: " + rEntry.getValue());
}
}
System.out.println("-- end of different keys");
assertEquals(expectedMap, new HashMap<>(result));
}
}
use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.
the class JobRestartWithSnapshotTest method when_jobRestartedGracefully_then_noOutputDuplicated.
@Test
public void when_jobRestartedGracefully_then_noOutputDuplicated() {
DAG dag = new DAG();
int elementsInPartition = 100;
SupplierEx<Processor> sup = () -> new SequencesInPartitionsGeneratorP(3, elementsInPartition, true);
Vertex generator = dag.newVertex("generator", throttle(sup, 30)).localParallelism(1);
Vertex sink = dag.newVertex("sink", writeListP("sink"));
dag.edge(between(generator, sink));
JobConfig config = new JobConfig();
config.setProcessingGuarantee(EXACTLY_ONCE);
// set long interval so that the first snapshot does not execute
config.setSnapshotIntervalMillis(3600_000);
Job job = instance1.getJet().newJob(dag, config);
// wait for the job to start producing output
List<Entry<Integer, Integer>> sinkList = instance1.getList("sink");
assertTrueEventually(() -> assertTrue(sinkList.size() > 10));
// When
job.restart();
job.join();
// Then
Set<Entry<Integer, Integer>> expected = IntStream.range(0, elementsInPartition).boxed().flatMap(i -> IntStream.range(0, 3).mapToObj(p -> entry(p, i))).collect(Collectors.toSet());
assertEquals(expected, new HashSet<>(sinkList));
}
use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.
the class ManualRestartTest method when_terminalSnapshotFails_then_previousSnapshotUsed.
@Test
public void when_terminalSnapshotFails_then_previousSnapshotUsed() {
MapConfig mapConfig = new MapConfig(JobRepository.SNAPSHOT_DATA_MAP_PREFIX + "*");
mapConfig.getMapStoreConfig().setClassName(FailingMapStore.class.getName()).setEnabled(true);
Config config = instances[0].getConfig();
((DynamicConfigurationAwareConfig) config).getStaticConfig().addMapConfig(mapConfig);
FailingMapStore.fail = false;
FailingMapStore.failed = false;
DAG dag = new DAG();
Vertex source = dag.newVertex("source", throttle(() -> new SequencesInPartitionsGeneratorP(2, 10000, true), 1000));
Vertex sink = dag.newVertex("sink", writeListP("sink"));
dag.edge(between(source, sink));
source.localParallelism(1);
Job job = instances[0].getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(2000));
// wait for the first snapshot
JetServiceBackend jetServiceBackend = getNode(instances[0]).nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
JobRepository jobRepository = jetServiceBackend.getJobCoordinationService().jobRepository();
assertJobStatusEventually(job, RUNNING);
assertTrueEventually(() -> assertTrue(jobRepository.getJobExecutionRecord(job.getId()).dataMapIndex() >= 0));
// When
sleepMillis(100);
FailingMapStore.fail = true;
job.restart();
assertTrueEventually(() -> assertTrue(FailingMapStore.failed));
FailingMapStore.fail = false;
job.join();
Map<Integer, Integer> actual = new ArrayList<>(instances[0].<Entry<Integer, Integer>>getList("sink")).stream().filter(// we'll only check partition 0
e -> e.getKey() == 0).map(Entry::getValue).collect(Collectors.toMap(e -> e, e -> 1, (o, n) -> o + n, TreeMap::new));
assertEquals("first item != 1, " + actual.toString(), (Integer) 1, actual.get(0));
assertEquals("last item != 1, " + actual.toString(), (Integer) 1, actual.get(9999));
// the result should be some ones, then some twos and then some ones. The twos should be during the time
// since the last successful snapshot until the actual termination, when there was reprocessing.
boolean sawTwo = false;
boolean sawOneAgain = false;
for (Integer v : actual.values()) {
if (v == 1) {
if (sawTwo) {
sawOneAgain = true;
}
} else if (v == 2) {
assertFalse("got a 2 in another group", sawOneAgain);
sawTwo = true;
} else {
fail("v=" + v);
}
}
assertTrue("didn't see any 2s", sawTwo);
}
use of com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE in project hazelcast by hazelcast.
the class StreamKafkaPTest method when_snapshotSaved_then_offsetsRestored.
@Test
public void when_snapshotSaved_then_offsetsRestored() throws Exception {
StreamKafkaP processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
kafkaTestSupport.produce(topic1Name, 0, "0");
assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
// create snapshot
TestInbox snapshot = saveSnapshot(processor, outbox);
Set<Entry<Object, Object>> snapshotItems = unwrapBroadcastKey(snapshot.queue());
// consume one more item
kafkaTestSupport.produce(topic1Name, 1, "1");
assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
// create new processor and restore snapshot
processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
outbox = new TestOutbox(new int[] { 10 }, 10);
processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
// restore snapshot
processor.restoreFromSnapshot(snapshot);
assertTrue("snapshot not fully processed", snapshot.isEmpty());
TestInbox snapshot2 = saveSnapshot(processor, outbox);
assertEquals("new snapshot not equal after restore", snapshotItems, unwrapBroadcastKey(snapshot2.queue()));
// the second item should be produced one more time
assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
assertNoMoreItems(processor, outbox);
}
Aggregations