use of com.hazelcast.jet.impl.JobProxy in project hazelcast by hazelcast.
the class KinesisIntegrationTest method restart_staticStream.
private void restart_staticStream(boolean graceful) {
HELPER.createStream(3);
JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE).setSnapshotIntervalMillis(SECONDS.toMillis(1));
Job job = hz().getJet().newJob(getPipeline(kinesisSource().build()), jobConfig);
Map<String, List<String>> expectedMessages = sendMessages();
// wait for some data to start coming out of the pipeline
assertTrueEventually(() -> assertFalse(results.isEmpty()));
((JobProxy) job).restart(graceful);
assertMessages(expectedMessages, true, !graceful);
}
use of com.hazelcast.jet.impl.JobProxy in project hazelcast by hazelcast.
the class KinesisIntegrationTest method restart_dynamicStream.
private void restart_dynamicStream(boolean graceful) {
HELPER.createStream(3);
JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE).setSnapshotIntervalMillis(SECONDS.toMillis(1));
Job job = hz().getJet().newJob(getPipeline(kinesisSource().build()), jobConfig);
Map<String, List<String>> expectedMessages = sendMessages();
// wait for some data to start coming out of the pipeline
assertTrueEventually(() -> assertFalse(results.isEmpty()));
List<Shard> openShards = listOpenShards();
Shard shard1 = openShards.get(0);
Shard shard2 = openShards.get(1);
Shard shard3 = openShards.get(2);
splitShard(shard1);
HELPER.waitForStreamToActivate();
assertOpenShards(4, shard1);
mergeShards(shard2, shard3);
HELPER.waitForStreamToActivate();
assertOpenShards(3, shard2, shard3);
((JobProxy) job).restart(graceful);
assertMessages(expectedMessages, false, !graceful);
}
use of com.hazelcast.jet.impl.JobProxy in project hazelcast by hazelcast.
the class JmsSourceIntegrationTestBase method stressTest.
private void stressTest(boolean graceful, ProcessingGuarantee maxGuarantee, boolean useTopic) throws Exception {
lastListInStressTest = null;
final int MESSAGE_COUNT = 4_000;
Pipeline p = Pipeline.create();
String destName = "queue-" + counter++;
JmsSourceBuilder sourceBuilder;
if (useTopic) {
sourceBuilder = Sources.jmsTopicBuilder(getConnectionFactory()).sharedConsumer(true).consumerFn(s -> s.createSharedDurableConsumer(s.createTopic(destName), "foo-consumer"));
// create the durable subscriber now so that it doesn't lose the initial messages
try (Connection conn = getConnectionFactory().get().createConnection()) {
conn.setClientID("foo-client-id");
try (Session sess = conn.createSession(false, DUPS_OK_ACKNOWLEDGE)) {
sess.createDurableSubscriber(sess.createTopic(destName), "foo-consumer");
}
}
} else {
sourceBuilder = Sources.jmsQueueBuilder(getConnectionFactory()).destinationName(destName);
}
p.readFrom(sourceBuilder.maxGuarantee(maxGuarantee).build(msg -> Long.parseLong(((TextMessage) msg).getText()))).withoutTimestamps().peek().mapStateful(CopyOnWriteArrayList<Long>::new, (list, item) -> {
lastListInStressTest = list;
list.add(item);
return null;
}).writeTo(Sinks.logger());
Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE).setSnapshotIntervalMillis(50));
assertJobStatusEventually(job, RUNNING);
// start a producer that will produce MESSAGE_COUNT messages on the background to the queue, 1000 msgs/s
@SuppressWarnings("rawtypes") Future producerFuture = spawn(() -> {
try (Connection connection = getConnectionFactory().get().createConnection();
Session session = connection.createSession(false, AUTO_ACKNOWLEDGE);
MessageProducer producer = session.createProducer(useTopic ? session.createTopic(destName) : session.createQueue(destName))) {
long startTime = System.nanoTime();
for (int i = 0; i < MESSAGE_COUNT; i++) {
producer.send(session.createTextMessage(String.valueOf(i)));
Thread.sleep(Math.max(0, i - NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}
} catch (Exception e) {
throw sneakyThrow(e);
}
});
int iteration = 0;
JobRepository jr = new JobRepository(instance());
waitForFirstSnapshot(jr, job.getId(), 20, true);
while (!producerFuture.isDone()) {
Thread.sleep(ThreadLocalRandom.current().nextInt(200));
// We also do it before the first restart to workaround https://issues.apache.org/jira/browse/ARTEMIS-2546
if (iteration++ % 3 == 0) {
waitForNextSnapshot(jr, job.getId(), 20, true);
}
((JobProxy) job).restart(graceful);
assertJobStatusEventually(job, RUNNING);
}
// call for the side-effect of throwing if the producer failed
producerFuture.get();
assertTrueEventually(() -> {
Map<Long, Long> counts = lastListInStressTest.stream().collect(Collectors.groupingBy(Function.identity(), TreeMap::new, Collectors.counting()));
for (long i = 0; i < MESSAGE_COUNT; i++) {
counts.putIfAbsent(i, 0L);
}
String countsStr = "counts: " + counts;
if (maxGuarantee == NONE) {
// we don't assert anything and only wait little more and check that the job didn't fail
sleepSeconds(1);
} else {
// in EXACTLY_ONCE the list must have each item exactly once
// in AT_LEAST_ONCE the list must have each item at least once
assertTrue(countsStr, counts.values().stream().allMatch(cnt -> maxGuarantee == EXACTLY_ONCE ? cnt == 1 : cnt > 0));
}
logger.info(countsStr);
}, 30);
assertEquals(job.getStatus(), RUNNING);
}
use of com.hazelcast.jet.impl.JobProxy in project hazelcast by hazelcast.
the class SinkStressTestUtil method test_withRestarts.
public static void test_withRestarts(@Nonnull HazelcastInstance instance, @Nonnull ILogger logger, @Nonnull Sink<Integer> sink, boolean graceful, boolean exactlyOnce, @Nonnull SupplierEx<List<Integer>> actualItemsSupplier) {
int numItems = 1000;
Pipeline p = Pipeline.create();
p.readFrom(SourceBuilder.stream("src", procCtx -> new int[] { procCtx.globalProcessorIndex() == 0 ? 0 : Integer.MAX_VALUE }).<Integer>fillBufferFn((ctx, buf) -> {
if (ctx[0] < numItems) {
buf.add(ctx[0]++);
sleepMillis(5);
}
}).distributed(1).createSnapshotFn(ctx -> ctx[0] < Integer.MAX_VALUE ? ctx[0] : null).restoreSnapshotFn((ctx, state) -> ctx[0] = ctx[0] != Integer.MAX_VALUE ? state.get(0) : Integer.MAX_VALUE).build()).withoutTimestamps().peek().writeTo(sink);
JobConfig config = new JobConfig().setProcessingGuarantee(exactlyOnce ? EXACTLY_ONCE : AT_LEAST_ONCE).setSnapshotIntervalMillis(50);
JobProxy job = (JobProxy) instance.getJet().newJob(p, config);
long endTime = System.nanoTime() + SECONDS.toNanos(TEST_TIMEOUT_SECONDS);
int lastCount = 0;
String expectedRows = IntStream.range(0, numItems).mapToObj(i -> i + (exactlyOnce ? "=1" : "")).collect(joining("\n"));
// We'll restart once, then restart again after a short sleep (possibly during initialization),
// and then assert some output so that the test isn't constantly restarting without any progress
Long lastExecutionId = null;
for (; ; ) {
lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
job.restart(graceful);
lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
sleepMillis(ThreadLocalRandom.current().nextInt(400));
job.restart(graceful);
try {
List<Integer> actualItems;
Set<Integer> distinctActualItems;
do {
actualItems = actualItemsSupplier.get();
distinctActualItems = new HashSet<>(actualItems);
} while (distinctActualItems.size() < Math.min(numItems, 100 + lastCount) && System.nanoTime() < endTime);
lastCount = distinctActualItems.size();
logger.info("number of committed items in the sink so far: " + lastCount);
if (exactlyOnce) {
String actualItemsStr = actualItems.stream().collect(groupingBy(identity(), TreeMap::new, counting())).entrySet().stream().map(Object::toString).collect(joining("\n"));
assertEquals(expectedRows, actualItemsStr);
} else {
assertEquals(expectedRows, distinctActualItems.stream().map(Objects::toString).collect(joining("\n")));
}
// if content matches, break the loop. Otherwise restart and try again
break;
} catch (AssertionError e) {
if (System.nanoTime() >= endTime) {
throw e;
}
}
}
}
use of com.hazelcast.jet.impl.JobProxy in project hazelcast by hazelcast.
the class PostgresCdcIntegrationTest method stressTest_exactlyOnce.
public void stressTest_exactlyOnce(boolean graceful) throws Exception {
int updates = 1000;
int restarts = 10;
int snapshotIntervalMs = 100;
Pipeline pipeline = customersPipeline(Long.MAX_VALUE);
JobConfig config = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE).setSnapshotIntervalMillis(snapshotIntervalMs);
HazelcastInstance hz = createHazelcastInstances(2)[0];
Job job = hz.getJet().newJob(pipeline, config);
JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
assertEqualsEventually(() -> hz.getMap("results").size(), 4);
// make sure the job stores a Postgres WAL offset so that it won't trigger database snapshots after any restart
// multiple snapshots are a problem for this test, because it is updating the same row, so subsequent snapshots
// will return different images
JobRepository jr = new JobRepository(hz);
waitForNextSnapshot(jr, job.getId(), 20, false);
String lsnFlushedBeforeRestart = getConfirmedFlushLsn();
Future<?> dbChangesFuture = spawn(() -> uncheckRun(() -> {
for (int i = 1; i <= updates; i++) {
executeBatch("UPDATE customers SET first_name='Anne" + i + "' WHERE id=1004");
}
}));
for (int i = 0; i < restarts; i++) {
((JobProxy) job).restart(graceful);
assertJobStatusEventually(job, RUNNING);
Thread.sleep(ThreadLocalRandom.current().nextInt(snapshotIntervalMs * 2));
}
JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
try {
List<String> expectedPatterns = new ArrayList<>(Arrays.asList("1001/00000:SYNC:Customer \\{id=1001, firstName=Sally, lastName=Thomas, " + "email=sally.thomas@acme.com\\}", "1002/00000:SYNC:Customer \\{id=1002, firstName=George, lastName=Bailey, " + "email=gbailey@foobar.com\\}", "1003/00000:SYNC:Customer \\{id=1003, firstName=Edward, lastName=Walker, " + "email=ed@walker.com\\}", "1004/00000:SYNC:Customer \\{id=1004, firstName=Anne, lastName=Kretchmar, " + "email=annek@noanswer.org\\}"));
for (int i = 1; i <= updates; i++) {
expectedPatterns.add("1004/" + format("%05d", i) + ":UPDATE:Customer \\{id=1004, firstName=Anne" + i + ", lastName=Kretchmar, email=annek@noanswer.org\\}");
}
assertTrueEventually(() -> assertMatch(expectedPatterns, mapResultsToSortedList(hz.getMap("results"))));
assertTrueEventually(() -> assertNotEquals(lsnFlushedBeforeRestart, getConfirmedFlushLsn()));
} finally {
job.cancel();
assertJobStatusEventually(job, JobStatus.FAILED);
dbChangesFuture.get();
}
}
Aggregations