use of com.hazelcast.jet.core.JobStatus.RUNNING in project hazelcast by hazelcast.
the class AsyncTransformUsingServiceBatchP_IntegrationTest method stressTestInt.
private void stressTestInt(boolean restart) {
/*
This is a stress test of the cooperative emission using the DAG api. Only through DAG
API we can configure edge queue sizes, which we use to cause more trouble for the
cooperative emission.
*/
// add more input to the source map
int numItems = 10_000;
journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
DAG dag = new DAG();
Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
BiFunctionEx<ExecutorService, List<Integer>, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5")).andThen(r -> r.thenApply(results -> traverseIterable(results).flatMap(Function.identity())));
ProcessorSupplier processorSupplier = AsyncTransformUsingServiceBatchedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, 128, flatMapAsyncFn);
Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
// Use a shorter queue to not block the barrier from the source for too long due to
// the backpressure from the slow mapper
EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
// Use a shorter queue on output from the mapper so that we experience backpressure
// from the sink
EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
Job job = instance().getJet().newJob(dag, jobConfig);
for (int i = 0; restart && i < 5; i++) {
assertNotNull(job);
assertTrueEventually(() -> {
JobStatus status = job.getStatus();
assertTrue("status=" + status, status == RUNNING || status == COMPLETED);
});
sleepMillis(100);
try {
job.restart();
} catch (IllegalStateException e) {
assertTrue(e.toString(), e.getMessage().startsWith("Cannot RESTART_GRACEFUL"));
break;
}
}
assertResult(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}
use of com.hazelcast.jet.core.JobStatus.RUNNING in project hazelcast by hazelcast.
the class JmsSourceIntegrationTestBase method stressTest.
private void stressTest(boolean graceful, ProcessingGuarantee maxGuarantee, boolean useTopic) throws Exception {
lastListInStressTest = null;
final int MESSAGE_COUNT = 4_000;
Pipeline p = Pipeline.create();
String destName = "queue-" + counter++;
JmsSourceBuilder sourceBuilder;
if (useTopic) {
sourceBuilder = Sources.jmsTopicBuilder(getConnectionFactory()).sharedConsumer(true).consumerFn(s -> s.createSharedDurableConsumer(s.createTopic(destName), "foo-consumer"));
// create the durable subscriber now so that it doesn't lose the initial messages
try (Connection conn = getConnectionFactory().get().createConnection()) {
conn.setClientID("foo-client-id");
try (Session sess = conn.createSession(false, DUPS_OK_ACKNOWLEDGE)) {
sess.createDurableSubscriber(sess.createTopic(destName), "foo-consumer");
}
}
} else {
sourceBuilder = Sources.jmsQueueBuilder(getConnectionFactory()).destinationName(destName);
}
p.readFrom(sourceBuilder.maxGuarantee(maxGuarantee).build(msg -> Long.parseLong(((TextMessage) msg).getText()))).withoutTimestamps().peek().mapStateful(CopyOnWriteArrayList<Long>::new, (list, item) -> {
lastListInStressTest = list;
list.add(item);
return null;
}).writeTo(Sinks.logger());
Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE).setSnapshotIntervalMillis(50));
assertJobStatusEventually(job, RUNNING);
// start a producer that will produce MESSAGE_COUNT messages on the background to the queue, 1000 msgs/s
@SuppressWarnings("rawtypes") Future producerFuture = spawn(() -> {
try (Connection connection = getConnectionFactory().get().createConnection();
Session session = connection.createSession(false, AUTO_ACKNOWLEDGE);
MessageProducer producer = session.createProducer(useTopic ? session.createTopic(destName) : session.createQueue(destName))) {
long startTime = System.nanoTime();
for (int i = 0; i < MESSAGE_COUNT; i++) {
producer.send(session.createTextMessage(String.valueOf(i)));
Thread.sleep(Math.max(0, i - NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}
} catch (Exception e) {
throw sneakyThrow(e);
}
});
int iteration = 0;
JobRepository jr = new JobRepository(instance());
waitForFirstSnapshot(jr, job.getId(), 20, true);
while (!producerFuture.isDone()) {
Thread.sleep(ThreadLocalRandom.current().nextInt(200));
// We also do it before the first restart to workaround https://issues.apache.org/jira/browse/ARTEMIS-2546
if (iteration++ % 3 == 0) {
waitForNextSnapshot(jr, job.getId(), 20, true);
}
((JobProxy) job).restart(graceful);
assertJobStatusEventually(job, RUNNING);
}
// call for the side-effect of throwing if the producer failed
producerFuture.get();
assertTrueEventually(() -> {
Map<Long, Long> counts = lastListInStressTest.stream().collect(Collectors.groupingBy(Function.identity(), TreeMap::new, Collectors.counting()));
for (long i = 0; i < MESSAGE_COUNT; i++) {
counts.putIfAbsent(i, 0L);
}
String countsStr = "counts: " + counts;
if (maxGuarantee == NONE) {
// we don't assert anything and only wait little more and check that the job didn't fail
sleepSeconds(1);
} else {
// in EXACTLY_ONCE the list must have each item exactly once
// in AT_LEAST_ONCE the list must have each item at least once
assertTrue(countsStr, counts.values().stream().allMatch(cnt -> maxGuarantee == EXACTLY_ONCE ? cnt == 1 : cnt > 0));
}
logger.info(countsStr);
}, 30);
assertEquals(job.getStatus(), RUNNING);
}
use of com.hazelcast.jet.core.JobStatus.RUNNING in project hazelcast by hazelcast.
the class AsyncTransformUsingServiceP_IntegrationTest method stressTestInt.
private void stressTestInt(boolean restart) {
/*
This is a stress test of the cooperative emission using the DAG api. Only through DAG
API we can configure edge queue sizes, which we use to cause more trouble for the
cooperative emission.
*/
// add more input to the source map
int numItems = 10_000;
journaledMap.putAll(IntStream.range(NUM_ITEMS, numItems).boxed().collect(toMap(i -> i, i -> i)));
DAG dag = new DAG();
Vertex source = dag.newVertex("source", throttle(streamMapP(journaledMap.getName(), alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_OLDEST, eventTimePolicy(i -> (long) ((Integer) i), WatermarkPolicy.limitingLag(10), 10, 0, 0)), 5000));
BiFunctionEx<ExecutorService, Integer, CompletableFuture<Traverser<String>>> flatMapAsyncFn = transformNotPartitionedFn(i -> traverseItems(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"));
ProcessorSupplier processorSupplier = ordered ? AsyncTransformUsingServiceOrderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn) : AsyncTransformUsingServiceUnorderedP.supplier(serviceFactory, DEFAULT_MAX_CONCURRENT_OPS, flatMapAsyncFn, identity());
Vertex map = dag.newVertex("map", processorSupplier).localParallelism(2);
Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP(sinkList.getName()));
// Use a shorter queue to not block the barrier from the source for too long due to
// the backpressure from the slow mapper
EdgeConfig edgeToMapperConfig = new EdgeConfig().setQueueSize(128);
// Use a shorter queue on output from the mapper so that we experience backpressure
// from the sink
EdgeConfig edgeFromMapperConfig = new EdgeConfig().setQueueSize(10);
dag.edge(between(source, map).setConfig(edgeToMapperConfig)).edge(between(map, sink).setConfig(edgeFromMapperConfig));
Job job = instance().getJet().newJob(dag, jobConfig);
for (int i = 0; restart && i < 5; i++) {
assertJobStatusEventually(job, RUNNING);
sleepMillis(100);
job.restart();
}
assertResultEventually(i -> Stream.of(i + "-1", i + "-2", i + "-3", i + "-4", i + "-5"), numItems);
}
use of com.hazelcast.jet.core.JobStatus.RUNNING in project hazelcast by hazelcast.
the class GracefulShutdownTest method when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot.
@Test
public void when_shutdownGracefulWhileRestartGraceful_then_restartsFromTerminalSnapshot() throws Exception {
MapConfig mapConfig = new MapConfig(JobRepository.SNAPSHOT_DATA_MAP_PREFIX + "*");
mapConfig.getMapStoreConfig().setClassName(BlockingMapStore.class.getName()).setEnabled(true);
Config config = instances[0].getConfig();
((DynamicConfigurationAwareConfig) config).getStaticConfig().addMapConfig(mapConfig);
BlockingMapStore.shouldBlock = false;
BlockingMapStore.wasBlocked = false;
DAG dag = new DAG();
int numItems = 5000;
Vertex source = dag.newVertex("source", throttle(() -> new EmitIntegersP(numItems), 500));
Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
dag.edge(between(source, sink));
source.localParallelism(1);
Job job = instances[0].getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(2000));
// wait for the first snapshot
JetServiceBackend jetServiceBackend = getNode(instances[0]).nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
JobRepository jobRepository = jetServiceBackend.getJobCoordinationService().jobRepository();
assertJobStatusEventually(job, RUNNING);
assertTrueEventually(() -> assertTrue(jobRepository.getJobExecutionRecord(job.getId()).dataMapIndex() >= 0));
// When
BlockingMapStore.shouldBlock = true;
job.restart();
assertTrueEventually(() -> assertTrue("blocking did not happen", BlockingMapStore.wasBlocked), 5);
Future shutdownFuture = spawn(() -> instances[1].shutdown());
logger.info("savedCounters=" + EmitIntegersP.savedCounters);
int minCounter = EmitIntegersP.savedCounters.values().stream().mapToInt(Integer::intValue).min().getAsInt();
BlockingMapStore.shouldBlock = false;
shutdownFuture.get();
// Then
job.join();
Map<Integer, Integer> actual = new ArrayList<>(instances[0].<Integer>getList("sink")).stream().collect(Collectors.toMap(Function.identity(), item -> 1, Integer::sum));
Map<Integer, Integer> expected = IntStream.range(0, numItems).boxed().collect(Collectors.toMap(Function.identity(), item -> item < minCounter ? 2 : 1));
assertEquals(expected, actual);
}
use of com.hazelcast.jet.core.JobStatus.RUNNING in project hazelcast by hazelcast.
the class ManualRestartTest method when_terminalSnapshotFails_then_previousSnapshotUsed.
@Test
public void when_terminalSnapshotFails_then_previousSnapshotUsed() {
MapConfig mapConfig = new MapConfig(JobRepository.SNAPSHOT_DATA_MAP_PREFIX + "*");
mapConfig.getMapStoreConfig().setClassName(FailingMapStore.class.getName()).setEnabled(true);
Config config = instances[0].getConfig();
((DynamicConfigurationAwareConfig) config).getStaticConfig().addMapConfig(mapConfig);
FailingMapStore.fail = false;
FailingMapStore.failed = false;
DAG dag = new DAG();
Vertex source = dag.newVertex("source", throttle(() -> new SequencesInPartitionsGeneratorP(2, 10000, true), 1000));
Vertex sink = dag.newVertex("sink", writeListP("sink"));
dag.edge(between(source, sink));
source.localParallelism(1);
Job job = instances[0].getJet().newJob(dag, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(2000));
// wait for the first snapshot
JetServiceBackend jetServiceBackend = getNode(instances[0]).nodeEngine.getService(JetServiceBackend.SERVICE_NAME);
JobRepository jobRepository = jetServiceBackend.getJobCoordinationService().jobRepository();
assertJobStatusEventually(job, RUNNING);
assertTrueEventually(() -> assertTrue(jobRepository.getJobExecutionRecord(job.getId()).dataMapIndex() >= 0));
// When
sleepMillis(100);
FailingMapStore.fail = true;
job.restart();
assertTrueEventually(() -> assertTrue(FailingMapStore.failed));
FailingMapStore.fail = false;
job.join();
Map<Integer, Integer> actual = new ArrayList<>(instances[0].<Entry<Integer, Integer>>getList("sink")).stream().filter(// we'll only check partition 0
e -> e.getKey() == 0).map(Entry::getValue).collect(Collectors.toMap(e -> e, e -> 1, (o, n) -> o + n, TreeMap::new));
assertEquals("first item != 1, " + actual.toString(), (Integer) 1, actual.get(0));
assertEquals("last item != 1, " + actual.toString(), (Integer) 1, actual.get(9999));
// the result should be some ones, then some twos and then some ones. The twos should be during the time
// since the last successful snapshot until the actual termination, when there was reprocessing.
boolean sawTwo = false;
boolean sawOneAgain = false;
for (Integer v : actual.values()) {
if (v == 1) {
if (sawTwo) {
sawOneAgain = true;
}
} else if (v == 2) {
assertFalse("got a 2 in another group", sawOneAgain);
sawTwo = true;
} else {
fail("v=" + v);
}
}
assertTrue("didn't see any 2s", sawTwo);
}
Aggregations