Examples with NONE com.hazelcast.jet.config.ProcessingGuarantee.NONE used on opensource projects

Example 1 with NONE

use of com.hazelcast.jet.config.ProcessingGuarantee.NONE in project hazelcast by hazelcast.

the class JmsSourceIntegrationTestBase method stressTest.

private void stressTest(boolean graceful, ProcessingGuarantee maxGuarantee, boolean useTopic) throws Exception {
    lastListInStressTest = null;
    final int MESSAGE_COUNT = 4_000;
    Pipeline p = Pipeline.create();
    String destName = "queue-" + counter++;
    JmsSourceBuilder sourceBuilder;
    if (useTopic) {
        sourceBuilder = Sources.jmsTopicBuilder(getConnectionFactory()).sharedConsumer(true).consumerFn(s -> s.createSharedDurableConsumer(s.createTopic(destName), "foo-consumer"));
        // create the durable subscriber now so that it doesn't lose the initial messages
        try (Connection conn = getConnectionFactory().get().createConnection()) {
            conn.setClientID("foo-client-id");
            try (Session sess = conn.createSession(false, DUPS_OK_ACKNOWLEDGE)) {
                sess.createDurableSubscriber(sess.createTopic(destName), "foo-consumer");
            }
        }
    } else {
        sourceBuilder = Sources.jmsQueueBuilder(getConnectionFactory()).destinationName(destName);
    }
    p.readFrom(sourceBuilder.maxGuarantee(maxGuarantee).build(msg -> Long.parseLong(((TextMessage) msg).getText()))).withoutTimestamps().peek().mapStateful(CopyOnWriteArrayList<Long>::new, (list, item) -> {
        lastListInStressTest = list;
        list.add(item);
        return null;
    }).writeTo(Sinks.logger());
    Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE).setSnapshotIntervalMillis(50));
    assertJobStatusEventually(job, RUNNING);
    // start a producer that will produce MESSAGE_COUNT messages on the background to the queue, 1000 msgs/s
    @SuppressWarnings("rawtypes") Future producerFuture = spawn(() -> {
        try (Connection connection = getConnectionFactory().get().createConnection();
            Session session = connection.createSession(false, AUTO_ACKNOWLEDGE);
            MessageProducer producer = session.createProducer(useTopic ? session.createTopic(destName) : session.createQueue(destName))) {
            long startTime = System.nanoTime();
            for (int i = 0; i < MESSAGE_COUNT; i++) {
                producer.send(session.createTextMessage(String.valueOf(i)));
                Thread.sleep(Math.max(0, i - NANOSECONDS.toMillis(System.nanoTime() - startTime)));
            }
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
    });
    int iteration = 0;
    JobRepository jr = new JobRepository(instance());
    waitForFirstSnapshot(jr, job.getId(), 20, true);
    while (!producerFuture.isDone()) {
        Thread.sleep(ThreadLocalRandom.current().nextInt(200));
        // We also do it before the first restart to workaround https://issues.apache.org/jira/browse/ARTEMIS-2546
        if (iteration++ % 3 == 0) {
            waitForNextSnapshot(jr, job.getId(), 20, true);
        }
        ((JobProxy) job).restart(graceful);
        assertJobStatusEventually(job, RUNNING);
    }
    // call for the side-effect of throwing if the producer failed
    producerFuture.get();
    assertTrueEventually(() -> {
        Map<Long, Long> counts = lastListInStressTest.stream().collect(Collectors.groupingBy(Function.identity(), TreeMap::new, Collectors.counting()));
        for (long i = 0; i < MESSAGE_COUNT; i++) {
            counts.putIfAbsent(i, 0L);
        }
        String countsStr = "counts: " + counts;
        if (maxGuarantee == NONE) {
            // we don't assert anything and only wait little more and check that the job didn't fail
            sleepSeconds(1);
        } else {
            // in EXACTLY_ONCE the list must have each item exactly once
            // in AT_LEAST_ONCE the list must have each item at least once
            assertTrue(countsStr, counts.values().stream().allMatch(cnt -> maxGuarantee == EXACTLY_ONCE ? cnt == 1 : cnt > 0));
        }
        logger.info(countsStr);
    }, 30);
    assertEquals(job.getStatus(), RUNNING);
}

Example 2 with NONE

use of com.hazelcast.jet.config.ProcessingGuarantee.NONE in project hazelcast by hazelcast.

the class GracefulShutdownTest method when_shutDown.

private void when_shutDown(boolean shutdownCoordinator, boolean snapshotted) {
    DAG dag = new DAG();
    final int numItems = 50_000;
    Vertex source = dag.newVertex("source", throttle(() -> new EmitIntegersP(numItems), 10_000)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(source, sink));
    Job job = client.getJet().newJob(dag, new JobConfig().setProcessingGuarantee(snapshotted ? EXACTLY_ONCE : NONE).setSnapshotIntervalMillis(HOURS.toMillis(1)));
    assertJobStatusEventually(job, JobStatus.RUNNING);
    logger.info("sleeping 1 sec");
    sleepSeconds(1);
    int shutDownInstance = shutdownCoordinator ? 0 : 1;
    int liveInstance = shutdownCoordinator ? 1 : 0;
    // When
    logger.info("Shutting down instance...");
    instances[shutDownInstance].shutdown();
    logger.info("Joining job...");
    job.join();
    logger.info("Joined");
    // Then
    // If the shutdown was graceful, output items must not be duplicated
    Map<Integer, Integer> expected;
    Map<Integer, Integer> actual = new ArrayList<>(instances[liveInstance].<Integer>getList("sink")).stream().collect(Collectors.toMap(Function.identity(), item -> 1, Integer::sum));
    if (snapshotted) {
        logger.info("savedCounters=" + EmitIntegersP.savedCounters);
        assertEquals(EmitIntegersP.savedCounters.toString(), 2, EmitIntegersP.savedCounters.size());
        int minCounter = EmitIntegersP.savedCounters.values().stream().mapToInt(Integer::intValue).min().getAsInt();
        expected = IntStream.range(0, numItems).boxed().collect(Collectors.toMap(Function.identity(), item -> item < minCounter ? 2 : 1));
        assertEquals(expected, actual);
    } else {
        // Items 0, 1, ... up to the point when the member was shut down should be 3 times
        // in the output: twice from before shutdown and one from after it, because it will start
        // from the beginning when the job is not snapshotted.
        assertEquals(3, actual.get(0).intValue());
        assertEquals(3, actual.get(1).intValue());
        assertEquals(1, actual.get(numItems - 1).intValue());
    }
}

Example 3 with NONE

use of com.hazelcast.jet.config.ProcessingGuarantee.NONE in project hazelcast by hazelcast.

the class StreamJmsPTest method initializeProcessor.

private void initializeProcessor(String destinationName, boolean isQueue, FunctionEx<Message, String> projectionFn) throws Exception {
    processorConnection = getConnectionFactory().createConnection();
    processorConnection.start();
    FunctionEx<Session, MessageConsumer> consumerFn = s -> s.createConsumer(isQueue ? s.createQueue(destinationName) : s.createTopic(destinationName));
    if (projectionFn == null) {
        projectionFn = m -> ((TextMessage) m).getText();
    }
    processor = new StreamJmsP<>(processorConnection, consumerFn, Message::getJMSMessageID, projectionFn, noEventTime(), NONE);
    outbox = new TestOutbox(1);
    processor.init(outbox, new TestProcessorContext());
}