Search in sources :

Example 1 with AT_LEAST_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE in project hazelcast by hazelcast.

the class JmsSourceIntegrationTestBase method stressTest.

private void stressTest(boolean graceful, ProcessingGuarantee maxGuarantee, boolean useTopic) throws Exception {
    lastListInStressTest = null;
    final int MESSAGE_COUNT = 4_000;
    Pipeline p = Pipeline.create();
    String destName = "queue-" + counter++;
    JmsSourceBuilder sourceBuilder;
    if (useTopic) {
        sourceBuilder = Sources.jmsTopicBuilder(getConnectionFactory()).sharedConsumer(true).consumerFn(s -> s.createSharedDurableConsumer(s.createTopic(destName), "foo-consumer"));
        // create the durable subscriber now so that it doesn't lose the initial messages
        try (Connection conn = getConnectionFactory().get().createConnection()) {
            conn.setClientID("foo-client-id");
            try (Session sess = conn.createSession(false, DUPS_OK_ACKNOWLEDGE)) {
                sess.createDurableSubscriber(sess.createTopic(destName), "foo-consumer");
            }
        }
    } else {
        sourceBuilder = Sources.jmsQueueBuilder(getConnectionFactory()).destinationName(destName);
    }
    p.readFrom(sourceBuilder.maxGuarantee(maxGuarantee).build(msg -> Long.parseLong(((TextMessage) msg).getText()))).withoutTimestamps().peek().mapStateful(CopyOnWriteArrayList<Long>::new, (list, item) -> {
        lastListInStressTest = list;
        list.add(item);
        return null;
    }).writeTo(Sinks.logger());
    Job job = instance().getJet().newJob(p, new JobConfig().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE).setSnapshotIntervalMillis(50));
    assertJobStatusEventually(job, RUNNING);
    // start a producer that will produce MESSAGE_COUNT messages on the background to the queue, 1000 msgs/s
    @SuppressWarnings("rawtypes") Future producerFuture = spawn(() -> {
        try (Connection connection = getConnectionFactory().get().createConnection();
            Session session = connection.createSession(false, AUTO_ACKNOWLEDGE);
            MessageProducer producer = session.createProducer(useTopic ? session.createTopic(destName) : session.createQueue(destName))) {
            long startTime = System.nanoTime();
            for (int i = 0; i < MESSAGE_COUNT; i++) {
                producer.send(session.createTextMessage(String.valueOf(i)));
                Thread.sleep(Math.max(0, i - NANOSECONDS.toMillis(System.nanoTime() - startTime)));
            }
        } catch (Exception e) {
            throw sneakyThrow(e);
        }
    });
    int iteration = 0;
    JobRepository jr = new JobRepository(instance());
    waitForFirstSnapshot(jr, job.getId(), 20, true);
    while (!producerFuture.isDone()) {
        Thread.sleep(ThreadLocalRandom.current().nextInt(200));
        // We also do it before the first restart to workaround https://issues.apache.org/jira/browse/ARTEMIS-2546
        if (iteration++ % 3 == 0) {
            waitForNextSnapshot(jr, job.getId(), 20, true);
        }
        ((JobProxy) job).restart(graceful);
        assertJobStatusEventually(job, RUNNING);
    }
    // call for the side-effect of throwing if the producer failed
    producerFuture.get();
    assertTrueEventually(() -> {
        Map<Long, Long> counts = lastListInStressTest.stream().collect(Collectors.groupingBy(Function.identity(), TreeMap::new, Collectors.counting()));
        for (long i = 0; i < MESSAGE_COUNT; i++) {
            counts.putIfAbsent(i, 0L);
        }
        String countsStr = "counts: " + counts;
        if (maxGuarantee == NONE) {
            // we don't assert anything and only wait little more and check that the job didn't fail
            sleepSeconds(1);
        } else {
            // in EXACTLY_ONCE the list must have each item exactly once
            // in AT_LEAST_ONCE the list must have each item at least once
            assertTrue(countsStr, counts.values().stream().allMatch(cnt -> maxGuarantee == EXACTLY_ONCE ? cnt == 1 : cnt > 0));
        }
        logger.info(countsStr);
    }, 30);
    assertEquals(job.getStatus(), RUNNING);
}
Also used : AggregateOperations.counting(com.hazelcast.jet.aggregate.AggregateOperations.counting) Mockito.doThrow(org.mockito.Mockito.doThrow) Session(javax.jms.Session) Future(java.util.concurrent.Future) WindowDefinition.tumbling(com.hazelcast.jet.pipeline.WindowDefinition.tumbling) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) MessageProducer(javax.jms.MessageProducer) JobStatus(com.hazelcast.jet.core.JobStatus) SimpleTestInClusterSupport(com.hazelcast.jet.SimpleTestInClusterSupport) FunctionEx(com.hazelcast.function.FunctionEx) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JobConfig(com.hazelcast.jet.config.JobConfig) PredicateEx.alwaysFalse(com.hazelcast.function.PredicateEx.alwaysFalse) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Instant(java.time.Instant) DUPS_OK_ACKNOWLEDGE(javax.jms.Session.DUPS_OK_ACKNOWLEDGE) JMSException(javax.jms.JMSException) Collectors(java.util.stream.Collectors) SupplierEx(com.hazelcast.function.SupplierEx) ZoneId(java.time.ZoneId) Sources(com.hazelcast.jet.pipeline.Sources) List(java.util.List) MessageConsumer(javax.jms.MessageConsumer) Assert.assertFalse(org.junit.Assert.assertFalse) WindowResult(com.hazelcast.jet.datamodel.WindowResult) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) BeforeClass(org.junit.BeforeClass) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) AUTO_ACKNOWLEDGE(javax.jms.Session.AUTO_ACKNOWLEDGE) MINUTES(java.util.concurrent.TimeUnit.MINUTES) JobProxy(com.hazelcast.jet.impl.JobProxy) StreamSource(com.hazelcast.jet.pipeline.StreamSource) ExceptionUtil.sneakyThrow(com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Function(java.util.function.Function) ArrayList(java.util.ArrayList) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) MapWatermarksToString.mapWatermarksToString(com.hazelcast.jet.core.TestProcessors.MapWatermarksToString.mapWatermarksToString) JmsSourceBuilder(com.hazelcast.jet.pipeline.JmsSourceBuilder) Message(javax.jms.Message) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) Job(com.hazelcast.jet.Job) Before(org.junit.Before) IList(com.hazelcast.collection.IList) JobRepository(com.hazelcast.jet.impl.JobRepository) Connection(javax.jms.Connection) TextMessage(javax.jms.TextMessage) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) Sinks(com.hazelcast.jet.pipeline.Sinks) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) NONE(com.hazelcast.jet.config.ProcessingGuarantee.NONE) TreeMap(java.util.TreeMap) JmsTestUtil.consumeMessages(com.hazelcast.jet.impl.connector.JmsTestUtil.consumeMessages) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) AT_LEAST_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE) ConnectionFactory(javax.jms.ConnectionFactory) Assert.assertEquals(org.junit.Assert.assertEquals) Connection(javax.jms.Connection) MapWatermarksToString.mapWatermarksToString(com.hazelcast.jet.core.TestProcessors.MapWatermarksToString.mapWatermarksToString) JobRepository(com.hazelcast.jet.impl.JobRepository) JmsSourceBuilder(com.hazelcast.jet.pipeline.JmsSourceBuilder) JobConfig(com.hazelcast.jet.config.JobConfig) JMSException(javax.jms.JMSException) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JobProxy(com.hazelcast.jet.impl.JobProxy) Future(java.util.concurrent.Future) MessageProducer(javax.jms.MessageProducer) Job(com.hazelcast.jet.Job) Session(javax.jms.Session)

Example 2 with AT_LEAST_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE in project hazelcast by hazelcast.

the class SinkStressTestUtil method test_withRestarts.

public static void test_withRestarts(@Nonnull HazelcastInstance instance, @Nonnull ILogger logger, @Nonnull Sink<Integer> sink, boolean graceful, boolean exactlyOnce, @Nonnull SupplierEx<List<Integer>> actualItemsSupplier) {
    int numItems = 1000;
    Pipeline p = Pipeline.create();
    p.readFrom(SourceBuilder.stream("src", procCtx -> new int[] { procCtx.globalProcessorIndex() == 0 ? 0 : Integer.MAX_VALUE }).<Integer>fillBufferFn((ctx, buf) -> {
        if (ctx[0] < numItems) {
            buf.add(ctx[0]++);
            sleepMillis(5);
        }
    }).distributed(1).createSnapshotFn(ctx -> ctx[0] < Integer.MAX_VALUE ? ctx[0] : null).restoreSnapshotFn((ctx, state) -> ctx[0] = ctx[0] != Integer.MAX_VALUE ? state.get(0) : Integer.MAX_VALUE).build()).withoutTimestamps().peek().writeTo(sink);
    JobConfig config = new JobConfig().setProcessingGuarantee(exactlyOnce ? EXACTLY_ONCE : AT_LEAST_ONCE).setSnapshotIntervalMillis(50);
    JobProxy job = (JobProxy) instance.getJet().newJob(p, config);
    long endTime = System.nanoTime() + SECONDS.toNanos(TEST_TIMEOUT_SECONDS);
    int lastCount = 0;
    String expectedRows = IntStream.range(0, numItems).mapToObj(i -> i + (exactlyOnce ? "=1" : "")).collect(joining("\n"));
    // We'll restart once, then restart again after a short sleep (possibly during initialization),
    // and then assert some output so that the test isn't constantly restarting without any progress
    Long lastExecutionId = null;
    for (; ; ) {
        lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
        job.restart(graceful);
        lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
        sleepMillis(ThreadLocalRandom.current().nextInt(400));
        job.restart(graceful);
        try {
            List<Integer> actualItems;
            Set<Integer> distinctActualItems;
            do {
                actualItems = actualItemsSupplier.get();
                distinctActualItems = new HashSet<>(actualItems);
            } while (distinctActualItems.size() < Math.min(numItems, 100 + lastCount) && System.nanoTime() < endTime);
            lastCount = distinctActualItems.size();
            logger.info("number of committed items in the sink so far: " + lastCount);
            if (exactlyOnce) {
                String actualItemsStr = actualItems.stream().collect(groupingBy(identity(), TreeMap::new, counting())).entrySet().stream().map(Object::toString).collect(joining("\n"));
                assertEquals(expectedRows, actualItemsStr);
            } else {
                assertEquals(expectedRows, distinctActualItems.stream().map(Objects::toString).collect(joining("\n")));
            }
            // if content matches, break the loop. Otherwise restart and try again
            break;
        } catch (AssertionError e) {
            if (System.nanoTime() >= endTime) {
                throw e;
            }
        }
    }
}
Also used : IntStream(java.util.stream.IntStream) Collectors.counting(java.util.stream.Collectors.counting) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) JobProxy(com.hazelcast.jet.impl.JobProxy) HashSet(java.util.HashSet) ILogger(com.hazelcast.logging.ILogger) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Nonnull(javax.annotation.Nonnull) HazelcastInstance(com.hazelcast.core.HazelcastInstance) HazelcastTestSupport.sleepMillis(com.hazelcast.test.HazelcastTestSupport.sleepMillis) Pipeline(com.hazelcast.jet.pipeline.Pipeline) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) SupplierEx(com.hazelcast.function.SupplierEx) Collectors.joining(java.util.stream.Collectors.joining) Objects(java.util.Objects) List(java.util.List) TreeMap(java.util.TreeMap) JetTestSupport.assertJobRunningEventually(com.hazelcast.jet.core.JetTestSupport.assertJobRunningEventually) Function.identity(java.util.function.Function.identity) SourceBuilder(com.hazelcast.jet.pipeline.SourceBuilder) AT_LEAST_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE) Sink(com.hazelcast.jet.pipeline.Sink) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Assert.assertEquals(org.junit.Assert.assertEquals) JobConfig(com.hazelcast.jet.config.JobConfig) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JobProxy(com.hazelcast.jet.impl.JobProxy) Objects(java.util.Objects)

Example 3 with AT_LEAST_ONCE

use of com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE in project hazelcast by hazelcast.

the class WriteKafkaP method init.

@Override
public void init(@Nonnull Outbox outbox, @Nonnull Context context) {
    this.context = context;
    ProcessingGuarantee guarantee = context.processingGuarantee() == EXACTLY_ONCE && !exactlyOnce ? AT_LEAST_ONCE : context.processingGuarantee();
    snapshotUtility = new TransactionPoolSnapshotUtility<>(outbox, context, false, guarantee, TXN_POOL_SIZE, (processorIndex, txnIndex) -> new KafkaTransactionId(context.jobId(), context.jobConfig().getName(), context.vertexName(), processorIndex, txnIndex), txnId -> {
        if (txnId != null) {
            properties.put("transactional.id", txnId.getKafkaId());
        }
        return new KafkaTransaction<>(txnId, properties, context.logger());
    }, txnId -> {
        try {
            recoverTransaction(txnId, true);
        } catch (ProducerFencedException e) {
            context.logger().warning("Failed to finish the commit of a transaction ID saved in the " + "snapshot, data loss can occur. Transaction id: " + txnId.getKafkaId(), e);
        }
    }, txnId -> recoverTransaction(txnId, false));
}
Also used : ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) Outbox(com.hazelcast.jet.core.Outbox) Processor(com.hazelcast.jet.core.Processor) HashMap(java.util.HashMap) KafkaProcessors(com.hazelcast.jet.kafka.KafkaProcessors) ExceptionUtil.sneakyThrow(com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Watermark(com.hazelcast.jet.core.Watermark) KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) ILogger(com.hazelcast.logging.ILogger) Map(java.util.Map) Inbox(com.hazelcast.jet.core.Inbox) Nonnull(javax.annotation.Nonnull) FunctionEx(com.hazelcast.function.FunctionEx) TransactionalResource(com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility.TransactionalResource) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Properties(java.util.Properties) TransactionPoolSnapshotUtility(com.hazelcast.jet.impl.processor.TransactionPoolSnapshotUtility) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) SupplierEx(com.hazelcast.function.SupplierEx) Serializable(java.io.Serializable) Objects(java.util.Objects) Util.idToString(com.hazelcast.jet.Util.idToString) InvalidTxnStateException(org.apache.kafka.common.errors.InvalidTxnStateException) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Callback(org.apache.kafka.clients.producer.Callback) AT_LEAST_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE) TwoPhaseSnapshotCommitUtility(com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException)

Aggregations

SupplierEx (com.hazelcast.function.SupplierEx)3 AT_LEAST_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE)3 EXACTLY_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE)3 FunctionEx (com.hazelcast.function.FunctionEx)2 JobConfig (com.hazelcast.jet.config.JobConfig)2 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)2 JobProxy (com.hazelcast.jet.impl.JobProxy)2 ExceptionUtil.sneakyThrow (com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow)2 Pipeline (com.hazelcast.jet.pipeline.Pipeline)2 ILogger (com.hazelcast.logging.ILogger)2 List (java.util.List)2 TreeMap (java.util.TreeMap)2 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)2 Assert.assertEquals (org.junit.Assert.assertEquals)2 IList (com.hazelcast.collection.IList)1 HazelcastInstance (com.hazelcast.core.HazelcastInstance)1 PredicateEx.alwaysFalse (com.hazelcast.function.PredicateEx.alwaysFalse)1 Job (com.hazelcast.jet.Job)1 SimpleTestInClusterSupport (com.hazelcast.jet.SimpleTestInClusterSupport)1 Util.idToString (com.hazelcast.jet.Util.idToString)1