Search in sources :

Example 1 with TransactionPoolSnapshotUtility

use of com.hazelcast.jet.impl.processor.TransactionPoolSnapshotUtility in project hazelcast by hazelcast.

the class WriteKafkaP method init.

@Override
public void init(@Nonnull Outbox outbox, @Nonnull Context context) {
    this.context = context;
    ProcessingGuarantee guarantee = context.processingGuarantee() == EXACTLY_ONCE && !exactlyOnce ? AT_LEAST_ONCE : context.processingGuarantee();
    snapshotUtility = new TransactionPoolSnapshotUtility<>(outbox, context, false, guarantee, TXN_POOL_SIZE, (processorIndex, txnIndex) -> new KafkaTransactionId(context.jobId(), context.jobConfig().getName(), context.vertexName(), processorIndex, txnIndex), txnId -> {
        if (txnId != null) {
            properties.put("transactional.id", txnId.getKafkaId());
        }
        return new KafkaTransaction<>(txnId, properties, context.logger());
    }, txnId -> {
        try {
            recoverTransaction(txnId, true);
        } catch (ProducerFencedException e) {
            context.logger().warning("Failed to finish the commit of a transaction ID saved in the " + "snapshot, data loss can occur. Transaction id: " + txnId.getKafkaId(), e);
        }
    }, txnId -> recoverTransaction(txnId, false));
}
Also used : ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) LoggingUtil(com.hazelcast.jet.impl.util.LoggingUtil) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) Outbox(com.hazelcast.jet.core.Outbox) Processor(com.hazelcast.jet.core.Processor) HashMap(java.util.HashMap) KafkaProcessors(com.hazelcast.jet.kafka.KafkaProcessors) ExceptionUtil.sneakyThrow(com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Watermark(com.hazelcast.jet.core.Watermark) KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) ILogger(com.hazelcast.logging.ILogger) Map(java.util.Map) Inbox(com.hazelcast.jet.core.Inbox) Nonnull(javax.annotation.Nonnull) FunctionEx(com.hazelcast.function.FunctionEx) TransactionalResource(com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility.TransactionalResource) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Properties(java.util.Properties) TransactionPoolSnapshotUtility(com.hazelcast.jet.impl.processor.TransactionPoolSnapshotUtility) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) SupplierEx(com.hazelcast.function.SupplierEx) Serializable(java.io.Serializable) Objects(java.util.Objects) Util.idToString(com.hazelcast.jet.Util.idToString) InvalidTxnStateException(org.apache.kafka.common.errors.InvalidTxnStateException) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Callback(org.apache.kafka.clients.producer.Callback) AT_LEAST_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE) TwoPhaseSnapshotCommitUtility(com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException)

Aggregations

FunctionEx (com.hazelcast.function.FunctionEx)1 SupplierEx (com.hazelcast.function.SupplierEx)1 Util.idToString (com.hazelcast.jet.Util.idToString)1 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)1 AT_LEAST_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.AT_LEAST_ONCE)1 EXACTLY_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE)1 Inbox (com.hazelcast.jet.core.Inbox)1 Outbox (com.hazelcast.jet.core.Outbox)1 Processor (com.hazelcast.jet.core.Processor)1 Watermark (com.hazelcast.jet.core.Watermark)1 TransactionPoolSnapshotUtility (com.hazelcast.jet.impl.processor.TransactionPoolSnapshotUtility)1 TwoPhaseSnapshotCommitUtility (com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility)1 TransactionalResource (com.hazelcast.jet.impl.processor.TwoPhaseSnapshotCommitUtility.TransactionalResource)1 ExceptionUtil.sneakyThrow (com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow)1 LoggingUtil (com.hazelcast.jet.impl.util.LoggingUtil)1 KafkaProcessors (com.hazelcast.jet.kafka.KafkaProcessors)1 ILogger (com.hazelcast.logging.ILogger)1 Serializable (java.io.Serializable)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1