Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 41 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project testcases by coheigea.

the class KafkaAuthorizerSASLGSSTest method testUnauthorizedWrite.

// "client" is not authorized to write to "dev"
@org.junit.Test
public void testUnauthorizedWrite() throws Exception {
    // Create the Producer
    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", "localhost:" + port);
    producerProps.put("acks", "all");
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    producerProps.put("sasl.mechanism", "GSSAPI");
    producerProps.put("sasl.kerberos.service.name", "kafka");
    producerProps.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "JKS");
    producerProps.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, this.getClass().getResource("/servicestore.jks").getPath());
    producerProps.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
    producerProps.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, this.getClass().getResource("/truststore.jks").getPath());
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
    final Producer<String, String> producer = new KafkaProducer<>(producerProps);
    // Send a message
    try {
        Future<RecordMetadata> record = producer.send(new ProducerRecord<String, String>("dev", "somekey", "somevalue"));
        producer.flush();
        record.get();
        Assert.fail("Authorization failure expected");
    } catch (Exception ex) {
        Assert.assertTrue(ex.getMessage().contains("Not authorized to access topics"));
    }
    producer.close();
}

Example 42 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project testcases by coheigea.

the class KafkaSentryAuthorizerTest method testUnauthorizedWrite.

// The "public" group can't write to "test" or "dev"
@org.junit.Test
public void testUnauthorizedWrite() throws Exception {
    // Create the Producer
    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", "localhost:" + port);
    producerProps.put("acks", "all");
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
    producerProps.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "JKS");
    producerProps.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, this.getClass().getResource("/clientstore.jks").getPath());
    producerProps.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "cspass");
    producerProps.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "ckpass");
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, this.getClass().getResource("/truststore.jks").getPath());
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
    final Producer<String, String> producer = new KafkaProducer<>(producerProps);
    // Send a message
    try {
        Future<RecordMetadata> record = producer.send(new ProducerRecord<String, String>("test", "somekey2", "somevalue2"));
        producer.flush();
        record.get();
        Assert.fail("Authorization failure expected");
    } catch (Exception ex) {
        Assert.assertTrue(ex.getMessage().contains("Not authorized to access topics"));
    }
    producer.close();
}

Example 43 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project drill by apache.

the class KafkaMessageGenerator method populateJsonMsgWithTimestamps.

public void populateJsonMsgWithTimestamps(String topic, int numMsg) throws ExecutionException, InterruptedException {
    try (KafkaProducer<String, String> producer = new KafkaProducer<>(producerProperties)) {
        int halfCount = numMsg / 2;
        for (PartitionInfo tpInfo : producer.partitionsFor(topic)) {
            for (int i = 1; i <= numMsg; ++i) {
                JsonObject object = new JsonObject();
                object.addProperty("stringKey", UUID.randomUUID().toString());
                object.addProperty("intKey", numMsg - i);
                object.addProperty("boolKey", i % 2 == 0);
                long timestamp = i < halfCount ? (halfCount - i) : i;
                ProducerRecord<String, String> message = new ProducerRecord<>(tpInfo.topic(), tpInfo.partition(), timestamp, "key" + i, object.toString());
                logger.info("Publishing message : {}", message);
                Future<RecordMetadata> future = producer.send(message);
                logger.info("Committed offset of the message : {}", future.get().offset());
            }
        }
    }
}

Example 44 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project flink by apache.

the class FlinkKafkaProducer method open.

// ----------------------------------- Utilities --------------------------
/**
 * Initializes the connection to Kafka.
 */
@Override
public void open(Configuration configuration) throws Exception {
    if (logFailuresOnly) {
        callback = new Callback() {

            @Override
            public void onCompletion(RecordMetadata metadata, Exception e) {
                if (e != null) {
                    LOG.error("Error while sending record to Kafka: " + e.getMessage(), e);
                }
                acknowledgeMessage();
            }
        };
    } else {
        callback = new Callback() {

            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
                if (exception != null && asyncException == null) {
                    asyncException = exception;
                }
                acknowledgeMessage();
            }
        };
    }
    RuntimeContext ctx = getRuntimeContext();
    if (flinkKafkaPartitioner != null) {
        flinkKafkaPartitioner.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks());
    }
    if (kafkaSchema instanceof KafkaContextAware) {
        KafkaContextAware<IN> contextAwareSchema = (KafkaContextAware<IN>) kafkaSchema;
        contextAwareSchema.setParallelInstanceId(ctx.getIndexOfThisSubtask());
        contextAwareSchema.setNumParallelInstances(ctx.getNumberOfParallelSubtasks());
    }
    if (kafkaSchema != null) {
        kafkaSchema.open(RuntimeContextInitializationContextAdapters.serializationAdapter(getRuntimeContext(), metricGroup -> metricGroup.addGroup("user")));
    }
    super.open(configuration);
}

Example 45 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project flink by apache.

the class FlinkKafkaProducerBase method open.

// ----------------------------------- Utilities --------------------------
/**
 * Initializes the connection to Kafka.
 */
@Override
public void open(Configuration configuration) throws Exception {
    if (schema instanceof KeyedSerializationSchemaWrapper) {
        ((KeyedSerializationSchemaWrapper<IN>) schema).getSerializationSchema().open(RuntimeContextInitializationContextAdapters.serializationAdapter(getRuntimeContext(), metricGroup -> metricGroup.addGroup("user")));
    }
    producer = getKafkaProducer(this.producerConfig);
    RuntimeContext ctx = getRuntimeContext();
    if (null != flinkKafkaPartitioner) {
        flinkKafkaPartitioner.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks());
    }
    LOG.info("Starting FlinkKafkaProducer ({}/{}) to produce into default topic {}", ctx.getIndexOfThisSubtask() + 1, ctx.getNumberOfParallelSubtasks(), defaultTopicId);
    // register Kafka metrics to Flink accumulators
    if (!Boolean.parseBoolean(producerConfig.getProperty(KEY_DISABLE_METRICS, "false"))) {
        Map<MetricName, ? extends Metric> metrics = this.producer.metrics();
        if (metrics == null) {
            // MapR's Kafka implementation returns null here.
            LOG.info("Producer implementation does not support metrics");
        } else {
            final MetricGroup kafkaMetricGroup = getRuntimeContext().getMetricGroup().addGroup("KafkaProducer");
            for (Map.Entry<MetricName, ? extends Metric> metric : metrics.entrySet()) {
                kafkaMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
            }
        }
    }
    if (flushOnCheckpoint && !((StreamingRuntimeContext) this.getRuntimeContext()).isCheckpointingEnabled()) {
        LOG.warn("Flushing on checkpoint is enabled, but checkpointing is not enabled. Disabling flushing.");
        flushOnCheckpoint = false;
    }
    if (logFailuresOnly) {
        callback = new Callback() {

            @Override
            public void onCompletion(RecordMetadata metadata, Exception e) {
                if (e != null) {
                    LOG.error("Error while sending record to Kafka: " + e.getMessage(), e);
                }
                acknowledgeMessage();
            }
        };
    } else {
        callback = new Callback() {

            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
                if (exception != null && asyncException == null) {
                    asyncException = exception;
                }
                acknowledgeMessage();
            }
        };
    }
}