Examples with ProducerConfig kafka.producer.ProducerConfig used on opensource projects

Example 6 with ProducerConfig

use of kafka.producer.ProducerConfig in project druid by druid-io.

the class TestKafkaExtractionCluster method setUp.

@Before
public void setUp() throws Exception {
    zkTestServer = new TestingServer(-1, temporaryFolder.newFolder(), true);
    zkTestServer.start();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            zkTestServer.stop();
        }
    });
    zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            zkClient.close();
        }
    });
    if (!zkClient.exists("/kafka")) {
        zkClient.create("/kafka", null, CreateMode.PERSISTENT);
    }
    log.info("---------------------------Started ZK---------------------------");
    final String zkKafkaPath = "/kafka";
    final Properties serverProperties = new Properties();
    serverProperties.putAll(kafkaProperties);
    serverProperties.put("broker.id", "0");
    serverProperties.put("log.dir", temporaryFolder.newFolder().getAbsolutePath());
    serverProperties.put("log.cleaner.enable", "true");
    serverProperties.put("host.name", "127.0.0.1");
    serverProperties.put("zookeeper.connect", zkTestServer.getConnectString() + zkKafkaPath);
    serverProperties.put("zookeeper.session.timeout.ms", "10000");
    serverProperties.put("zookeeper.sync.time.ms", "200");
    kafkaConfig = new KafkaConfig(serverProperties);
    final long time = DateTime.parse("2015-01-01").getMillis();
    kafkaServer = new KafkaServer(kafkaConfig, new Time() {

        @Override
        public long milliseconds() {
            return time;
        }

        @Override
        public long nanoseconds() {
            return milliseconds() * 1_000_000;
        }

        @Override
        public void sleep(long ms) {
            try {
                Thread.sleep(ms);
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        }
    });
    kafkaServer.startup();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            kafkaServer.shutdown();
            kafkaServer.awaitShutdown();
        }
    });
    int sleepCount = 0;
    while (!kafkaServer.kafkaController().isActive()) {
        Thread.sleep(100);
        if (++sleepCount > 10) {
            throw new InterruptedException("Controller took to long to awaken");
        }
    }
    log.info("---------------------------Started Kafka Server---------------------------");
    final ZkClient zkClient = new ZkClient(zkTestServer.getConnectString() + zkKafkaPath, 10000, 10000, ZKStringSerializer$.MODULE$);
    try (final AutoCloseable autoCloseable = new AutoCloseable() {

        @Override
        public void close() throws Exception {
            if (zkClient.exists(zkKafkaPath)) {
                try {
                    zkClient.deleteRecursive(zkKafkaPath);
                } catch (org.I0Itec.zkclient.exception.ZkException ex) {
                    log.warn(ex, "error deleting %s zk node", zkKafkaPath);
                }
            }
            zkClient.close();
        }
    }) {
        final Properties topicProperties = new Properties();
        topicProperties.put("cleanup.policy", "compact");
        if (!AdminUtils.topicExists(zkClient, topicName)) {
            AdminUtils.createTopic(zkClient, topicName, 1, 1, topicProperties);
        }
        log.info("---------------------------Created topic---------------------------");
        Assert.assertTrue(AdminUtils.topicExists(zkClient, topicName));
    }
    final Properties kafkaProducerProperties = makeProducerProperties();
    final Producer<byte[], byte[]> producer = new Producer<>(new ProducerConfig(kafkaProducerProperties));
    try (final AutoCloseable autoCloseable = new AutoCloseable() {

        @Override
        public void close() throws Exception {
            producer.close();
        }
    }) {
        producer.send(new KeyedMessage<>(topicName, StringUtils.toUtf8("abcdefg"), StringUtils.toUtf8("abcdefg")));
    }
    System.setProperty("druid.extensions.searchCurrentClassloader", "false");
    injector = Initialization.makeInjectorWithModules(GuiceInjectors.makeStartupInjector(), ImmutableList.of(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to("test");
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
        }
    }, // These injections fail under IntelliJ but are required for maven
    new NamespaceExtractionModule(), new KafkaExtractionNamespaceModule()));
    mapper = injector.getInstance(ObjectMapper.class);
    log.info("--------------------------- placed default item via producer ---------------------------");
    final Map<String, String> consumerProperties = new HashMap<>(kafkaProperties);
    consumerProperties.put("zookeeper.connect", zkTestServer.getConnectString() + zkKafkaPath);
    consumerProperties.put("zookeeper.session.timeout.ms", "10000");
    consumerProperties.put("zookeeper.sync.time.ms", "200");
    final KafkaLookupExtractorFactory kafkaLookupExtractorFactory = new KafkaLookupExtractorFactory(null, topicName, consumerProperties);
    factory = (KafkaLookupExtractorFactory) mapper.readValue(mapper.writeValueAsString(kafkaLookupExtractorFactory), LookupExtractorFactory.class);
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaTopic(), factory.getKafkaTopic());
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaProperties(), factory.getKafkaProperties());
    factory.start();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            factory.close();
        }
    });
    log.info("--------------------------- started rename manager ---------------------------");
}

Example 7 with ProducerConfig

use of kafka.producer.ProducerConfig in project pinot by linkedin.

the class StreamAvroIntoKafkaCommand method execute.

@Override
public boolean execute() throws IOException {
    int messageDelayMillis = Integer.parseInt(_millisBetweenMessages);
    final boolean sleepRequired = 0 < messageDelayMillis;
    if (sleepRequired) {
        LOGGER.info("Streaming Avro file into Kafka topic {} with {} ms between messages", _kafkaTopic, _millisBetweenMessages);
    } else {
        LOGGER.info("Streaming Avro file into Kafka topic {} with no delay between messages", _kafkaTopic);
    }
    // Create Kafka producer
    Properties properties = new Properties();
    properties.put("metadata.broker.list", _kafkaBrokerList);
    properties.put("serializer.class", "kafka.serializer.DefaultEncoder");
    properties.put("request.required.acks", "1");
    ProducerConfig producerConfig = new ProducerConfig(properties);
    Producer<byte[], byte[]> producer = new Producer<byte[], byte[]>(producerConfig);
    try {
        // Open the Avro file
        DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(new File(_avroFile));
        // Iterate over every record
        for (GenericRecord genericRecord : reader) {
            // Write the message to Kafka
            String recordJson = genericRecord.toString();
            byte[] bytes = recordJson.getBytes("utf-8");
            KeyedMessage<byte[], byte[]> data = new KeyedMessage<byte[], byte[]>(_kafkaTopic, Longs.toByteArray(HashUtil.hash64(bytes, bytes.length)), bytes);
            producer.send(data);
            // Sleep between messages
            if (sleepRequired) {
                Uninterruptibles.sleepUninterruptibly(messageDelayMillis, TimeUnit.MILLISECONDS);
            }
        }
        reader.close();
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
    savePID(System.getProperty("java.io.tmpdir") + File.separator + ".streamAvro.pid");
    return true;
}

Example 8 with ProducerConfig

use of kafka.producer.ProducerConfig in project pinot by linkedin.

the class BaseClusterIntegrationTest method pushRandomAvroIntoKafka.

public static void pushRandomAvroIntoKafka(File avroFile, String kafkaBroker, String kafkaTopic, int rowCount, Random random) {
    Properties properties = new Properties();
    properties.put("metadata.broker.list", kafkaBroker);
    properties.put("serializer.class", "kafka.serializer.DefaultEncoder");
    properties.put("request.required.acks", "1");
    ProducerConfig producerConfig = new ProducerConfig(properties);
    Producer<String, byte[]> producer = new Producer<String, byte[]>(producerConfig);
    try {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream(65536);
        DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile);
        BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null);
        Schema avroSchema = reader.getSchema();
        GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(avroSchema);
        int recordCount = 0;
        int rowsRemaining = rowCount;
        int messagesInThisBatch = 0;
        while (rowsRemaining > 0) {
            int rowsInThisBatch = Math.min(rowsRemaining, MAX_MESSAGES_PER_BATCH);
            List<KeyedMessage<String, byte[]>> messagesToWrite = new ArrayList<KeyedMessage<String, byte[]>>(rowsInThisBatch);
            GenericRecord genericRecord = new GenericData.Record(avroSchema);
            for (int i = 0; i < rowsInThisBatch; ++i) {
                generateRandomRecord(genericRecord, avroSchema, random);
                outputStream.reset();
                datumWriter.write(genericRecord, binaryEncoder);
                binaryEncoder.flush();
                byte[] bytes = outputStream.toByteArray();
                KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(kafkaTopic, bytes);
                if (BATCH_KAFKA_MESSAGES) {
                    messagesToWrite.add(data);
                    messagesInThisBatch++;
                    if (MAX_MESSAGES_PER_BATCH <= messagesInThisBatch) {
                        messagesInThisBatch = 0;
                        producer.send(messagesToWrite);
                        messagesToWrite.clear();
                        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
                    }
                } else {
                    producer.send(data);
                }
                recordCount += 1;
            }
            if (BATCH_KAFKA_MESSAGES) {
                producer.send(messagesToWrite);
            }
            //        System.out.println("rowsRemaining = " + rowsRemaining);
            rowsRemaining -= rowsInThisBatch;
        }
        outputStream.close();
        reader.close();
        LOGGER.info("Finished writing " + recordCount + " records from " + avroFile.getName() + " into Kafka topic " + kafkaTopic);
        int totalRecordCount = totalAvroRecordWrittenCount.addAndGet(recordCount);
        LOGGER.info("Total records written so far " + totalRecordCount);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 9 with ProducerConfig

use of kafka.producer.ProducerConfig in project avro-kafka-storm by ransilberman.

the class MainTest method testGenericRecord.

@Test
public void testGenericRecord() throws IOException, InterruptedException {
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(getClass().getResourceAsStream("LPEvent.avsc"));
    GenericRecord datum = new GenericData.Record(schema);
    datum.put("revision", 1L);
    datum.put("siteId", "28280110");
    datum.put("eventType", "PLine");
    datum.put("timeStamp", System.currentTimeMillis());
    datum.put("sessionId", "123456II");
    Map<String, Schema> unions = new HashMap<String, Schema>();
    List<Schema> typeList = schema.getField("subrecord").schema().getTypes();
    for (Schema sch : typeList) {
        unions.put(sch.getName(), sch);
    }
    GenericRecord plineDatum = new GenericData.Record(unions.get("pline"));
    plineDatum.put("text", "How can I help you?");
    plineDatum.put("lineType", 1);
    plineDatum.put("repId", "REPID12345");
    datum.put("subrecord", plineDatum);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    writer.write(datum, encoder);
    encoder.flush();
    out.close();
    Message message = new Message(out.toByteArray());
    Properties props = new Properties();
    props.put("zk.connect", zkConnection);
    Producer<Message, Message> producer = new kafka.javaapi.producer.Producer<Message, Message>(new ProducerConfig(props));
    producer.send(new ProducerData<Message, Message>(topic, message));
}