Examples with SimpleConsumer kafka.javaapi.consumer.SimpleConsumer used on opensource projects

Example 21 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class KafkaUtilsTest method setup.

@Before
public void setup() {
    broker = new KafkaTestBroker();
    GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(TEST_TOPIC);
    globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString()));
    brokerHosts = new StaticHosts(globalPartitionInformation);
    config = new KafkaConfig(brokerHosts, TEST_TOPIC);
    simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
}

Example 22 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class KafkaUtilsTest method brokerIsDown.

@Test(expected = FailedFetchException.class)
public void brokerIsDown() throws Exception {
    int port = broker.getPort();
    broker.shutdown();
    SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient");
    try {
        KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), OffsetRequest.LatestTime());
    } finally {
        simpleConsumer.close();
    }
}

Example 23 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class TestUtils method getKafkaConsumer.

public static SimpleConsumer getKafkaConsumer(KafkaTestBroker broker) {
    BrokerHosts brokerHosts = getBrokerHosts(broker);
    KafkaConfig kafkaConfig = new KafkaConfig(brokerHosts, TOPIC);
    SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
    return simpleConsumer;
}

Example 24 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project storm by apache.

the class ZkCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    server = new TestingServer();
    String connectionString = server.getConnectString();
    ZkHosts hosts = new ZkHosts(connectionString);
    hosts.refreshFreqSecs = 1;
    spoutConfig = new SpoutConfig(hosts, "topic", "/test", "id");
    Map conf = buildZookeeperConfig(server);
    state = new ZkState(conf);
    simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
    when(dynamicPartitionConnections.register(any(Broker.class), any(String.class), anyInt())).thenReturn(simpleConsumer);
}

Example 25 with SimpleConsumer

use of kafka.javaapi.consumer.SimpleConsumer in project druid by druid-io.

the class KafkaSimpleConsumer method findLeader.

private PartitionMetadata findLeader() throws InterruptedException {
    for (HostAndPort broker : replicaBrokers) {
        SimpleConsumer consumer = null;
        try {
            log.info("Finding new leader from Kafka brokers, try broker [%s]", broker.toString());
            consumer = new SimpleConsumer(broker.getHostText(), broker.getPort(), SO_TIMEOUT, BUFFER_SIZE, leaderLookupClientId);
            TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
            List<TopicMetadata> metaData = resp.topicsMetadata();
            for (TopicMetadata item : metaData) {
                if (topic.equals(item.topic())) {
                    for (PartitionMetadata part : item.partitionsMetadata()) {
                        if (part.partitionId() == partitionId) {
                            return part;
                        }
                    }
                }
            }
        } catch (Exception e) {
            ensureNotInterrupted(e);
            log.warn(e, "error communicating with Kafka Broker [%s] to find leader for [%s] - [%s]", broker, topic, partitionId);
        } finally {
            if (consumer != null) {
                consumer.close();
            }
        }
    }
    return null;
}