Examples with Metrics org.apache.kafka.common.metrics.Metrics used on opensource projects

Example 21 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method shouldAddUserDefinedEndPointToSubscription.

@Test
public void shouldAddUserDefinedEndPointToSubscription() throws Exception {
    final Properties properties = configProps();
    properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, "localhost:8080");
    final StreamsConfig config = new StreamsConfig(properties);
    final String applicationId = "application-id";
    builder.setApplicationId(applicationId);
    builder.addSource("source", "input");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source");
    builder.addSink("sink", "output", "processor");
    final UUID uuid1 = UUID.randomUUID();
    final String client1 = "client1";
    final StreamThread streamThread = new StreamThread(builder, config, mockClientSupplier, applicationId, client1, uuid1, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(streamThread, applicationId, client1));
    final PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("input"));
    final SubscriptionInfo subscriptionInfo = SubscriptionInfo.decode(subscription.userData());
    assertEquals("localhost:8080", subscriptionInfo.userEndPoint);
}

Example 22 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method testSubscription.

@SuppressWarnings("unchecked")
@Test
public void testSubscription() throws Exception {
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
    final Set<TaskId> prevTasks = Utils.mkSet(new TaskId(0, 1), new TaskId(1, 1), new TaskId(2, 1));
    final Set<TaskId> cachedTasks = Utils.mkSet(new TaskId(0, 1), new TaskId(1, 1), new TaskId(2, 1), new TaskId(0, 2), new TaskId(1, 2), new TaskId(2, 2));
    String clientId = "client-id";
    UUID processId = UUID.randomUUID();
    StreamThread thread = new StreamThread(builder, config, new MockClientSupplier(), "test", clientId, processId, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

        @Override
        public Set<TaskId> prevActiveTasks() {
            return prevTasks;
        }

        @Override
        public Set<TaskId> cachedTasks() {
            return cachedTasks;
        }
    };
    partitionAssignor.configure(config.getConsumerConfigs(thread, "test", clientId));
    PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("topic1", "topic2"));
    Collections.sort(subscription.topics());
    assertEquals(Utils.mkList("topic1", "topic2"), subscription.topics());
    Set<TaskId> standbyTasks = new HashSet<>(cachedTasks);
    standbyTasks.removeAll(prevTasks);
    SubscriptionInfo info = new SubscriptionInfo(processId, prevTasks, standbyTasks, null);
    assertEquals(info.encode(), subscription.userData());
}

Example 23 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method testAssignWithInternalTopics.

@Test
public void testAssignWithInternalTopics() throws Exception {
    String applicationId = "test";
    builder.setApplicationId(applicationId);
    builder.addInternalTopic("topicX");
    builder.addSource("source1", "topic1");
    builder.addProcessor("processor1", new MockProcessorSupplier(), "source1");
    builder.addSink("sink1", "topicX", "processor1");
    builder.addSource("source2", "topicX");
    builder.addProcessor("processor2", new MockProcessorSupplier(), "source2");
    List<String> topics = Utils.mkList("topic1", "test-topicX");
    Set<TaskId> allTasks = Utils.mkSet(task0, task1, task2);
    UUID uuid1 = UUID.randomUUID();
    String client1 = "client1";
    StreamThread thread10 = new StreamThread(builder, config, mockClientSupplier, applicationId, client1, uuid1, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(thread10, applicationId, client1));
    MockInternalTopicManager internalTopicManager = new MockInternalTopicManager(thread10.config, mockClientSupplier.restoreConsumer);
    partitionAssignor.setInternalTopicManager(internalTopicManager);
    Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put("consumer10", new PartitionAssignor.Subscription(topics, new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode()));
    partitionAssignor.assign(metadata, subscriptions);
    // check prepared internal topics
    assertEquals(1, internalTopicManager.readyTopics.size());
    assertEquals(allTasks.size(), (long) internalTopicManager.readyTopics.get("test-topicX"));
}

Example 24 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.

@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Exception {
    final Properties props = configProps();
    props.setProperty(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, "1");
    final StreamsConfig config = new StreamsConfig(props);
    final KStreamBuilder builder = new KStreamBuilder();
    final String applicationId = "appId";
    builder.setApplicationId(applicationId);
    builder.stream("topic1").groupByKey().count("count");
    final UUID uuid = UUID.randomUUID();
    final String client = "client1";
    final StreamThread streamThread = new StreamThread(builder, config, mockClientSupplier, applicationId, client, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(streamThread, applicationId, client));
    partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamThread.config, mockClientSupplier.restoreConsumer));
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put("consumer1", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    subscriptions.put("consumer2", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(UUID.randomUUID(), emptyTasks, emptyTasks, "other:9090").encode()));
    final Set<TopicPartition> allPartitions = Utils.mkSet(t1p0, t1p1, t1p2);
    final Map<String, PartitionAssignor.Assignment> assign = partitionAssignor.assign(metadata, subscriptions);
    final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
    final Set<TopicPartition> consumer1partitions = assignmentInfo.partitionsByHost.get(new HostInfo("localhost", 2171));
    final Set<TopicPartition> consumer2Partitions = assignmentInfo.partitionsByHost.get(new HostInfo("other", 9090));
    final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1partitions);
    allAssignedPartitions.addAll(consumer2Partitions);
    assertThat(consumer1partitions, not(allPartitions));
    assertThat(consumer2Partitions, not(allPartitions));
    assertThat(allAssignedPartitions, equalTo(allPartitions));
}

Example 25 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method shouldThrowExceptionIfApplicationServerConfigPortIsNotAnInteger.

@Test
public void shouldThrowExceptionIfApplicationServerConfigPortIsNotAnInteger() throws Exception {
    final Properties properties = configProps();
    final String myEndPoint = "localhost:j87yhk";
    properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, myEndPoint);
    final StreamsConfig config = new StreamsConfig(properties);
    final UUID uuid1 = UUID.randomUUID();
    final String client1 = "client1";
    final String applicationId = "application-id";
    builder.setApplicationId(applicationId);
    final StreamThread streamThread = new StreamThread(builder, config, mockClientSupplier, applicationId, client1, uuid1, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    try {
        partitionAssignor.configure(config.getConsumerConfigs(streamThread, applicationId, client1));
        Assert.fail("expected to an exception due to invalid config");
    } catch (ConfigException e) {
    // pass
    }
}