Examples with Metrics org.apache.kafka.common.metrics.Metrics used on opensource projects

Example 76 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class ThreadCacheTest method basicPutGet.

@Test
public void basicPutGet() throws IOException {
    List<KeyValue<String, String>> toInsert = Arrays.asList(new KeyValue<>("K1", "V1"), new KeyValue<>("K2", "V2"), new KeyValue<>("K3", "V3"), new KeyValue<>("K4", "V4"), new KeyValue<>("K5", "V5"));
    final KeyValue<String, String> kv = toInsert.get(0);
    final String name = "name";
    ThreadCache cache = new ThreadCache("testCache", toInsert.size() * memoryCacheEntrySize(kv.key.getBytes(), kv.value.getBytes(), ""), new MockStreamsMetrics(new Metrics()));
    for (KeyValue<String, String> kvToInsert : toInsert) {
        Bytes key = Bytes.wrap(kvToInsert.key.getBytes());
        byte[] value = kvToInsert.value.getBytes();
        cache.put(name, key, new LRUCacheEntry(value, true, 1L, 1L, 1, ""));
    }
    for (KeyValue<String, String> kvToInsert : toInsert) {
        Bytes key = Bytes.wrap(kvToInsert.key.getBytes());
        LRUCacheEntry entry = cache.get(name, key);
        assertEquals(entry.isDirty(), true);
        assertEquals(new String(entry.value), kvToInsert.value);
    }
    assertEquals(cache.gets(), 5);
    assertEquals(cache.puts(), 5);
    assertEquals(cache.evicts(), 0);
    assertEquals(cache.flushes(), 0);
}

Example 77 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class ThreadCacheTest method shouldPutIfAbsent.

@Test
public void shouldPutIfAbsent() throws Exception {
    final ThreadCache cache = new ThreadCache("testCache", 100000, new MockStreamsMetrics(new Metrics()));
    final Bytes key = Bytes.wrap(new byte[] { 10 });
    final byte[] value = { 30 };
    assertNull(cache.putIfAbsent("n", key, dirtyEntry(value)));
    assertArrayEquals(value, cache.putIfAbsent("n", key, dirtyEntry(new byte[] { 8 })).value);
    assertArrayEquals(value, cache.get("n", key).value);
}

Example 78 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class ThreadCacheTest method shouldEvictImmediatelyIfCacheSizeIsZero.

@Test
public void shouldEvictImmediatelyIfCacheSizeIsZero() throws Exception {
    final ThreadCache cache = new ThreadCache("testCache", 0, new MockStreamsMetrics(new Metrics()));
    shouldEvictImmediatelyIfCacheSizeIsZeroOrVerySmall(cache);
}

Example 79 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class ThreadCacheTest method checkOverheads.

private void checkOverheads(double entryFactor, double systemFactor, long desiredCacheSize, int keySizeBytes, int valueSizeBytes) {
    Runtime runtime = Runtime.getRuntime();
    final String name = "name";
    long numElements = desiredCacheSize / memoryCacheEntrySize(new byte[keySizeBytes], new byte[valueSizeBytes], "");
    System.gc();
    long prevRuntimeMemory = runtime.totalMemory() - runtime.freeMemory();
    ThreadCache cache = new ThreadCache("testCache", desiredCacheSize, new MockStreamsMetrics(new Metrics()));
    long size = cache.sizeBytes();
    assertEquals(size, 0);
    for (int i = 0; i < numElements; i++) {
        String keyStr = "K" + i;
        Bytes key = Bytes.wrap(keyStr.getBytes());
        byte[] value = new byte[valueSizeBytes];
        cache.put(name, key, new LRUCacheEntry(value, true, 1L, 1L, 1, ""));
    }
    System.gc();
    double ceiling = desiredCacheSize + desiredCacheSize * entryFactor;
    long usedRuntimeMemory = runtime.totalMemory() - runtime.freeMemory() - prevRuntimeMemory;
    assertTrue((double) cache.sizeBytes() <= ceiling);
    assertTrue("Used memory size " + usedRuntimeMemory + " greater than expected " + cache.sizeBytes() * systemFactor, cache.sizeBytes() * systemFactor >= usedRuntimeMemory);
}

Example 80 with Metrics

use of org.apache.kafka.common.metrics.Metrics in project kafka by apache.

the class StreamPartitionAssignorTest method testAssignWithNewTasks.

@Test
public void testAssignWithNewTasks() throws Exception {
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addSource("source3", "topic3");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2", "source3");
    List<String> topics = Utils.mkList("topic1", "topic2", "topic3");
    Set<TaskId> allTasks = Utils.mkSet(task0, task1, task2, task3);
    // assuming that previous tasks do not have topic3
    final Set<TaskId> prevTasks10 = Utils.mkSet(task0);
    final Set<TaskId> prevTasks11 = Utils.mkSet(task1);
    final Set<TaskId> prevTasks20 = Utils.mkSet(task2);
    UUID uuid1 = UUID.randomUUID();
    UUID uuid2 = UUID.randomUUID();
    String client1 = "client1";
    StreamThread thread10 = new StreamThread(builder, config, mockClientSupplier, "test", client1, uuid1, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(thread10, "test", client1));
    partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(thread10.config, mockClientSupplier.restoreConsumer));
    Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    subscriptions.put("consumer10", new PartitionAssignor.Subscription(topics, new SubscriptionInfo(uuid1, prevTasks10, Collections.<TaskId>emptySet(), userEndPoint).encode()));
    subscriptions.put("consumer11", new PartitionAssignor.Subscription(topics, new SubscriptionInfo(uuid1, prevTasks11, Collections.<TaskId>emptySet(), userEndPoint).encode()));
    subscriptions.put("consumer20", new PartitionAssignor.Subscription(topics, new SubscriptionInfo(uuid2, prevTasks20, Collections.<TaskId>emptySet(), userEndPoint).encode()));
    Map<String, PartitionAssignor.Assignment> assignments = partitionAssignor.assign(metadata, subscriptions);
    // check assigned partitions: since there is no previous task for topic 3 it will be assigned randomly so we cannot check exact match
    // also note that previously assigned partitions / tasks may not stay on the previous host since we may assign the new task first and
    // then later ones will be re-assigned to other hosts due to load balancing
    Set<TaskId> allActiveTasks = new HashSet<>();
    Set<TopicPartition> allPartitions = new HashSet<>();
    AssignmentInfo info;
    info = AssignmentInfo.decode(assignments.get("consumer10").userData());
    allActiveTasks.addAll(info.activeTasks);
    allPartitions.addAll(assignments.get("consumer10").partitions());
    info = AssignmentInfo.decode(assignments.get("consumer11").userData());
    allActiveTasks.addAll(info.activeTasks);
    allPartitions.addAll(assignments.get("consumer11").partitions());
    info = AssignmentInfo.decode(assignments.get("consumer20").userData());
    allActiveTasks.addAll(info.activeTasks);
    allPartitions.addAll(assignments.get("consumer20").partitions());
    assertEquals(allTasks, allActiveTasks);
    assertEquals(Utils.mkSet(t1p0, t1p1, t1p2, t2p0, t2p1, t2p2, t3p0, t3p1, t3p2, t3p3), allPartitions);
}