Examples with MockTime org.apache.kafka.common.utils.MockTime used on opensource projects

Example 46 with MockTime

use of org.apache.kafka.common.utils.MockTime in project kafka by apache.

the class KafkaStatusBackingStoreTest method putSafeOverridesValueSetBySameWorker.

@Test
public void putSafeOverridesValueSetBySameWorker() {
    final byte[] value = new byte[0];
    KafkaBasedLog<String, byte[]> kafkaBasedLog = mock(KafkaBasedLog.class);
    Converter converter = mock(Converter.class);
    final KafkaStatusBackingStore store = new KafkaStatusBackingStore(new MockTime(), converter, STATUS_TOPIC, kafkaBasedLog);
    // the persisted came from the same host, but has a newer generation
    Map<String, Object> firstStatusRead = new HashMap<>();
    firstStatusRead.put("worker_id", WORKER_ID);
    firstStatusRead.put("state", "RUNNING");
    firstStatusRead.put("generation", 1L);
    Map<String, Object> secondStatusRead = new HashMap<>();
    secondStatusRead.put("worker_id", WORKER_ID);
    secondStatusRead.put("state", "UNASSIGNED");
    secondStatusRead.put("generation", 0L);
    expect(converter.toConnectData(STATUS_TOPIC, value)).andReturn(new SchemaAndValue(null, firstStatusRead)).andReturn(new SchemaAndValue(null, secondStatusRead));
    expect(converter.fromConnectData(eq(STATUS_TOPIC), anyObject(Schema.class), anyObject(Struct.class))).andStubReturn(value);
    final Capture<Callback> callbackCapture = newCapture();
    kafkaBasedLog.send(eq("status-connector-conn"), eq(value), capture(callbackCapture));
    expectLastCall().andAnswer(new IAnswer<Void>() {

        @Override
        public Void answer() throws Throwable {
            callbackCapture.getValue().onCompletion(null, null);
            store.read(consumerRecord(1, "status-connector-conn", value));
            return null;
        }
    });
    replayAll();
    store.read(consumerRecord(0, "status-connector-conn", value));
    store.putSafe(new ConnectorStatus(CONNECTOR, ConnectorStatus.State.UNASSIGNED, WORKER_ID, 0));
    ConnectorStatus status = new ConnectorStatus(CONNECTOR, ConnectorStatus.State.UNASSIGNED, WORKER_ID, 0);
    assertEquals(status, store.get(CONNECTOR));
    verifyAll();
}

Example 47 with MockTime

use of org.apache.kafka.common.utils.MockTime in project kafka by apache.

the class KafkaStatusBackingStoreTest method putConnectorStateShouldOverride.

@Test
public void putConnectorStateShouldOverride() {
    final byte[] value = new byte[0];
    String otherWorkerId = "anotherhost:8083";
    KafkaBasedLog<String, byte[]> kafkaBasedLog = mock(KafkaBasedLog.class);
    Converter converter = mock(Converter.class);
    final KafkaStatusBackingStore store = new KafkaStatusBackingStore(new MockTime(), converter, STATUS_TOPIC, kafkaBasedLog);
    // the persisted came from a different host and has a newer generation
    Map<String, Object> firstStatusRead = new HashMap<>();
    firstStatusRead.put("worker_id", otherWorkerId);
    firstStatusRead.put("state", "RUNNING");
    firstStatusRead.put("generation", 1L);
    Map<String, Object> secondStatusRead = new HashMap<>();
    secondStatusRead.put("worker_id", WORKER_ID);
    secondStatusRead.put("state", "UNASSIGNED");
    secondStatusRead.put("generation", 0L);
    expect(converter.toConnectData(STATUS_TOPIC, value)).andReturn(new SchemaAndValue(null, firstStatusRead)).andReturn(new SchemaAndValue(null, secondStatusRead));
    expect(converter.fromConnectData(eq(STATUS_TOPIC), anyObject(Schema.class), anyObject(Struct.class))).andStubReturn(value);
    final Capture<Callback> callbackCapture = newCapture();
    kafkaBasedLog.send(eq("status-connector-conn"), eq(value), capture(callbackCapture));
    expectLastCall().andAnswer(new IAnswer<Void>() {

        @Override
        public Void answer() throws Throwable {
            callbackCapture.getValue().onCompletion(null, null);
            store.read(consumerRecord(1, "status-connector-conn", value));
            return null;
        }
    });
    replayAll();
    store.read(consumerRecord(0, "status-connector-conn", value));
    ConnectorStatus status = new ConnectorStatus(CONNECTOR, ConnectorStatus.State.UNASSIGNED, WORKER_ID, 0);
    store.put(status);
    assertEquals(status, store.get(CONNECTOR));
    verifyAll();
}

Example 48 with MockTime

use of org.apache.kafka.common.utils.MockTime in project kafka by apache.

the class StreamThreadTest method shouldCloseActiveTasksThatAreAssignedToThisStreamThreadButAssignmentHasChangedBeforeCreatingNewTasks.

@Test
public void shouldCloseActiveTasksThatAreAssignedToThisStreamThreadButAssignmentHasChangedBeforeCreatingNewTasks() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream(Pattern.compile("t.*")).to("out");
    final StreamsConfig config = new StreamsConfig(configProps());
    final MockClientSupplier clientSupplier = new MockClientSupplier();
    final Map<Collection<TopicPartition>, TestStreamTask> createdTasks = new HashMap<>();
    final StreamThread thread = new StreamThread(builder, config, clientSupplier, applicationId, clientId, processId, new Metrics(), new MockTime(), new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            final ProcessorTopology topology = builder.build(id.topicGroupId);
            final TestStreamTask task = new TestStreamTask(id, applicationId, partitions, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
            createdTasks.put(partitions, task);
            return task;
        }
    };
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    final TopicPartition t1 = new TopicPartition("t1", 0);
    final Set<TopicPartition> task00Partitions = new HashSet<>();
    task00Partitions.add(t1);
    final TaskId taskId = new TaskId(0, 0);
    activeTasks.put(taskId, task00Partitions);
    thread.partitionAssignor(new StreamPartitionAssignor() {

        @Override
        Map<TaskId, Set<TopicPartition>> activeTasks() {
            return activeTasks;
        }
    });
    // should create task for id 0_0 with a single partition
    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(task00Partitions);
    final TestStreamTask firstTask = createdTasks.get(task00Partitions);
    assertThat(firstTask.id(), is(taskId));
    // update assignment for the task 0_0 so it now has 2 partitions
    task00Partitions.add(new TopicPartition("t2", 0));
    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(task00Partitions);
    // should close the first task as the assignment has changed
    assertTrue("task should have been closed as assignment has changed", firstTask.closed);
    assertTrue("tasks state manager should have been closed as assignment has changed", firstTask.closedStateManager);
    // should have created a new task for 00
    assertThat(createdTasks.get(task00Partitions).id(), is(taskId));
}

Example 49 with MockTime

use of org.apache.kafka.common.utils.MockTime in project kafka by apache.

the class StreamThreadTest method shouldNotViolateAtLeastOnceWhenAnExceptionOccursOnTaskCloseDuringShutdown.

@Test
public void shouldNotViolateAtLeastOnceWhenAnExceptionOccursOnTaskCloseDuringShutdown() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream("t1").groupByKey();
    final StreamsConfig config = new StreamsConfig(configProps());
    final MockClientSupplier clientSupplier = new MockClientSupplier();
    final TestStreamTask testStreamTask = new TestStreamTask(new TaskId(0, 0), applicationId, Utils.mkSet(new TopicPartition("t1", 0)), builder.build(0), clientSupplier.consumer, clientSupplier.producer, clientSupplier.restoreConsumer, config, new MockStreamsMetrics(new Metrics()), new StateDirectory(applicationId, config.getString(StreamsConfig.STATE_DIR_CONFIG), time)) {

        @Override
        public void close() {
            throw new RuntimeException("KABOOM!");
        }
    };
    final StreamsConfig config1 = new StreamsConfig(configProps());
    final StreamThread thread = new StreamThread(builder, config1, clientSupplier, applicationId, clientId, processId, new Metrics(), new MockTime(), new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            return testStreamTask;
        }
    };
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    activeTasks.put(testStreamTask.id, testStreamTask.partitions);
    thread.partitionAssignor(new MockStreamsPartitionAssignor(activeTasks));
    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(testStreamTask.partitions);
    thread.start();
    thread.close();
    thread.join();
    assertFalse("task shouldn't have been committed as there was an exception during shutdown", testStreamTask.committed);
}

Example 50 with MockTime

use of org.apache.kafka.common.utils.MockTime in project kafka by apache.

the class StreamThreadTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    TopologyBuilder builder = new TopologyBuilder().setApplicationId("MetricsApp");
    StreamsConfig config = new StreamsConfig(configProps());
    MockClientSupplier clientSupplier = new MockClientSupplier();
    Metrics metrics = new Metrics();
    StreamThread thread = new StreamThread(builder, config, clientSupplier, applicationId, clientId, processId, metrics, new MockTime(), new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    String defaultGroupName = "stream-metrics";
    String defaultPrefix = "thread." + thread.threadClientId();
    Map<String, String> defaultTags = Collections.singletonMap("client-id", thread.threadClientId());
    assertNotNull(metrics.getSensor(defaultPrefix + ".commit-latency"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".poll-latency"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".process-latency"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".punctuate-latency"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".task-created"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".task-closed"));
    assertNotNull(metrics.getSensor(defaultPrefix + ".skipped-records"));
    assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-avg", defaultGroupName, "The average commit time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("commit-latency-max", defaultGroupName, "The maximum commit time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("commit-rate", defaultGroupName, "The average per-second number of commit calls", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("poll-latency-avg", defaultGroupName, "The average poll time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("poll-latency-max", defaultGroupName, "The maximum poll time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("poll-rate", defaultGroupName, "The average per-second number of record-poll calls", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("process-latency-avg", defaultGroupName, "The average process time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("process-latency-max", defaultGroupName, "The maximum process time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("process-rate", defaultGroupName, "The average per-second number of process calls", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-latency-avg", defaultGroupName, "The average punctuate time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-latency-max", defaultGroupName, "The maximum punctuate time in ms", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("punctuate-rate", defaultGroupName, "The average per-second number of punctuate calls", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("task-created-rate", defaultGroupName, "The average per-second number of newly created tasks", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("task-closed-rate", defaultGroupName, "The average per-second number of closed tasks", defaultTags)));
    assertNotNull(metrics.metrics().get(metrics.metricName("skipped-records-rate", defaultGroupName, "The average per-second number of skipped records.", defaultTags)));
}