Examples with TopicStatus org.apache.kafka.connect.runtime.TopicStatus used on opensource projects

Example 1 with TopicStatus

use of org.apache.kafka.connect.runtime.TopicStatus in project kafka by apache.

the class KafkaStatusBackingStore method parseTopicStatus.

protected TopicStatus parseTopicStatus(byte[] data) {
    try {
        SchemaAndValue schemaAndValue = converter.toConnectData(statusTopic, data);
        if (!(schemaAndValue.value() instanceof Map)) {
            log.error("Invalid topic status value {}", schemaAndValue.value());
            return null;
        }
        @SuppressWarnings("unchecked") Object innerValue = ((Map<String, Object>) schemaAndValue.value()).get(TOPIC_STATE_KEY);
        if (!(innerValue instanceof Map)) {
            log.error("Invalid topic status value {} for field {}", innerValue, TOPIC_STATE_KEY);
            return null;
        }
        @SuppressWarnings("unchecked") Map<String, Object> topicStatusMetadata = (Map<String, Object>) innerValue;
        return new TopicStatus((String) topicStatusMetadata.get(TOPIC_NAME_KEY), (String) topicStatusMetadata.get(TOPIC_CONNECTOR_KEY), ((Long) topicStatusMetadata.get(TOPIC_TASK_KEY)).intValue(), (long) topicStatusMetadata.get(TOPIC_DISCOVER_TIMESTAMP_KEY));
    } catch (Exception e) {
        log.error("Failed to deserialize topic status", e);
        return null;
    }
}

Example 2 with TopicStatus

use of org.apache.kafka.connect.runtime.TopicStatus in project kafka by apache.

the class KafkaStatusBackingStore method readTopicStatus.

private void readTopicStatus(String key, byte[] value) {
    int delimiterPos = key.indexOf(':');
    int beginPos = TOPIC_STATUS_PREFIX.length();
    if (beginPos > delimiterPos) {
        log.warn("Discarding record with invalid topic status key {}", key);
        return;
    }
    String topic = key.substring(beginPos, delimiterPos);
    if (topic.isEmpty()) {
        log.warn("Discarding record with invalid topic status key containing empty topic {}", key);
        return;
    }
    beginPos = delimiterPos + TOPIC_STATUS_SEPARATOR.length();
    int endPos = key.length();
    if (beginPos > endPos) {
        log.warn("Discarding record with invalid topic status key {}", key);
        return;
    }
    String connector = key.substring(beginPos);
    if (connector.isEmpty()) {
        log.warn("Discarding record with invalid topic status key containing empty connector {}", key);
        return;
    }
    if (value == null) {
        log.trace("Removing status for topic {} and connector {}", topic, connector);
        removeTopic(topic, connector);
        return;
    }
    TopicStatus status = parseTopicStatus(value);
    if (status == null) {
        log.warn("Failed to parse topic status with key {}", key);
        return;
    }
    log.trace("Received topic status update {}", status);
    topics.computeIfAbsent(connector, k -> new ConcurrentHashMap<>()).put(topic, status);
}

Example 3 with TopicStatus

use of org.apache.kafka.connect.runtime.TopicStatus in project kafka by apache.

the class DistributedHerderTest method testDestroyConnector.

@Test
public void testDestroyConnector() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
    // Start with one connector
    EasyMock.expect(worker.getPlugins()).andReturn(plugins);
    expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    Capture<Callback<TargetState>> onStart = newCapture();
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onStart));
    PowerMock.expectLastCall().andAnswer(() -> {
        onStart.getValue().onCompletion(null, TargetState.STARTED);
        return true;
    });
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
    // And delete the connector
    member.wakeup();
    PowerMock.expectLastCall();
    configBackingStore.removeConnectorConfig(CONN1);
    PowerMock.expectLastCall();
    putConnectorCallback.onCompletion(null, new Herder.Created<>(false, null));
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // The change eventually is reflected to the config topic and the deleted connector and
    // tasks are revoked
    member.wakeup();
    PowerMock.expectLastCall();
    TopicStatus fooStatus = new TopicStatus(FOO_TOPIC, CONN1, 0, time.milliseconds());
    TopicStatus barStatus = new TopicStatus(BAR_TOPIC, CONN1, 0, time.milliseconds());
    EasyMock.expect(statusBackingStore.getAllTopics(EasyMock.eq(CONN1))).andReturn(new HashSet<>(Arrays.asList(fooStatus, barStatus))).times(2);
    statusBackingStore.deleteTopic(EasyMock.eq(CONN1), EasyMock.eq(FOO_TOPIC));
    PowerMock.expectLastCall().times(2);
    statusBackingStore.deleteTopic(EasyMock.eq(CONN1), EasyMock.eq(BAR_TOPIC));
    PowerMock.expectLastCall().times(2);
    expectRebalance(Arrays.asList(CONN1), Arrays.asList(TASK1), ConnectProtocol.Assignment.NO_ERROR, 2, Collections.emptyList(), Collections.emptyList(), 0);
    expectPostRebalanceCatchup(ClusterConfigState.EMPTY);
    member.requestRejoin();
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.deleteConnectorConfig(CONN1, putConnectorCallback);
    herder.tick();
    time.sleep(1000L);
    assertStatistics("leaderUrl", false, 3, 1, 100, 1000L);
    // read updated config that removes the connector
    configUpdateListener.onConnectorConfigRemove(CONN1);
    herder.configState = ClusterConfigState.EMPTY;
    herder.tick();
    time.sleep(1000L);
    assertStatistics("leaderUrl", true, 3, 1, 100, 2100L);
    PowerMock.verifyAll();
}

Example 4 with TopicStatus

use of org.apache.kafka.connect.runtime.TopicStatus in project kafka by apache.

the class KafkaStatusBackingStoreFormatTest method putTopicStateRetriableFailure.

@Test
public void putTopicStateRetriableFailure() {
    TopicStatus topicStatus = new TopicStatus(FOO_TOPIC, new ConnectorTaskId(FOO_CONNECTOR, 0), time.milliseconds());
    String key = TOPIC_STATUS_PREFIX + FOO_TOPIC + TOPIC_STATUS_SEPARATOR + FOO_CONNECTOR;
    ArgumentCaptor<byte[]> valueCaptor = ArgumentCaptor.forClass(byte[].class);
    doAnswer(invocation -> {
        ((Callback) invocation.getArgument(2)).onCompletion(null, new TimeoutException());
        return null;
    }).doAnswer(invocation -> {
        ((Callback) invocation.getArgument(2)).onCompletion(null, null);
        return null;
    }).when(kafkaBasedLog).send(eq(key), valueCaptor.capture(), any(Callback.class));
    store.put(topicStatus);
    verify(kafkaBasedLog, times(2)).send(any(), any(), any());
    // check capture state
    assertEquals(topicStatus, store.parseTopicStatus(valueCaptor.getValue()));
    // state is not visible until read back from the log
    assertNull(store.getTopic(FOO_CONNECTOR, FOO_TOPIC));
}

Example 5 with TopicStatus

use of org.apache.kafka.connect.runtime.TopicStatus in project kafka by apache.

the class KafkaStatusBackingStoreFormatTest method readTopicStatus.

@Test
public void readTopicStatus() {
    TopicStatus topicStatus = new TopicStatus(FOO_TOPIC, new ConnectorTaskId(FOO_CONNECTOR, 0), Time.SYSTEM.milliseconds());
    String key = TOPIC_STATUS_PREFIX + FOO_TOPIC + TOPIC_STATUS_SEPARATOR + FOO_CONNECTOR;
    byte[] value = store.serializeTopicStatus(topicStatus);
    ConsumerRecord<String, byte[]> statusRecord = new ConsumerRecord<>(STATUS_TOPIC, 0, 0, key, value);
    store.read(statusRecord);
    assertTrue(store.topics.containsKey(FOO_CONNECTOR));
    assertTrue(store.topics.get(FOO_CONNECTOR).containsKey(FOO_TOPIC));
    assertEquals(topicStatus, store.topics.get(FOO_CONNECTOR).get(FOO_TOPIC));
}