Examples with Connector org.apache.kafka.connect.connector.Connector used on opensource projects

Example 71 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class ConnectorPluginsResourceTest method testValidateConfigWithAlias.

@Test
public void testValidateConfigWithAlias() throws Throwable {
    @SuppressWarnings("unchecked") ArgumentCaptor<Callback<ConfigInfos>> configInfosCallback = ArgumentCaptor.forClass(Callback.class);
    doAnswer(invocation -> {
        ConfigDef connectorConfigDef = ConnectorConfig.configDef();
        List<ConfigValue> connectorConfigValues = connectorConfigDef.validate(PROPS);
        Connector connector = new ConnectorPluginsResourceTestConnector();
        Config config = connector.validate(PROPS);
        ConfigDef configDef = connector.config();
        Map<String, ConfigDef.ConfigKey> configKeys = configDef.configKeys();
        List<ConfigValue> configValues = config.configValues();
        Map<String, ConfigDef.ConfigKey> resultConfigKeys = new HashMap<>(configKeys);
        resultConfigKeys.putAll(connectorConfigDef.configKeys());
        configValues.addAll(connectorConfigValues);
        ConfigInfos configInfos = AbstractHerder.generateResult(ConnectorPluginsResourceTestConnector.class.getName(), resultConfigKeys, configValues, Collections.singletonList("Test"));
        configInfosCallback.getValue().onCompletion(null, configInfos);
        return null;
    }).when(herder).validateConnectorConfig(eq(PROPS), configInfosCallback.capture(), anyBoolean());
    // make a request to connector-plugins resource using a valid alias.
    ConfigInfos configInfos = connectorPluginsResource.validateConfigs("ConnectorPluginsResourceTest", PROPS);
    assertEquals(CONFIG_INFOS.name(), configInfos.name());
    assertEquals(0, configInfos.errorCount());
    assertEquals(CONFIG_INFOS.groups(), configInfos.groups());
    assertEquals(new HashSet<>(CONFIG_INFOS.values()), new HashSet<>(configInfos.values()));
    verify(herder).validateConnectorConfig(eq(PROPS), any(), anyBoolean());
}

Example 72 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class AbstractHerderTest method testConfigValidationTransformsExtendResults.

@Test()
public void testConfigValidationTransformsExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(SampleSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(SampleSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, SourceConnectorConfig.OFFSETS_TOPIC_GROUP, "Transforms: xformA", "Transforms: xformB");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(26, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformB.type", infos.get("transforms.xformB.type").configValue().name());
    assertFalse(infos.get("transforms.xformB.type").configValue().errors().isEmpty());
    verifyAll();
}

Example 73 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class AbstractHerderTest method testConfigValidationPredicatesExtendResults.

@Test()
public void testConfigValidationPredicatesExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(SampleSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(1);
    Set<PluginDesc<Predicate<?>>> predicates = new HashSet<>();
    predicates.add(predicatePluginDesc());
    EasyMock.expect(plugins.predicates()).andReturn(predicates).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, SampleSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.predicate", "predX");
    config.put(ConnectorConfig.PREDICATES_CONFIG, "predX,predY");
    config.put(ConnectorConfig.PREDICATES_CONFIG + ".predX.type", SamplePredicate.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(SampleSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, SourceConnectorConfig.OFFSETS_TOPIC_GROUP, "Transforms: xformA", "Predicates: predX", "Predicates: predY");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(28, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformA.predicate", infos.get("transforms.xformA.predicate").configValue().name());
    assertTrue(infos.get("transforms.xformA.predicate").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.negate", infos.get("transforms.xformA.negate").configValue().name());
    assertTrue(infos.get("transforms.xformA.negate").configValue().errors().isEmpty());
    assertEquals("predicates.predX.type", infos.get("predicates.predX.type").configValue().name());
    assertEquals("predicates.predX.predconfig", infos.get("predicates.predX.predconfig").configValue().name());
    assertEquals("predicates.predY.type", infos.get("predicates.predY.type").configValue().name());
    assertFalse(infos.get("predicates.predY.type").configValue().errors().isEmpty());
    verifyAll();
}

Example 74 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class AbstractHerderTest method createConfigValidationHerder.

private AbstractHerder createConfigValidationHerder(Class<? extends Connector> connectorClass, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, int countOfCallingNewConnector) {
    ConfigBackingStore configStore = strictMock(ConfigBackingStore.class);
    StatusBackingStore statusStore = strictMock(StatusBackingStore.class);
    AbstractHerder herder = partialMockBuilder(AbstractHerder.class).withConstructor(Worker.class, String.class, String.class, StatusBackingStore.class, ConfigBackingStore.class, ConnectorClientConfigOverridePolicy.class).withArgs(worker, workerId, kafkaClusterId, statusStore, configStore, connectorClientConfigOverridePolicy).addMockedMethod("generation").createMock();
    EasyMock.expect(herder.generation()).andStubReturn(generation);
    // Call to validateConnectorConfig
    EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
    final Capture<Map<String, String>> configCapture = EasyMock.newCapture();
    EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
    EasyMock.expect(worker.getPlugins()).andStubReturn(plugins);
    final Connector connector;
    try {
        connector = connectorClass.getConstructor().newInstance();
    } catch (ReflectiveOperationException e) {
        throw new RuntimeException("Couldn't create connector", e);
    }
    if (countOfCallingNewConnector > 0) {
        EasyMock.expect(plugins.newConnector(connectorClass.getName())).andReturn(connector).times(countOfCallingNewConnector);
        EasyMock.expect(plugins.compareAndSwapLoaders(connector)).andReturn(classLoader).times(countOfCallingNewConnector);
    }
    return herder;
}

Example 75 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class DistributedHerder method startConnector.

// Helper for starting a connector with the given name, which will extract & parse the config, generate connector
// context and add to the worker. This needs to be called from within the main worker thread for this herder.
// The callback is invoked after the connector has finished startup and generated task configs, or failed in the process.
private void startConnector(String connectorName, Callback<Void> callback) {
    log.info("Starting connector {}", connectorName);
    final Map<String, String> configProps = configState.connectorConfig(connectorName);
    final CloseableConnectorContext ctx = new HerderConnectorContext(this, connectorName);
    final TargetState initialState = configState.targetState(connectorName);
    final Callback<TargetState> onInitialStateChange = (error, newState) -> {
        if (error != null) {
            callback.onCompletion(new ConnectException("Failed to start connector: " + connectorName, error), null);
            return;
        }
        // Use newState here in case the connector has been paused right after being created
        if (newState == TargetState.STARTED) {
            addRequest(() -> {
                // Request configuration since this could be a brand new connector. However, also only update those
                // task configs if they are actually different from the existing ones to avoid unnecessary updates when this is
                // just restoring an existing connector.
                reconfigureConnectorTasksWithRetry(time.milliseconds(), connectorName);
                callback.onCompletion(null, null);
                return null;
            }, forwardErrorCallback(callback));
        } else {
            callback.onCompletion(null, null);
        }
    };
    worker.startConnector(connectorName, configProps, ctx, this, initialState, onInitialStateChange);
}