Examples with Connector org.apache.kafka.connect.connector.Connector used on opensource projects

Example 26 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class ConnectorPluginsResourceTest method testValidateConfigWithSingleErrorDueToMissingConnectorClassname.

@Test
public void testValidateConfigWithSingleErrorDueToMissingConnectorClassname() throws Throwable {
    Capture<Callback<ConfigInfos>> configInfosCallback = EasyMock.newCapture();
    herder.validateConnectorConfig(EasyMock.eq(partialProps), EasyMock.capture(configInfosCallback), EasyMock.anyBoolean());
    PowerMock.expectLastCall().andAnswer((IAnswer<Void>) () -> {
        ConfigDef connectorConfigDef = ConnectorConfig.configDef();
        List<ConfigValue> connectorConfigValues = connectorConfigDef.validate(partialProps);
        Connector connector = new ConnectorPluginsResourceTestConnector();
        Config config = connector.validate(partialProps);
        ConfigDef configDef = connector.config();
        Map<String, ConfigDef.ConfigKey> configKeys = configDef.configKeys();
        List<ConfigValue> configValues = config.configValues();
        Map<String, ConfigDef.ConfigKey> resultConfigKeys = new HashMap<>(configKeys);
        resultConfigKeys.putAll(connectorConfigDef.configKeys());
        configValues.addAll(connectorConfigValues);
        ConfigInfos configInfos = AbstractHerder.generateResult(ConnectorPluginsResourceTestConnector.class.getName(), resultConfigKeys, configValues, Collections.singletonList("Test"));
        configInfosCallback.getValue().onCompletion(null, configInfos);
        return null;
    });
    PowerMock.replayAll();
    // This call to validateConfigs does not throw a BadRequestException because we've mocked
    // validateConnectorConfig.
    ConfigInfos configInfos = connectorPluginsResource.validateConfigs(ConnectorPluginsResourceTestConnector.class.getSimpleName(), partialProps);
    assertEquals(PARTIAL_CONFIG_INFOS.name(), configInfos.name());
    assertEquals(PARTIAL_CONFIG_INFOS.errorCount(), configInfos.errorCount());
    assertEquals(PARTIAL_CONFIG_INFOS.groups(), configInfos.groups());
    assertEquals(new HashSet<>(PARTIAL_CONFIG_INFOS.values()), new HashSet<>(configInfos.values()));
    PowerMock.verifyAll();
}

Example 27 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedValidation.

@Test
public void testCreateConnectorFailedValidation() {
    // Basic validation should be performed and return an error, but should still evaluate the connector's config
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    config.remove(ConnectorConfig.NAME_CONFIG);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
    final Capture<Map<String, String>> configCapture = EasyMock.newCapture();
    EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    ExecutionException exception = assertThrows(ExecutionException.class, () -> createCallback.get(1000L, TimeUnit.SECONDS));
    assertEquals(BadRequestException.class, exception.getCause().getClass());
    PowerMock.verifyAll();
}

Example 28 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class StandaloneHerderTest method testRestartConnectorAndTasksNoStatus.

@Test
public void testRestartConnectorAndTasksNoStatus() throws Exception {
    RestartRequest restartRequest = new RestartRequest(CONNECTOR_NAME, false, true);
    EasyMock.expect(herder.buildRestartPlan(restartRequest)).andReturn(Optional.empty()).anyTimes();
    connector = PowerMock.createMock(BogusSinkConnector.class);
    expectAdd(SourceSink.SINK);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SINK);
    Connector connectorMock = PowerMock.createMock(SinkConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SINK), connectorInfo.result());
    FutureCallback<ConnectorStateInfo> restartCallback = new FutureCallback<>();
    herder.restartConnectorAndTasks(restartRequest, restartCallback);
    ExecutionException ee = assertThrows(ExecutionException.class, () -> restartCallback.get(1000L, TimeUnit.MILLISECONDS));
    assertTrue(ee.getCause() instanceof NotFoundException);
    assertTrue(ee.getMessage().contains("Status for connector"));
    PowerMock.verifyAll();
}

Example 29 with Connector

use of org.apache.kafka.connect.connector.Connector in project connect-utils by jcustenborder.

the class PluginLoader method configDef.

// ConfigDef transformationConfig(Class<? extends Transformation> transformation) {
// try {
// return transformation.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
// 
// ConfigDef connectorConfig(Class<? extends Connector> connectorClass) {
// try {
// return connectorClass.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
ConfigDef configDef(Class<?> pluginClass) {
    ConfigDef result;
    try {
        if (Connector.class.isAssignableFrom(pluginClass)) {
            Connector connector = (Connector) pluginClass.newInstance();
            result = connector.config();
        } else if (Transformation.class.isAssignableFrom(pluginClass)) {
            Transformation<?> connector = (Transformation<?>) pluginClass.newInstance();
            result = connector.config();
        } else if (Converter.class.isAssignableFrom(pluginClass) || ConfigProvider.class.isAssignableFrom(pluginClass)) {
            Optional<Method> optionalConfigMethod = Stream.of(pluginClass.getMethods()).filter(method -> Modifier.isPublic(method.getModifiers())).filter(method -> ConfigDef.class.equals(method.getReturnType())).findFirst();
            if (optionalConfigMethod.isPresent()) {
                Method configMethod = optionalConfigMethod.get();
                Object instance;
                if (Modifier.isStatic(configMethod.getModifiers())) {
                    instance = null;
                } else {
                    instance = pluginClass.newInstance();
                }
                result = (ConfigDef) configMethod.invoke(instance);
            } else {
                result = null;
            }
        } else {
            throw new UnsupportedOperationException(String.format("Type %s is not supported", pluginClass.getName()));
        }
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new IllegalStateException(e);
    }
    return result;
}

Example 30 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method testCreateConnector.

@Test
public void testCreateConnector() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall();
    // config validation
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(CONN2_CONFIG)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    // CONN2 is new, should succeed
    configBackingStore.putConnectorConfig(CONN2, CONN2_CONFIG);
    PowerMock.expectLastCall();
    ConnectorInfo info = new ConnectorInfo(CONN2, CONN2_CONFIG, Collections.<ConnectorTaskId>emptyList(), ConnectorType.SOURCE);
    putConnectorCallback.onCompletion(null, new Herder.Created<>(true, info));
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, CONN2_CONFIG, false, putConnectorCallback);
    herder.tick();
    time.sleep(1000L);
    assertStatistics(3, 1, 100, 1000L);
    PowerMock.verifyAll();
}