Examples with Connector org.apache.kafka.connect.connector.Connector used on opensource projects

Example 41 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class Worker method startConnector.

/**
 * Start a connector managed by this worker.
 *
 * @param connName the connector name.
 * @param connProps the properties of the connector.
 * @param ctx the connector runtime context.
 * @param statusListener a listener for the runtime status transitions of the connector.
 * @param initialState the initial state of the connector.
 * @return true if the connector started successfully.
 */
public boolean startConnector(String connName, Map<String, String> connProps, ConnectorContext ctx, ConnectorStatus.Listener statusListener, TargetState initialState) {
    if (connectors.containsKey(connName))
        throw new ConnectException("Connector with name " + connName + " already exists");
    final WorkerConnector workerConnector;
    ClassLoader savedLoader = plugins.currentThreadLoader();
    try {
        final ConnectorConfig connConfig = new ConnectorConfig(plugins, connProps);
        final String connClass = connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
        log.info("Creating connector {} of type {}", connName, connClass);
        final Connector connector = plugins.newConnector(connClass);
        workerConnector = new WorkerConnector(connName, connector, ctx, metrics, statusListener);
        log.info("Instantiated connector {} with version {} of type {}", connName, connector.version(), connector.getClass());
        savedLoader = plugins.compareAndSwapLoaders(connector);
        workerConnector.initialize(connConfig);
        workerConnector.transitionTo(initialState);
        Plugins.compareAndSwapLoaders(savedLoader);
    } catch (Throwable t) {
        log.error("Failed to start connector {}", connName, t);
        // Can't be put in a finally block because it needs to be swapped before the call on
        // statusListener
        Plugins.compareAndSwapLoaders(savedLoader);
        workerMetricsGroup.recordConnectorStartupFailure();
        statusListener.onFailure(connName, t);
        return false;
    }
    WorkerConnector existing = connectors.putIfAbsent(connName, workerConnector);
    if (existing != null)
        throw new ConnectException("Connector with name " + connName + " already exists");
    log.info("Finished creating connector {}", connName);
    workerMetricsGroup.recordConnectorStartupSuccess();
    return true;
}

Example 42 with Connector

use of org.apache.kafka.connect.connector.Connector in project connect-utils by jcustenborder.

the class PluginLoader method configDef.

// ConfigDef transformationConfig(Class<? extends Transformation> transformation) {
// try {
// return transformation.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
// 
// ConfigDef connectorConfig(Class<? extends Connector> connectorClass) {
// try {
// return connectorClass.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
ConfigDef configDef(Class<?> pluginClass) {
    ConfigDef result;
    try {
        if (Connector.class.isAssignableFrom(pluginClass)) {
            Connector connector = (Connector) pluginClass.newInstance();
            result = connector.config();
        } else if (Transformation.class.isAssignableFrom(pluginClass)) {
            Transformation<?> connector = (Transformation<?>) pluginClass.newInstance();
            result = connector.config();
        } else if (Converter.class.isAssignableFrom(pluginClass) || ConfigProvider.class.isAssignableFrom(pluginClass)) {
            Optional<Method> optionalConfigMethod = Stream.of(pluginClass.getMethods()).filter(method -> Modifier.isPublic(method.getModifiers())).filter(method -> ConfigDef.class.equals(method.getReturnType())).findFirst();
            if (optionalConfigMethod.isPresent()) {
                Method configMethod = optionalConfigMethod.get();
                Object instance;
                if (Modifier.isStatic(configMethod.getModifiers())) {
                    instance = null;
                } else {
                    instance = pluginClass.newInstance();
                }
                result = (ConfigDef) configMethod.invoke(instance);
            } else {
                result = null;
            }
        } else {
            throw new UnsupportedOperationException(String.format("Type %s is not supported", pluginClass.getName()));
        }
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new IllegalStateException(e);
    }
    return result;
}

Example 43 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class Worker method startConnector.

/**
     * Start a connector managed by this worker.
     *
     * @param connName the connector name.
     * @param connProps the properties of the connector.
     * @param ctx the connector runtime context.
     * @param statusListener a listener for the runtime status transitions of the connector.
     * @param initialState the initial state of the connector.
     * @return true if the connector started successfully.
     */
public boolean startConnector(String connName, Map<String, String> connProps, ConnectorContext ctx, ConnectorStatus.Listener statusListener, TargetState initialState) {
    if (connectors.containsKey(connName))
        throw new ConnectException("Connector with name " + connName + " already exists");
    final WorkerConnector workerConnector;
    try {
        final ConnectorConfig connConfig = new ConnectorConfig(connProps);
        final String connClass = connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
        log.info("Creating connector {} of type {}", connName, connClass);
        final Connector connector = connectorFactory.newConnector(connClass);
        workerConnector = new WorkerConnector(connName, connector, ctx, statusListener);
        log.info("Instantiated connector {} with version {} of type {}", connName, connector.version(), connector.getClass());
        workerConnector.initialize(connConfig);
        workerConnector.transitionTo(initialState);
    } catch (Throwable t) {
        log.error("Failed to start connector {}", connName, t);
        statusListener.onFailure(connName, t);
        return false;
    }
    WorkerConnector existing = connectors.putIfAbsent(connName, workerConnector);
    if (existing != null)
        throw new ConnectException("Connector with name " + connName + " already exists");
    log.info("Finished creating connector {}", connName);
    return true;
}

Example 44 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorConfig) {
    String connType = connectorConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorConfig + " contains no connector type");
    Connector connector = getConnector(connType);
    final ConfigDef connectorConfigDef = ConnectorConfig.enrich((connector instanceof SourceConnector) ? SourceConnectorConfig.configDef() : SinkConnectorConfig.configDef(), connectorConfig, false);
    List<ConfigValue> configValues = new ArrayList<>();
    Map<String, ConfigKey> configKeys = new HashMap<>();
    List<String> allGroups = new ArrayList<>();
    // do basic connector validation (name, connector type, etc.)
    Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, connectorConfigDef, connectorConfig);
    configValues.addAll(validatedConnectorConfig.values());
    configKeys.putAll(connectorConfigDef.configKeys());
    allGroups.addAll(connectorConfigDef.groups());
    // do custom connector-specific validation
    Config config = connector.validate(connectorConfig);
    ConfigDef configDef = connector.config();
    configKeys.putAll(configDef.configKeys());
    allGroups.addAll(configDef.groups());
    configValues.addAll(config.configValues());
    return generateResult(connType, configKeys, configValues, allGroups);
}

Example 45 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class StandaloneHerderTest method expectConfigValidation.

private void expectConfigValidation(Map<String, String>... configs) {
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    for (Map<String, String> config : configs) EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
}