Examples with Plugins org.apache.kafka.connect.runtime.isolation.Plugins used on opensource projects

Example 16 with Plugins

use of org.apache.kafka.connect.runtime.isolation.Plugins in project apache-kafka-on-k8s by banzaicloud.

the class ConnectDistributed method main.

public static void main(String[] args) throws Exception {
    if (args.length < 1) {
        log.info("Usage: ConnectDistributed worker.properties");
        Exit.exit(1);
    }
    try {
        Time time = Time.SYSTEM;
        log.info("Kafka Connect distributed worker initializing ...");
        long initStart = time.hiResClockMs();
        WorkerInfo initInfo = new WorkerInfo();
        initInfo.logAll();
        String workerPropsFile = args[0];
        Map<String, String> workerProps = !workerPropsFile.isEmpty() ? Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.<String, String>emptyMap();
        log.info("Scanning for plugin classes. This might take a moment ...");
        Plugins plugins = new Plugins(workerProps);
        plugins.compareAndSwapWithDelegatingLoader();
        DistributedConfig config = new DistributedConfig(workerProps);
        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
        log.debug("Kafka cluster ID: {}", kafkaClusterId);
        RestServer rest = new RestServer(config);
        URI advertisedUrl = rest.advertisedUrl();
        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
        KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
        offsetBackingStore.configure(config);
        Worker worker = new Worker(workerId, time, plugins, config, offsetBackingStore);
        Converter internalValueConverter = worker.getInternalValueConverter();
        StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
        statusBackingStore.configure(config);
        ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(internalValueConverter, config);
        DistributedHerder herder = new DistributedHerder(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl.toString());
        final Connect connect = new Connect(herder, rest);
        log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
        try {
            connect.start();
        } catch (Exception e) {
            log.error("Failed to start Connect", e);
            connect.stop();
            Exit.exit(3);
        }
        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
        connect.awaitStop();
    } catch (Throwable t) {
        log.error("Stopping due to error", t);
        Exit.exit(2);
    }
}

Example 17 with Plugins

use of org.apache.kafka.connect.runtime.isolation.Plugins in project apache-kafka-on-k8s by banzaicloud.

the class WorkerSourceTaskTest method setup.

@Override
public void setup() {
    super.setup();
    Map<String, String> workerProps = new HashMap<>();
    workerProps.put("key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter.schemas.enable", "false");
    workerProps.put("internal.value.converter.schemas.enable", "false");
    workerProps.put("offset.storage.file.filename", "/tmp/connect.offsets");
    plugins = new Plugins(workerProps);
    config = new StandaloneConfig(workerProps);
    producerCallbacks = EasyMock.newCapture();
    metrics = new MockConnectMetrics();
}

Example 18 with Plugins

use of org.apache.kafka.connect.runtime.isolation.Plugins in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method setUp.

@Before
public void setUp() throws Exception {
    time = new MockTime();
    metrics = new MockConnectMetrics(time);
    worker = PowerMock.createMock(Worker.class);
    EasyMock.expect(worker.isSinkConnector(CONN1)).andStubReturn(Boolean.TRUE);
    herder = PowerMock.createPartialMock(DistributedHerder.class, new String[] { "backoff", "connectorTypeForClass", "updateDeletedConnectorStatus" }, new DistributedConfig(HERDER_CONFIG), worker, WORKER_ID, KAFKA_CLUSTER_ID, statusBackingStore, configBackingStore, member, MEMBER_URL, metrics, time);
    configUpdateListener = herder.new ConfigUpdateListener();
    rebalanceListener = herder.new RebalanceListener();
    plugins = PowerMock.createMock(Plugins.class);
    conn1SinkConfig = new SinkConnectorConfig(plugins, CONN1_CONFIG);
    conn1SinkConfigUpdated = new SinkConnectorConfig(plugins, CONN1_CONFIG_UPDATED);
    EasyMock.expect(herder.connectorTypeForClass(BogusSourceConnector.class.getName())).andReturn(ConnectorType.SOURCE).anyTimes();
    pluginLoader = PowerMock.createMock(PluginClassLoader.class);
    delegatingLoader = PowerMock.createMock(DelegatingClassLoader.class);
    PowerMock.mockStatic(Plugins.class);
    PowerMock.expectPrivate(herder, "updateDeletedConnectorStatus").andVoid().anyTimes();
}

Example 19 with Plugins

use of org.apache.kafka.connect.runtime.isolation.Plugins in project apache-kafka-on-k8s by banzaicloud.

the class TransformationConfigTest method testEmbeddedConfigValueToKey.

@Test
public void testEmbeddedConfigValueToKey() {
    // Validate that we can construct a Connector config containing the extended config for the transform
    HashMap<String, String> connProps = new HashMap<>();
    connProps.put("name", "foo");
    connProps.put("connector.class", MockConnector.class.getName());
    connProps.put("transforms", "example");
    connProps.put("transforms.example.type", ValueToKey.class.getName());
    connProps.put("transforms.example.fields", "field");
    // Safe when we're only constructing the config
    Plugins plugins = null;
    new ConnectorConfig(plugins, connProps);
}

Example 20 with Plugins

use of org.apache.kafka.connect.runtime.isolation.Plugins in project apache-kafka-on-k8s by banzaicloud.

the class TransformationConfigTest method testEmbeddedConfigRegexRouter.

@Test
public void testEmbeddedConfigRegexRouter() {
    // Validate that we can construct a Connector config containing the extended config for the transform
    HashMap<String, String> connProps = new HashMap<>();
    connProps.put("name", "foo");
    connProps.put("connector.class", MockConnector.class.getName());
    connProps.put("transforms", "example");
    connProps.put("transforms.example.type", RegexRouter.class.getName());
    connProps.put("transforms.example.regex", "(.*)");
    connProps.put("transforms.example.replacement", "prefix-$1");
    // Safe when we're only constructing the config
    Plugins plugins = null;
    new ConnectorConfig(plugins, connProps);
}