Examples with StandaloneConfig org.apache.kafka.connect.runtime.standalone.StandaloneConfig used on opensource projects

Example 1 with StandaloneConfig

use of org.apache.kafka.connect.runtime.standalone.StandaloneConfig in project ignite by apache.

the class IgniteSourceConnectorTest method beforeTest.

/**
 * {@inheritDoc}
 */
@Override
protected void beforeTest() throws Exception {
    kafkaBroker = new TestKafkaBroker();
    WorkerConfig workerCfg = new StandaloneConfig(makeWorkerProps());
    MemoryOffsetBackingStore offBackingStore = new MemoryOffsetBackingStore();
    offBackingStore.configure(workerCfg);
    worker = new Worker(WORKER_ID, new SystemTime(), workerCfg, offBackingStore);
    worker.start();
    herder = new StandaloneHerder(worker);
    herder.start();
}

Example 2 with StandaloneConfig

use of org.apache.kafka.connect.runtime.standalone.StandaloneConfig in project apache-kafka-on-k8s by banzaicloud.

the class WorkerSinkTaskThreadedTest method setup.

@Override
public void setup() {
    super.setup();
    time = new MockTime();
    metrics = new MockConnectMetrics();
    Map<String, String> workerProps = new HashMap<>();
    workerProps.put("key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter.schemas.enable", "false");
    workerProps.put("internal.value.converter.schemas.enable", "false");
    workerProps.put("offset.storage.file.filename", "/tmp/connect.offsets");
    pluginLoader = PowerMock.createMock(PluginClassLoader.class);
    workerConfig = new StandaloneConfig(workerProps);
    workerTask = PowerMock.createPartialMock(WorkerSinkTask.class, new String[] { "createConsumer" }, taskId, sinkTask, statusListener, initialState, workerConfig, metrics, keyConverter, valueConverter, headerConverter, TransformationChain.noOp(), pluginLoader, time);
    recordsReturned = 0;
}

Example 3 with StandaloneConfig

use of org.apache.kafka.connect.runtime.standalone.StandaloneConfig in project apache-kafka-on-k8s by banzaicloud.

the class WorkerTest method setup.

@Before
public void setup() {
    super.setup();
    Map<String, String> workerProps = new HashMap<>();
    workerProps.put("key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter.schemas.enable", "false");
    workerProps.put("internal.value.converter.schemas.enable", "false");
    workerProps.put("offset.storage.file.filename", "/tmp/connect.offsets");
    workerProps.put(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
    config = new StandaloneConfig(workerProps);
    PowerMock.mockStatic(Plugins.class);
}

Example 4 with StandaloneConfig

use of org.apache.kafka.connect.runtime.standalone.StandaloneConfig in project apache-kafka-on-k8s by banzaicloud.

the class ConnectStandalone method main.

public static void main(String[] args) throws Exception {
    if (args.length < 2) {
        log.info("Usage: ConnectStandalone worker.properties connector1.properties [connector2.properties ...]");
        Exit.exit(1);
    }
    try {
        Time time = Time.SYSTEM;
        log.info("Kafka Connect standalone worker initializing ...");
        long initStart = time.hiResClockMs();
        WorkerInfo initInfo = new WorkerInfo();
        initInfo.logAll();
        String workerPropsFile = args[0];
        Map<String, String> workerProps = !workerPropsFile.isEmpty() ? Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.<String, String>emptyMap();
        log.info("Scanning for plugin classes. This might take a moment ...");
        Plugins plugins = new Plugins(workerProps);
        plugins.compareAndSwapWithDelegatingLoader();
        StandaloneConfig config = new StandaloneConfig(workerProps);
        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
        log.debug("Kafka cluster ID: {}", kafkaClusterId);
        RestServer rest = new RestServer(config);
        URI advertisedUrl = rest.advertisedUrl();
        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
        Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore());
        Herder herder = new StandaloneHerder(worker, kafkaClusterId);
        final Connect connect = new Connect(herder, rest);
        log.info("Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart);
        try {
            connect.start();
            for (final String connectorPropsFile : Arrays.copyOfRange(args, 1, args.length)) {
                Map<String, String> connectorProps = Utils.propsToStringMap(Utils.loadProps(connectorPropsFile));
                FutureCallback<Herder.Created<ConnectorInfo>> cb = new FutureCallback<>(new Callback<Herder.Created<ConnectorInfo>>() {

                    @Override
                    public void onCompletion(Throwable error, Herder.Created<ConnectorInfo> info) {
                        if (error != null)
                            log.error("Failed to create job for {}", connectorPropsFile);
                        else
                            log.info("Created connector {}", info.result().name());
                    }
                });
                herder.putConnectorConfig(connectorProps.get(ConnectorConfig.NAME_CONFIG), connectorProps, false, cb);
                cb.get();
            }
        } catch (Throwable t) {
            log.error("Stopping after connector error", t);
            connect.stop();
            Exit.exit(3);
        }
        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
        connect.awaitStop();
    } catch (Throwable t) {
        log.error("Stopping due to error", t);
        Exit.exit(2);
    }
}

Example 5 with StandaloneConfig

use of org.apache.kafka.connect.runtime.standalone.StandaloneConfig in project kafka by apache.

the class ErrorHandlingTaskTest method setup.

@Before
public void setup() {
    time = new MockTime(0, 0, 0);
    metrics = new MockConnectMetrics();
    Map<String, String> workerProps = new HashMap<>();
    workerProps.put("key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("offset.storage.file.filename", "/tmp/connect.offsets");
    workerProps.put(TOPIC_CREATION_ENABLE_CONFIG, String.valueOf(enableTopicCreation));
    pluginLoader = PowerMock.createMock(PluginClassLoader.class);
    workerConfig = new StandaloneConfig(workerProps);
    sourceConfig = new SourceConnectorConfig(plugins, sourceConnectorProps(TOPIC), true);
    errorHandlingMetrics = new ErrorHandlingMetrics(taskId, metrics);
}