Examples with Connector org.apache.kafka.connect.connector.Connector used on opensource projects

Example 11 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class WorkerTest method testAddConnectorByShortAlias.

@Test
public void testAddConnectorByShortAlias() throws Exception {
    expectStartStorage();
    // Create
    Connector connector = PowerMock.createMock(Connector.class);
    ConnectorContext ctx = PowerMock.createMock(ConnectorContext.class);
    EasyMock.expect(connectorFactory.newConnector("WorkerTest")).andReturn(connector);
    EasyMock.expect(connector.version()).andReturn("1.0");
    Map<String, String> props = new HashMap<>();
    props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
    props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
    props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
    props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, "WorkerTest");
    connector.initialize(EasyMock.anyObject(ConnectorContext.class));
    EasyMock.expectLastCall();
    connector.start(props);
    EasyMock.expectLastCall();
    connectorStatusListener.onStartup(CONNECTOR_ID);
    EasyMock.expectLastCall();
    // Remove
    connector.stop();
    EasyMock.expectLastCall();
    connectorStatusListener.onShutdown(CONNECTOR_ID);
    EasyMock.expectLastCall();
    expectStopStorage();
    PowerMock.replayAll();
    worker = new Worker(WORKER_ID, new MockTime(), connectorFactory, config, offsetBackingStore);
    worker.start();
    assertEquals(Collections.emptySet(), worker.connectorNames());
    worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
    assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_ID)), worker.connectorNames());
    worker.stopConnector(CONNECTOR_ID);
    assertEquals(Collections.emptySet(), worker.connectorNames());
    // Nothing should be left, so this should effectively be a nop
    worker.stop();
    PowerMock.verifyAll();
}

Example 12 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class WorkerTest method testReconfigureConnectorTasks.

@Test
public void testReconfigureConnectorTasks() throws Exception {
    expectStartStorage();
    // Create
    Connector connector = PowerMock.createMock(Connector.class);
    ConnectorContext ctx = PowerMock.createMock(ConnectorContext.class);
    EasyMock.expect(connectorFactory.newConnector(WorkerTestConnector.class.getName())).andReturn(connector);
    EasyMock.expect(connector.version()).andReturn("1.0");
    Map<String, String> props = new HashMap<>();
    props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
    props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
    props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
    props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName());
    connector.initialize(EasyMock.anyObject(ConnectorContext.class));
    EasyMock.expectLastCall();
    connector.start(props);
    EasyMock.expectLastCall();
    connectorStatusListener.onStartup(CONNECTOR_ID);
    EasyMock.expectLastCall();
    // Reconfigure
    EasyMock.<Class<? extends Task>>expect(connector.taskClass()).andReturn(TestSourceTask.class);
    Map<String, String> taskProps = new HashMap<>();
    taskProps.put("foo", "bar");
    EasyMock.expect(connector.taskConfigs(2)).andReturn(Arrays.asList(taskProps, taskProps));
    // Remove
    connector.stop();
    EasyMock.expectLastCall();
    connectorStatusListener.onShutdown(CONNECTOR_ID);
    EasyMock.expectLastCall();
    expectStopStorage();
    PowerMock.replayAll();
    worker = new Worker(WORKER_ID, new MockTime(), connectorFactory, config, offsetBackingStore);
    worker.start();
    assertEquals(Collections.emptySet(), worker.connectorNames());
    worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
    assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_ID)), worker.connectorNames());
    try {
        worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
        fail("Should have thrown exception when trying to add connector with same name.");
    } catch (ConnectException e) {
    // expected
    }
    List<Map<String, String>> taskConfigs = worker.connectorTaskConfigs(CONNECTOR_ID, 2, Arrays.asList("foo", "bar"));
    Map<String, String> expectedTaskProps = new HashMap<>();
    expectedTaskProps.put("foo", "bar");
    expectedTaskProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName());
    expectedTaskProps.put(SinkTask.TOPICS_CONFIG, "foo,bar");
    assertEquals(2, taskConfigs.size());
    assertEquals(expectedTaskProps, taskConfigs.get(0));
    assertEquals(expectedTaskProps, taskConfigs.get(1));
    worker.stopConnector(CONNECTOR_ID);
    assertEquals(Collections.emptySet(), worker.connectorNames());
    // Nothing should be left, so this should effectively be a nop
    worker.stop();
    PowerMock.verifyAll();
}

Example 13 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class WorkerTest method testStartAndStopConnector.

@Test
public void testStartAndStopConnector() throws Exception {
    expectStartStorage();
    // Create
    Connector connector = PowerMock.createMock(Connector.class);
    ConnectorContext ctx = PowerMock.createMock(ConnectorContext.class);
    EasyMock.expect(connectorFactory.newConnector(WorkerTestConnector.class.getName())).andReturn(connector);
    EasyMock.expect(connector.version()).andReturn("1.0");
    Map<String, String> props = new HashMap<>();
    props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
    props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
    props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
    props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName());
    connector.initialize(EasyMock.anyObject(ConnectorContext.class));
    EasyMock.expectLastCall();
    connector.start(props);
    EasyMock.expectLastCall();
    connectorStatusListener.onStartup(CONNECTOR_ID);
    EasyMock.expectLastCall();
    // Remove
    connector.stop();
    EasyMock.expectLastCall();
    connectorStatusListener.onShutdown(CONNECTOR_ID);
    EasyMock.expectLastCall();
    expectStopStorage();
    PowerMock.replayAll();
    worker = new Worker(WORKER_ID, new MockTime(), connectorFactory, config, offsetBackingStore);
    worker.start();
    assertEquals(Collections.emptySet(), worker.connectorNames());
    worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
    assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_ID)), worker.connectorNames());
    try {
        worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
        fail("Should have thrown exception when trying to add connector with same name.");
    } catch (ConnectException e) {
    // expected
    }
    worker.stopConnector(CONNECTOR_ID);
    assertEquals(Collections.emptySet(), worker.connectorNames());
    // Nothing should be left, so this should effectively be a nop
    worker.stop();
    PowerMock.verifyAll();
}

Example 14 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorConfig) {
    String connType = connectorConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorConfig + " contains no connector type");
    Connector connector = getConnector(connType);
    final ConfigDef connectorConfigDef = ConnectorConfig.enrich((connector instanceof SourceConnector) ? SourceConnectorConfig.configDef() : SinkConnectorConfig.configDef(), connectorConfig, false);
    List<ConfigValue> configValues = new ArrayList<>();
    Map<String, ConfigKey> configKeys = new HashMap<>();
    List<String> allGroups = new ArrayList<>();
    // do basic connector validation (name, connector type, etc.)
    Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, connectorConfigDef, connectorConfig);
    configValues.addAll(validatedConnectorConfig.values());
    configKeys.putAll(connectorConfigDef.configKeys());
    allGroups.addAll(connectorConfigDef.groups());
    // do custom connector-specific validation
    Config config = connector.validate(connectorConfig);
    ConfigDef configDef = connector.config();
    configKeys.putAll(configDef.configKeys());
    allGroups.addAll(configDef.groups());
    configValues.addAll(config.configValues());
    return generateResult(connType, configKeys, configValues, allGroups);
}

Example 15 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class StandaloneHerderTest method expectConfigValidation.

private void expectConfigValidation(Map<String, String>... configs) {
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    for (Map<String, String> config : configs) EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
}