Examples with ConnectorTaskId org.apache.kafka.connect.util.ConnectorTaskId used on opensource projects

Example 16 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class WorkerCoordinator method performTaskAssignment.

private Map<String, ByteBuffer> performTaskAssignment(String leaderId, long maxOffset, Map<String, ConnectProtocol.WorkerState> memberConfigs) {
    Map<String, List<String>> connectorAssignments = new HashMap<>();
    Map<String, List<ConnectorTaskId>> taskAssignments = new HashMap<>();
    // Perform round-robin task assignment. Assign all connectors and then all tasks because assigning both the
    // connector and its tasks can lead to very uneven distribution of work in some common cases (e.g. for connectors
    // that generate only 1 task each; in a cluster of 2 or an even # of nodes, only even nodes will be assigned
    // connectors and only odd nodes will be assigned tasks, but tasks are, on average, actually more resource
    // intensive than connectors).
    List<String> connectorsSorted = sorted(configSnapshot.connectors());
    CircularIterator<String> memberIt = new CircularIterator<>(sorted(memberConfigs.keySet()));
    for (String connectorId : connectorsSorted) {
        String connectorAssignedTo = memberIt.next();
        log.trace("Assigning connector {} to {}", connectorId, connectorAssignedTo);
        List<String> memberConnectors = connectorAssignments.get(connectorAssignedTo);
        if (memberConnectors == null) {
            memberConnectors = new ArrayList<>();
            connectorAssignments.put(connectorAssignedTo, memberConnectors);
        }
        memberConnectors.add(connectorId);
    }
    for (String connectorId : connectorsSorted) {
        for (ConnectorTaskId taskId : sorted(configSnapshot.tasks(connectorId))) {
            String taskAssignedTo = memberIt.next();
            log.trace("Assigning task {} to {}", taskId, taskAssignedTo);
            List<ConnectorTaskId> memberTasks = taskAssignments.get(taskAssignedTo);
            if (memberTasks == null) {
                memberTasks = new ArrayList<>();
                taskAssignments.put(taskAssignedTo, memberTasks);
            }
            memberTasks.add(taskId);
        }
    }
    this.leaderState = new LeaderState(memberConfigs, connectorAssignments, taskAssignments);
    return fillAssignmentsAndSerialize(memberConfigs.keySet(), ConnectProtocol.Assignment.NO_ERROR, leaderId, memberConfigs.get(leaderId).url(), maxOffset, connectorAssignments, taskAssignments);
}

Example 17 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class MemoryStatusBackingStoreTest method deleteTaskStatus.

@Test
public void deleteTaskStatus() {
    MemoryStatusBackingStore store = new MemoryStatusBackingStore();
    ConnectorTaskId taskId = new ConnectorTaskId("connector", 0);
    store.put(new TaskStatus(taskId, ConnectorStatus.State.RUNNING, "localhost:8083", 0));
    store.put(new TaskStatus(taskId, ConnectorStatus.State.DESTROYED, "localhost:8083", 0));
    assertNull(store.get(taskId));
}

Example 18 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method expectStop.

private void expectStop() {
    ConnectorTaskId task = new ConnectorTaskId(CONNECTOR_NAME, 0);
    worker.stopAndAwaitTasks(singletonList(task));
    EasyMock.expectLastCall();
    worker.stopConnector(CONNECTOR_NAME);
    EasyMock.expectLastCall().andReturn(true);
}

Example 19 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method expectAdd.

private void expectAdd(SourceSink sourceSink) throws Exception {
    Map<String, String> connectorProps = connectorConfig(sourceSink);
    ConnectorConfig connConfig = sourceSink == SourceSink.SOURCE ? new SourceConnectorConfig(plugins, connectorProps) : new SinkConnectorConfig(plugins, connectorProps);
    worker.startConnector(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(connectorProps), EasyMock.anyObject(HerderConnectorContext.class), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    EasyMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONNECTOR_NAME)).andReturn(true);
    ConnectorInfo connInfo = new ConnectorInfo(CONNECTOR_NAME, connectorProps, Arrays.asList(new ConnectorTaskId(CONNECTOR_NAME, 0)), SourceSink.SOURCE == sourceSink ? ConnectorType.SOURCE : ConnectorType.SINK);
    createCallback.onCompletion(null, new Herder.Created<>(true, connInfo));
    EasyMock.expectLastCall();
    // And we should instantiate the tasks. For a sink task, we should see added properties for the input topic partitions
    Map<String, String> generatedTaskProps = taskConfig(sourceSink);
    EasyMock.expect(worker.connectorTaskConfigs(CONNECTOR_NAME, connConfig)).andReturn(singletonList(generatedTaskProps));
    worker.startTask(new ConnectorTaskId(CONNECTOR_NAME, 0), connectorConfig(sourceSink), generatedTaskProps, herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(true);
    EasyMock.expect(herder.connectorTypeForClass(BogusSourceConnector.class.getName())).andReturn(ConnectorType.SOURCE).anyTimes();
    EasyMock.expect(herder.connectorTypeForClass(BogusSinkConnector.class.getName())).andReturn(ConnectorType.SINK).anyTimes();
    worker.isSinkConnector(CONNECTOR_NAME);
    PowerMock.expectLastCall().andReturn(sourceSink == SourceSink.SINK);
}

Example 20 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class Worker method connectorClientConfigOverrides.

private static Map<String, Object> connectorClientConfigOverrides(ConnectorTaskId id, ConnectorConfig connConfig, Class<? extends Connector> connectorClass, String clientConfigPrefix, ConnectorType connectorType, ConnectorClientConfigRequest.ClientType clientType, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
    Map<String, Object> clientOverrides = connConfig.originalsWithPrefix(clientConfigPrefix);
    ConnectorClientConfigRequest connectorClientConfigRequest = new ConnectorClientConfigRequest(id.connector(), connectorType, connectorClass, clientOverrides, clientType);
    List<ConfigValue> configValues = connectorClientConfigOverridePolicy.validate(connectorClientConfigRequest);
    List<ConfigValue> errorConfigs = configValues.stream().filter(configValue -> configValue.errorMessages().size() > 0).collect(Collectors.toList());
    // These should be caught when the herder validates the connector configuration, but just in case
    if (errorConfigs.size() > 0) {
        throw new ConnectException("Client Config Overrides not allowed " + errorConfigs);
    }
    return clientOverrides;
}