Examples with ConnectorTaskId org.apache.kafka.connect.util.ConnectorTaskId used on opensource projects

Example 6 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class WorkerCoordinatorTest method testJoinLeaderCannotAssign.

@Test
public void testJoinLeaderCannotAssign() {
    // If the selected leader can't get up to the maximum offset, it will fail to assign and we should immediately
    // need to retry the join.
    // When the first round fails, we'll take an updated config snapshot
    EasyMock.expect(configStorage.snapshot()).andReturn(configState1);
    EasyMock.expect(configStorage.snapshot()).andReturn(configState2);
    PowerMock.replayAll();
    final String memberId = "member";
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady();
    // config mismatch results in assignment error
    client.prepareResponse(joinGroupFollowerResponse(1, memberId, "leader", Errors.NONE));
    MockClient.RequestMatcher matcher = new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            SyncGroupRequest sync = (SyncGroupRequest) body;
            return sync.memberId().equals(memberId) && sync.generationId() == 1 && sync.groupAssignment().isEmpty();
        }
    };
    client.prepareResponse(matcher, syncGroupResponse(ConnectProtocol.Assignment.CONFIG_MISMATCH, "leader", 10L, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList(), Errors.NONE));
    client.prepareResponse(joinGroupFollowerResponse(1, memberId, "leader", Errors.NONE));
    client.prepareResponse(matcher, syncGroupResponse(ConnectProtocol.Assignment.NO_ERROR, "leader", 1L, Collections.<String>emptyList(), Collections.singletonList(taskId1x0), Errors.NONE));
    coordinator.ensureActiveGroup();
    PowerMock.verifyAll();
}

Example 7 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class ClusterConfigState method tasks.

/**
 * Get the current set of task IDs for the specified connector.
 * @param connectorName the name of the connector to look up task configs for
 * @return the current set of connector task IDs
 */
public List<ConnectorTaskId> tasks(String connectorName) {
    if (inconsistentConnectors.contains(connectorName))
        return Collections.emptyList();
    Integer numTasks = connectorTaskCounts.get(connectorName);
    if (numTasks == null)
        return Collections.emptyList();
    List<ConnectorTaskId> taskIds = new ArrayList<>();
    for (int taskIndex = 0; taskIndex < numTasks; taskIndex++) {
        ConnectorTaskId taskId = new ConnectorTaskId(connectorName, taskIndex);
        taskIds.add(taskId);
    }
    return taskIds;
}

Example 8 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class ConnectProtocol method deserializeAssignment.

public static Assignment deserializeAssignment(ByteBuffer buffer) {
    Struct header = CONNECT_PROTOCOL_HEADER_SCHEMA.read(buffer);
    Short version = header.getShort(VERSION_KEY_NAME);
    checkVersionCompatibility(version);
    Struct struct = ASSIGNMENT_V0.read(buffer);
    short error = struct.getShort(ERROR_KEY_NAME);
    String leader = struct.getString(LEADER_KEY_NAME);
    String leaderUrl = struct.getString(LEADER_URL_KEY_NAME);
    long offset = struct.getLong(CONFIG_OFFSET_KEY_NAME);
    List<String> connectorIds = new ArrayList<>();
    List<ConnectorTaskId> taskIds = new ArrayList<>();
    for (Object structObj : struct.getArray(ASSIGNMENT_KEY_NAME)) {
        Struct assignment = (Struct) structObj;
        String connector = assignment.getString(CONNECTOR_KEY_NAME);
        for (Object taskIdObj : assignment.getArray(TASKS_KEY_NAME)) {
            Integer taskId = (Integer) taskIdObj;
            if (taskId == CONNECTOR_TASK)
                connectorIds.add(connector);
            else
                taskIds.add(new ConnectorTaskId(connector, taskId));
        }
    }
    return new Assignment(error, leader, leaderUrl, offset, connectorIds, taskIds);
}

Example 9 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerder method reconfigureConnector.

// Updates configurations for a connector by requesting them from the connector, filling in parameters provided
// by the system, then checks whether any configs have actually changed before submitting the new configs to storage
private void reconfigureConnector(final String connName, final Callback<Void> cb) {
    try {
        if (!worker.isRunning(connName)) {
            log.info("Skipping reconfiguration of connector {} since it is not running", connName);
            return;
        }
        Map<String, String> configs = configState.connectorConfig(connName);
        ConnectorConfig connConfig;
        if (worker.isSinkConnector(connName)) {
            connConfig = new SinkConnectorConfig(plugins(), configs);
        } else {
            connConfig = new SourceConnectorConfig(plugins(), configs);
        }
        final List<Map<String, String>> taskProps = worker.connectorTaskConfigs(connName, connConfig);
        boolean changed = false;
        int currentNumTasks = configState.taskCount(connName);
        if (taskProps.size() != currentNumTasks) {
            log.debug("Change in connector task count from {} to {}, writing updated task configurations", currentNumTasks, taskProps.size());
            changed = true;
        } else {
            int index = 0;
            for (Map<String, String> taskConfig : taskProps) {
                if (!taskConfig.equals(configState.taskConfig(new ConnectorTaskId(connName, index)))) {
                    log.debug("Change in task configurations, writing updated task configurations");
                    changed = true;
                    break;
                }
                index++;
            }
        }
        if (changed) {
            if (isLeader()) {
                configBackingStore.putTaskConfigs(connName, taskProps);
                cb.onCompletion(null, null);
            } else {
                // We cannot forward the request on the same thread because this reconfiguration can happen as a result of connector
                // addition or removal. If we blocked waiting for the response from leader, we may be kicked out of the worker group.
                forwardRequestExecutor.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            String reconfigUrl = RestServer.urlJoin(leaderUrl(), "/connectors/" + connName + "/tasks");
                            RestClient.httpRequest(reconfigUrl, "POST", taskProps, null, config);
                            cb.onCompletion(null, null);
                        } catch (ConnectException e) {
                            log.error("Request to leader to reconfigure connector tasks failed", e);
                            cb.onCompletion(e, null);
                        }
                    }
                });
            }
        }
    } catch (Throwable t) {
        cb.onCompletion(t, null);
    }
}

Example 10 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerder method taskConfigs.

@Override
public synchronized void taskConfigs(String connName, Callback<List<TaskInfo>> callback) {
    if (!configState.contains(connName)) {
        callback.onCompletion(new NotFoundException("Connector " + connName + " not found", null), null);
        return;
    }
    List<TaskInfo> result = new ArrayList<>();
    for (ConnectorTaskId taskId : configState.tasks(connName)) result.add(new TaskInfo(taskId, configState.taskConfig(taskId)));
    callback.onCompletion(null, result);
}