Examples with ConnectorTaskId org.apache.kafka.connect.util.ConnectorTaskId used on opensource projects

Example 56 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class ConnectorsResourceTest method testRestartTaskNotFound.

@Test
public void testRestartTaskNotFound() {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    final Capture<Callback<Void>> cb = Capture.newInstance();
    herder.restartTask(EasyMock.eq(taskId), EasyMock.capture(cb));
    expectAndCallbackException(cb, new NotFoundException("not found"));
    PowerMock.replayAll();
    assertThrows(NotFoundException.class, () -> connectorsResource.restartTask(CONNECTOR_NAME, 0, NULL_HEADERS, FORWARD));
    PowerMock.verifyAll();
}

Example 57 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class ConnectorsResourceTest method testGetTasksConfigConnectorNotFound.

@Test(expected = NotFoundException.class)
public void testGetTasksConfigConnectorNotFound() throws Throwable {
    final Capture<Callback<Map<ConnectorTaskId, Map<String, String>>>> cb = Capture.newInstance();
    herder.tasksConfig(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb));
    expectAndCallbackException(cb, new NotFoundException("not found"));
    PowerMock.replayAll();
    connectorsResource.getTasksConfig(CONNECTOR_NAME, NULL_HEADERS, FORWARD);
    PowerMock.verifyAll();
}

Example 58 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class SourceTaskOffsetCommitterTest method testSchedule.

@Test
public void testSchedule() throws Exception {
    Capture<Runnable> taskWrapper = EasyMock.newCapture();
    ScheduledFuture commitFuture = PowerMock.createMock(ScheduledFuture.class);
    EasyMock.expect(executor.scheduleWithFixedDelay(EasyMock.capture(taskWrapper), eq(DEFAULT_OFFSET_COMMIT_INTERVAL_MS), eq(DEFAULT_OFFSET_COMMIT_INTERVAL_MS), eq(TimeUnit.MILLISECONDS))).andReturn(commitFuture);
    ConnectorTaskId taskId = PowerMock.createMock(ConnectorTaskId.class);
    WorkerSourceTask task = PowerMock.createMock(WorkerSourceTask.class);
    EasyMock.expect(committers.put(taskId, commitFuture)).andReturn(null);
    PowerMock.replayAll();
    committer.schedule(taskId, task);
    assertTrue(taskWrapper.hasCaptured());
    assertNotNull(taskWrapper.getValue());
    PowerMock.verifyAll();
}

Example 59 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class WorkerCoordinator method performTaskAssignment.

private Map<String, ByteBuffer> performTaskAssignment(String leaderId, long maxOffset, Map<String, ConnectProtocol.WorkerState> memberConfigs) {
    Map<String, List<String>> connectorAssignments = new HashMap<>();
    Map<String, List<ConnectorTaskId>> taskAssignments = new HashMap<>();
    // Perform round-robin task assignment. Assign all connectors and then all tasks because assigning both the
    // connector and its tasks can lead to very uneven distribution of work in some common cases (e.g. for connectors
    // that generate only 1 task each; in a cluster of 2 or an even # of nodes, only even nodes will be assigned
    // connectors and only odd nodes will be assigned tasks, but tasks are, on average, actually more resource
    // intensive than connectors).
    List<String> connectorsSorted = sorted(configSnapshot.connectors());
    CircularIterator<String> memberIt = new CircularIterator<>(sorted(memberConfigs.keySet()));
    for (String connectorId : connectorsSorted) {
        String connectorAssignedTo = memberIt.next();
        log.trace("Assigning connector {} to {}", connectorId, connectorAssignedTo);
        List<String> memberConnectors = connectorAssignments.get(connectorAssignedTo);
        if (memberConnectors == null) {
            memberConnectors = new ArrayList<>();
            connectorAssignments.put(connectorAssignedTo, memberConnectors);
        }
        memberConnectors.add(connectorId);
    }
    for (String connectorId : connectorsSorted) {
        for (ConnectorTaskId taskId : sorted(configSnapshot.tasks(connectorId))) {
            String taskAssignedTo = memberIt.next();
            log.trace("Assigning task {} to {}", taskId, taskAssignedTo);
            List<ConnectorTaskId> memberTasks = taskAssignments.get(taskAssignedTo);
            if (memberTasks == null) {
                memberTasks = new ArrayList<>();
                taskAssignments.put(taskAssignedTo, memberTasks);
            }
            memberTasks.add(taskId);
        }
    }
    this.leaderState = new LeaderState(memberConfigs, connectorAssignments, taskAssignments);
    return fillAssignmentsAndSerialize(memberConfigs.keySet(), ConnectProtocol.Assignment.NO_ERROR, leaderId, memberConfigs.get(leaderId).url(), maxOffset, connectorAssignments, taskAssignments);
}

Example 60 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class StandaloneHerder method createConnectorTasks.

private void createConnectorTasks(String connName, TargetState initialState) {
    Map<String, String> connConfigs = configState.connectorConfig(connName);
    for (ConnectorTaskId taskId : configState.tasks(connName)) {
        Map<String, String> taskConfigMap = configState.taskConfig(taskId);
        worker.startTask(taskId, connConfigs, taskConfigMap, this, initialState);
    }
}