Examples with RestartPlan org.apache.kafka.connect.runtime.RestartPlan used on opensource projects

Example 1 with RestartPlan

use of org.apache.kafka.connect.runtime.RestartPlan in project kafka by apache.

the class DistributedHerder method doRestartConnectorAndTasks.

/**
 * Builds and executes a restart plan for the connector and its tasks from <code>request</code>.
 * Execution of a plan involves triggering the stop of eligible connector/tasks and then queuing the start for eligible connector/tasks.
 *
 * @param request the request to restart connector and tasks
 */
protected synchronized void doRestartConnectorAndTasks(RestartRequest request) {
    String connectorName = request.connectorName();
    Optional<RestartPlan> maybePlan = buildRestartPlan(request);
    if (!maybePlan.isPresent()) {
        log.debug("Skipping restart of connector '{}' since no status is available: {}", connectorName, request);
        return;
    }
    RestartPlan plan = maybePlan.get();
    log.info("Executing {}", plan);
    // If requested, stop the connector and any tasks, marking each as restarting
    final ExtendedAssignment currentAssignments = assignment;
    final Collection<ConnectorTaskId> assignedIdsToRestart = plan.taskIdsToRestart().stream().filter(taskId -> currentAssignments.tasks().contains(taskId)).collect(Collectors.toList());
    final boolean restartConnector = plan.shouldRestartConnector() && currentAssignments.connectors().contains(connectorName);
    final boolean restartTasks = !assignedIdsToRestart.isEmpty();
    if (restartConnector) {
        worker.stopAndAwaitConnector(connectorName);
        onRestart(connectorName);
    }
    if (restartTasks) {
        // Stop the tasks and mark as restarting
        worker.stopAndAwaitTasks(assignedIdsToRestart);
        assignedIdsToRestart.forEach(this::onRestart);
    }
    // Now restart the connector and tasks
    if (restartConnector) {
        try {
            startConnector(connectorName, (error, targetState) -> {
                if (error == null) {
                    log.info("Connector '{}' restart successful", connectorName);
                } else {
                    log.error("Connector '{}' restart failed", connectorName, error);
                }
            });
        } catch (Throwable t) {
            log.error("Connector '{}' restart failed", connectorName, t);
        }
    }
    if (restartTasks) {
        log.debug("Restarting {} of {} tasks for {}", plan.restartTaskCount(), plan.totalTaskCount(), request);
        plan.taskIdsToRestart().forEach(taskId -> {
            try {
                if (startTask(taskId)) {
                    log.info("Task '{}' restart successful", taskId);
                } else {
                    log.error("Task '{}' restart failed", taskId);
                }
            } catch (Throwable t) {
                log.error("Task '{}' restart failed", taskId, t);
            }
        });
        log.debug("Restarted {} of {} tasks for {} as requested", plan.restartTaskCount(), plan.totalTaskCount(), request);
    }
    log.info("Completed {}", plan);
}

Example 2 with RestartPlan

use of org.apache.kafka.connect.runtime.RestartPlan in project kafka by apache.

the class DistributedHerderTest method testDoRestartConnectorAndTasksOnlyTasks.

@Test
public void testDoRestartConnectorAndTasksOnlyTasks() {
    ConnectorTaskId taskId = new ConnectorTaskId(CONN1, 0);
    RestartRequest restartRequest = new RestartRequest(CONN1, false, true);
    RestartPlan restartPlan = PowerMock.createMock(RestartPlan.class);
    EasyMock.expect(restartPlan.shouldRestartConnector()).andReturn(true).anyTimes();
    EasyMock.expect(restartPlan.shouldRestartTasks()).andReturn(true).anyTimes();
    EasyMock.expect(restartPlan.taskIdsToRestart()).andReturn(Collections.singletonList(taskId)).anyTimes();
    EasyMock.expect(restartPlan.restartTaskCount()).andReturn(1).anyTimes();
    EasyMock.expect(restartPlan.totalTaskCount()).andReturn(1).anyTimes();
    EasyMock.expect(herder.buildRestartPlan(restartRequest)).andReturn(Optional.of(restartPlan)).anyTimes();
    herder.assignment = PowerMock.createMock(ExtendedAssignment.class);
    EasyMock.expect(herder.assignment.connectors()).andReturn(Collections.emptyList()).anyTimes();
    EasyMock.expect(herder.assignment.tasks()).andReturn(Collections.singletonList(taskId)).anyTimes();
    worker.stopAndAwaitTasks(Collections.singletonList(taskId));
    PowerMock.expectLastCall();
    herder.onRestart(taskId);
    EasyMock.expectLastCall();
    worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.anyObject(TargetState.class));
    PowerMock.expectLastCall().andReturn(true);
    PowerMock.replayAll();
    herder.doRestartConnectorAndTasks(restartRequest);
    PowerMock.verifyAll();
}

Example 3 with RestartPlan

use of org.apache.kafka.connect.runtime.RestartPlan in project kafka by apache.

the class DistributedHerderTest method testDoRestartConnectorAndTasksOnlyConnector.

@Test
public void testDoRestartConnectorAndTasksOnlyConnector() {
    ConnectorTaskId taskId = new ConnectorTaskId(CONN1, 0);
    RestartRequest restartRequest = new RestartRequest(CONN1, false, true);
    RestartPlan restartPlan = PowerMock.createMock(RestartPlan.class);
    EasyMock.expect(restartPlan.shouldRestartConnector()).andReturn(true).anyTimes();
    EasyMock.expect(restartPlan.shouldRestartTasks()).andReturn(true).anyTimes();
    EasyMock.expect(restartPlan.taskIdsToRestart()).andReturn(Collections.singletonList(taskId)).anyTimes();
    EasyMock.expect(herder.buildRestartPlan(restartRequest)).andReturn(Optional.of(restartPlan)).anyTimes();
    herder.assignment = PowerMock.createMock(ExtendedAssignment.class);
    EasyMock.expect(herder.assignment.connectors()).andReturn(Collections.singletonList(CONN1)).anyTimes();
    EasyMock.expect(herder.assignment.tasks()).andReturn(Collections.emptyList()).anyTimes();
    worker.stopAndAwaitConnector(CONN1);
    PowerMock.expectLastCall();
    Capture<Callback<TargetState>> stateCallback = newCapture();
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.anyObject(TargetState.class), capture(stateCallback));
    herder.onRestart(CONN1);
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    herder.doRestartConnectorAndTasks(restartRequest);
    PowerMock.verifyAll();
}

Example 4 with RestartPlan

use of org.apache.kafka.connect.runtime.RestartPlan in project kafka by apache.

the class StandaloneHerderTest method testRestartConnectorAndTasksOnlyTasks.

@Test
public void testRestartConnectorAndTasksOnlyTasks() throws Exception {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    RestartRequest restartRequest = new RestartRequest(CONNECTOR_NAME, false, true);
    RestartPlan restartPlan = PowerMock.createMock(RestartPlan.class);
    ConnectorStateInfo connectorStateInfo = PowerMock.createMock(ConnectorStateInfo.class);
    EasyMock.expect(restartPlan.shouldRestartConnector()).andReturn(false).anyTimes();
    EasyMock.expect(restartPlan.shouldRestartTasks()).andReturn(true).anyTimes();
    EasyMock.expect(restartPlan.restartTaskCount()).andReturn(1).anyTimes();
    EasyMock.expect(restartPlan.totalTaskCount()).andReturn(1).anyTimes();
    EasyMock.expect(restartPlan.taskIdsToRestart()).andReturn(Collections.singletonList(taskId)).anyTimes();
    EasyMock.expect(restartPlan.restartConnectorStateInfo()).andReturn(connectorStateInfo).anyTimes();
    EasyMock.expect(herder.buildRestartPlan(restartRequest)).andReturn(Optional.of(restartPlan)).anyTimes();
    herder.onRestart(taskId);
    EasyMock.expectLastCall();
    connector = PowerMock.createMock(BogusSinkConnector.class);
    expectAdd(SourceSink.SINK);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SINK);
    Connector connectorMock = PowerMock.createMock(SinkConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    worker.stopAndAwaitTasks(Collections.singletonList(taskId));
    EasyMock.expectLastCall();
    ClusterConfigState configState = new ClusterConfigState(-1, null, Collections.singletonMap(CONNECTOR_NAME, 1), Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(taskId, taskConfig(SourceSink.SINK)), new HashSet<>(), transformer);
    worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SINK), herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(true);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SINK), connectorInfo.result());
    FutureCallback<ConnectorStateInfo> restartCallback = new FutureCallback<>();
    herder.restartConnectorAndTasks(restartRequest, restartCallback);
    assertEquals(connectorStateInfo, restartCallback.get(1000L, TimeUnit.MILLISECONDS));
    PowerMock.verifyAll();
}

Example 5 with RestartPlan

use of org.apache.kafka.connect.runtime.RestartPlan in project kafka by apache.

the class DistributedHerder method restartConnectorAndTasks.

@Override
public void restartConnectorAndTasks(RestartRequest request, Callback<ConnectorStateInfo> callback) {
    final String connectorName = request.connectorName();
    addRequest(() -> {
        if (checkRebalanceNeeded(callback)) {
            return null;
        }
        if (!configState.connectors().contains(request.connectorName())) {
            callback.onCompletion(new NotFoundException("Unknown connector: " + connectorName), null);
            return null;
        }
        if (isLeader()) {
            // Write a restart request to the config backing store, to be executed asynchronously in tick()
            configBackingStore.putRestartRequest(request);
            // Compute and send the response that this was accepted
            Optional<RestartPlan> plan = buildRestartPlan(request);
            if (!plan.isPresent()) {
                callback.onCompletion(new NotFoundException("Status for connector " + connectorName + " not found", null), null);
            } else {
                callback.onCompletion(null, plan.get().restartConnectorStateInfo());
            }
        } else {
            callback.onCompletion(new NotLeaderException("Only the leader can process restart requests.", leaderUrl()), null);
        }
        return null;
    }, forwardErrorCallback(callback));
}