Examples with FutureCallback org.apache.kafka.connect.util.FutureCallback used on opensource projects

Example 16 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project kafka by apache.

the class ConnectorsResource method restartTask.

@POST
@Path("/{connector}/tasks/{task}/restart")
public void restartTask(@PathParam("connector") final String connector, @PathParam("task") final Integer task, @Context final HttpHeaders headers, @QueryParam("forward") final Boolean forward) throws Throwable {
    FutureCallback<Void> cb = new FutureCallback<>();
    ConnectorTaskId taskId = new ConnectorTaskId(connector, task);
    herder.restartTask(taskId, cb);
    completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks/" + task + "/restart", "POST", headers, null, forward);
}

Example 17 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project kafka by apache.

the class ConnectStandalone method main.

public static void main(String[] args) {
    if (args.length < 2 || Arrays.asList(args).contains("--help")) {
        log.info("Usage: ConnectStandalone worker.properties connector1.properties [connector2.properties ...]");
        Exit.exit(1);
    }
    try {
        Time time = Time.SYSTEM;
        log.info("Kafka Connect standalone worker initializing ...");
        long initStart = time.hiResClockMs();
        WorkerInfo initInfo = new WorkerInfo();
        initInfo.logAll();
        String workerPropsFile = args[0];
        Map<String, String> workerProps = !workerPropsFile.isEmpty() ? Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.emptyMap();
        log.info("Scanning for plugin classes. This might take a moment ...");
        Plugins plugins = new Plugins(workerProps);
        plugins.compareAndSwapWithDelegatingLoader();
        StandaloneConfig config = new StandaloneConfig(workerProps);
        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
        log.debug("Kafka cluster ID: {}", kafkaClusterId);
        RestServer rest = new RestServer(config);
        rest.initializeServer();
        URI advertisedUrl = rest.advertisedUrl();
        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
        ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy = plugins.newPlugin(config.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG), config, ConnectorClientConfigOverridePolicy.class);
        Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore(), connectorClientConfigOverridePolicy);
        Herder herder = new StandaloneHerder(worker, kafkaClusterId, connectorClientConfigOverridePolicy);
        final Connect connect = new Connect(herder, rest);
        log.info("Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart);
        try {
            connect.start();
            for (final String connectorPropsFile : Arrays.copyOfRange(args, 1, args.length)) {
                Map<String, String> connectorProps = Utils.propsToStringMap(Utils.loadProps(connectorPropsFile));
                FutureCallback<Herder.Created<ConnectorInfo>> cb = new FutureCallback<>((error, info) -> {
                    if (error != null)
                        log.error("Failed to create job for {}", connectorPropsFile);
                    else
                        log.info("Created connector {}", info.result().name());
                });
                herder.putConnectorConfig(connectorProps.get(ConnectorConfig.NAME_CONFIG), connectorProps, false, cb);
                cb.get();
            }
        } catch (Throwable t) {
            log.error("Stopping after connector error", t);
            connect.stop();
            Exit.exit(3);
        }
        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
        connect.awaitStop();
    } catch (Throwable t) {
        log.error("Stopping due to error", t);
        Exit.exit(2);
    }
}

Example 18 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project kafka by apache.

the class DistributedHerderTest method testRestartConnectorAndTasksUnknownStatus.

@Test
public void testRestartConnectorAndTasksUnknownStatus() throws Exception {
    RestartRequest restartRequest = new RestartRequest(CONN1, false, true);
    EasyMock.expect(herder.buildRestartPlan(restartRequest)).andReturn(Optional.empty()).anyTimes();
    configBackingStore.putRestartRequest(restartRequest);
    PowerMock.expectLastCall();
    // get the initial assignment
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // now handle the connector restart
    member.wakeup();
    PowerMock.expectLastCall();
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.tick();
    FutureCallback<ConnectorStateInfo> callback = new FutureCallback<>();
    herder.restartConnectorAndTasks(restartRequest, callback);
    herder.tick();
    ExecutionException ee = assertThrows(ExecutionException.class, () -> callback.get(1000L, TimeUnit.MILLISECONDS));
    assertTrue(ee.getCause() instanceof NotFoundException);
    assertTrue(ee.getMessage().contains("Status for connector"));
    PowerMock.verifyAll();
}

Example 19 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project kafka by apache.

the class DistributedHerderTest method testAccessors.

@Test
public void testAccessors() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT).times(2);
    WorkerConfigTransformer configTransformer = EasyMock.mock(WorkerConfigTransformer.class);
    EasyMock.expect(configTransformer.transform(EasyMock.eq(CONN1), EasyMock.anyObject())).andThrow(new AssertionError("Config transformation should not occur when requesting connector or task info"));
    EasyMock.replay(configTransformer);
    ClusterConfigState snapshotWithTransform = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.emptySet(), configTransformer);
    expectPostRebalanceCatchup(snapshotWithTransform);
    member.wakeup();
    PowerMock.expectLastCall().anyTimes();
    // list connectors, get connector info, get connector config, get task configs
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
    PowerMock.replayAll();
    FutureCallback<Collection<String>> listConnectorsCb = new FutureCallback<>();
    herder.connectors(listConnectorsCb);
    FutureCallback<ConnectorInfo> connectorInfoCb = new FutureCallback<>();
    herder.connectorInfo(CONN1, connectorInfoCb);
    FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    FutureCallback<List<TaskInfo>> taskConfigsCb = new FutureCallback<>();
    herder.taskConfigs(CONN1, taskConfigsCb);
    herder.tick();
    assertTrue(listConnectorsCb.isDone());
    assertEquals(Collections.singleton(CONN1), listConnectorsCb.get());
    assertTrue(connectorInfoCb.isDone());
    ConnectorInfo info = new ConnectorInfo(CONN1, CONN1_CONFIG, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
    assertEquals(info, connectorInfoCb.get());
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG, connectorConfigCb.get());
    assertTrue(taskConfigsCb.isDone());
    assertEquals(Arrays.asList(new TaskInfo(TASK0, TASK_CONFIG), new TaskInfo(TASK1, TASK_CONFIG), new TaskInfo(TASK2, TASK_CONFIG)), taskConfigsCb.get());
    PowerMock.verifyAll();
}

Example 20 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project kafka by apache.

the class DistributedHerderTest method testRestartUnknownTask.

@Test
public void testRestartUnknownTask() throws Exception {
    // get the initial assignment
    EasyMock.expect(member.memberId()).andStubReturn("member");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    member.wakeup();
    PowerMock.expectLastCall();
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    FutureCallback<Void> callback = new FutureCallback<>();
    herder.tick();
    herder.restartTask(new ConnectorTaskId("blah", 0), callback);
    herder.tick();
    try {
        callback.get(1000L, TimeUnit.MILLISECONDS);
        fail("Expected NotLeaderException to be raised");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof NotFoundException);
    }
    PowerMock.verifyAll();
}