Examples with ConnectException org.apache.kafka.connect.errors.ConnectException used on opensource projects

Example 31 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class WorkerSinkTask method deliverMessages.

private void deliverMessages() {
    // Finally, deliver this batch to the sink
    try {
        // Since we reuse the messageBatch buffer, ensure we give the task its own copy
        task.put(new ArrayList<>(messageBatch));
        for (SinkRecord record : messageBatch) currentOffsets.put(new TopicPartition(record.topic(), record.kafkaPartition()), new OffsetAndMetadata(record.kafkaOffset() + 1));
        messageBatch.clear();
        // the task had not explicitly paused
        if (pausedForRedelivery) {
            if (!shouldPause())
                resumeAll();
            pausedForRedelivery = false;
        }
    } catch (RetriableException e) {
        log.error("RetriableException from SinkTask {}:", id, e);
        // If we're retrying a previous batch, make sure we've paused all topic partitions so we don't get new data,
        // but will still be able to poll in order to handle user-requested timeouts, keep group membership, etc.
        pausedForRedelivery = true;
        pauseAll();
    // Let this exit normally, the batch will be reprocessed on the next loop.
    } catch (Throwable t) {
        log.error("Task {} threw an uncaught and unrecoverable exception", id, t);
        log.error("Task is being killed and will not recover until manually restarted");
        throw new ConnectException("Exiting WorkerSinkTask due to unrecoverable exception.");
    }
}

Example 32 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class WorkerSinkTask method createConsumer.

private KafkaConsumer<byte[], byte[]> createConsumer() {
    // Include any unknown worker configs so consumer configs can be set globally on the worker
    // and through to the task
    Map<String, Object> props = new HashMap<>();
    props.put(ConsumerConfig.GROUP_ID_CONFIG, SinkUtils.consumerGroupId(id.connector()));
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, Utils.join(workerConfig.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ","));
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer");
    props.putAll(workerConfig.originalsWithPrefix("consumer."));
    KafkaConsumer<byte[], byte[]> newConsumer;
    try {
        newConsumer = new KafkaConsumer<>(props);
    } catch (Throwable t) {
        throw new ConnectException("Failed to create consumer", t);
    }
    return newConsumer;
}

Example 33 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class SourceTaskOffsetCommitterTest method testRemove.

@Test
public void testRemove() throws Exception {
    ConnectorTaskId taskId = PowerMock.createMock(ConnectorTaskId.class);
    ScheduledFuture task = PowerMock.createMock(ScheduledFuture.class);
    // Try to remove a non-existing task
    EasyMock.expect(committers.remove(taskId)).andReturn(null);
    PowerMock.replayAll();
    committer.remove(taskId);
    PowerMock.verifyAll();
    PowerMock.resetAll();
    // Try to remove an existing task
    EasyMock.expect(committers.remove(taskId)).andReturn(task);
    EasyMock.expect(task.cancel(eq(false))).andReturn(false);
    EasyMock.expect(task.isDone()).andReturn(false);
    EasyMock.expect(task.get()).andReturn(null);
    PowerMock.replayAll();
    committer.remove(taskId);
    PowerMock.verifyAll();
    PowerMock.resetAll();
    // Try to remove a cancelled task
    EasyMock.expect(committers.remove(taskId)).andReturn(task);
    EasyMock.expect(task.cancel(eq(false))).andReturn(false);
    EasyMock.expect(task.isDone()).andReturn(false);
    EasyMock.expect(task.get()).andThrow(new CancellationException());
    mockLog.trace(EasyMock.anyString(), EasyMock.<Object>anyObject());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    committer.remove(taskId);
    PowerMock.verifyAll();
    PowerMock.resetAll();
    // Try to remove an interrupted task
    EasyMock.expect(committers.remove(taskId)).andReturn(task);
    EasyMock.expect(task.cancel(eq(false))).andReturn(false);
    EasyMock.expect(task.isDone()).andReturn(false);
    EasyMock.expect(task.get()).andThrow(new InterruptedException());
    PowerMock.replayAll();
    try {
        committer.remove(taskId);
        fail("Expected ConnectException to be raised");
    } catch (ConnectException e) {
    //ignore
    }
    PowerMock.verifyAll();
}

Example 34 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class WorkerTest method testReconfigureConnectorTasks.

@Test
public void testReconfigureConnectorTasks() throws Exception {
    expectStartStorage();
    // Create
    Connector connector = PowerMock.createMock(Connector.class);
    ConnectorContext ctx = PowerMock.createMock(ConnectorContext.class);
    EasyMock.expect(connectorFactory.newConnector(WorkerTestConnector.class.getName())).andReturn(connector);
    EasyMock.expect(connector.version()).andReturn("1.0");
    Map<String, String> props = new HashMap<>();
    props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
    props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
    props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
    props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName());
    connector.initialize(EasyMock.anyObject(ConnectorContext.class));
    EasyMock.expectLastCall();
    connector.start(props);
    EasyMock.expectLastCall();
    connectorStatusListener.onStartup(CONNECTOR_ID);
    EasyMock.expectLastCall();
    // Reconfigure
    EasyMock.<Class<? extends Task>>expect(connector.taskClass()).andReturn(TestSourceTask.class);
    Map<String, String> taskProps = new HashMap<>();
    taskProps.put("foo", "bar");
    EasyMock.expect(connector.taskConfigs(2)).andReturn(Arrays.asList(taskProps, taskProps));
    // Remove
    connector.stop();
    EasyMock.expectLastCall();
    connectorStatusListener.onShutdown(CONNECTOR_ID);
    EasyMock.expectLastCall();
    expectStopStorage();
    PowerMock.replayAll();
    worker = new Worker(WORKER_ID, new MockTime(), connectorFactory, config, offsetBackingStore);
    worker.start();
    assertEquals(Collections.emptySet(), worker.connectorNames());
    worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
    assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_ID)), worker.connectorNames());
    try {
        worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
        fail("Should have thrown exception when trying to add connector with same name.");
    } catch (ConnectException e) {
    // expected
    }
    List<Map<String, String>> taskConfigs = worker.connectorTaskConfigs(CONNECTOR_ID, 2, Arrays.asList("foo", "bar"));
    Map<String, String> expectedTaskProps = new HashMap<>();
    expectedTaskProps.put("foo", "bar");
    expectedTaskProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName());
    expectedTaskProps.put(SinkTask.TOPICS_CONFIG, "foo,bar");
    assertEquals(2, taskConfigs.size());
    assertEquals(expectedTaskProps, taskConfigs.get(0));
    assertEquals(expectedTaskProps, taskConfigs.get(1));
    worker.stopConnector(CONNECTOR_ID);
    assertEquals(Collections.emptySet(), worker.connectorNames());
    // Nothing should be left, so this should effectively be a nop
    worker.stop();
    PowerMock.verifyAll();
}

Example 35 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class WorkerTest method testStartAndStopConnector.

@Test
public void testStartAndStopConnector() throws Exception {
    expectStartStorage();
    // Create
    Connector connector = PowerMock.createMock(Connector.class);
    ConnectorContext ctx = PowerMock.createMock(ConnectorContext.class);
    EasyMock.expect(connectorFactory.newConnector(WorkerTestConnector.class.getName())).andReturn(connector);
    EasyMock.expect(connector.version()).andReturn("1.0");
    Map<String, String> props = new HashMap<>();
    props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar");
    props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1");
    props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID);
    props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName());
    connector.initialize(EasyMock.anyObject(ConnectorContext.class));
    EasyMock.expectLastCall();
    connector.start(props);
    EasyMock.expectLastCall();
    connectorStatusListener.onStartup(CONNECTOR_ID);
    EasyMock.expectLastCall();
    // Remove
    connector.stop();
    EasyMock.expectLastCall();
    connectorStatusListener.onShutdown(CONNECTOR_ID);
    EasyMock.expectLastCall();
    expectStopStorage();
    PowerMock.replayAll();
    worker = new Worker(WORKER_ID, new MockTime(), connectorFactory, config, offsetBackingStore);
    worker.start();
    assertEquals(Collections.emptySet(), worker.connectorNames());
    worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
    assertEquals(new HashSet<>(Arrays.asList(CONNECTOR_ID)), worker.connectorNames());
    try {
        worker.startConnector(CONNECTOR_ID, props, ctx, connectorStatusListener, TargetState.STARTED);
        fail("Should have thrown exception when trying to add connector with same name.");
    } catch (ConnectException e) {
    // expected
    }
    worker.stopConnector(CONNECTOR_ID);
    assertEquals(Collections.emptySet(), worker.connectorNames());
    // Nothing should be left, so this should effectively be a nop
    worker.stop();
    PowerMock.verifyAll();
}