Examples with ConnectException org.apache.kafka.connect.errors.ConnectException used on opensource projects

Example 71 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class TopicAdmin method endOffsets.

/**
 * Fetch the most recent offset for each of the supplied {@link TopicPartition} objects.
 *
 * @param partitions the topic partitions
 * @return the map of offset for each topic partition, or an empty map if the supplied partitions
 *         are null or empty
 * @throws UnsupportedVersionException if the admin client cannot read end offsets
 * @throws TimeoutException if the offset metadata could not be fetched before the amount of time allocated
 *         by {@code request.timeout.ms} expires, and this call can be retried
 * @throws LeaderNotAvailableException if the leader was not available and this call can be retried
 * @throws RetriableException if a retriable error occurs, or the thread is interrupted while attempting
 *         to perform this operation
 * @throws ConnectException if a non retriable error occurs
 */
public Map<TopicPartition, Long> endOffsets(Set<TopicPartition> partitions) {
    if (partitions == null || partitions.isEmpty()) {
        return Collections.emptyMap();
    }
    Map<TopicPartition, OffsetSpec> offsetSpecMap = partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> OffsetSpec.latest()));
    ListOffsetsResult resultFuture = admin.listOffsets(offsetSpecMap);
    // Get the individual result for each topic partition so we have better error messages
    Map<TopicPartition, Long> result = new HashMap<>();
    for (TopicPartition partition : partitions) {
        try {
            ListOffsetsResultInfo info = resultFuture.partitionResult(partition).get();
            result.put(partition, info.offset());
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            String topic = partition.topic();
            if (cause instanceof AuthorizationException) {
                String msg = String.format("Not authorized to get the end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
                throw new ConnectException(msg, e);
            } else if (cause instanceof UnsupportedVersionException) {
                // Should theoretically never happen, because this method is the same as what the consumer uses and therefore
                // should exist in the broker since before the admin client was added
                String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers());
                throw new UnsupportedVersionException(msg, e);
            } else if (cause instanceof TimeoutException) {
                String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
                throw new TimeoutException(msg, e);
            } else if (cause instanceof LeaderNotAvailableException) {
                String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers());
                throw new LeaderNotAvailableException(msg, e);
            } else if (cause instanceof org.apache.kafka.common.errors.RetriableException) {
                throw (org.apache.kafka.common.errors.RetriableException) cause;
            } else {
                String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers());
                throw new ConnectException(msg, e);
            }
        } catch (InterruptedException e) {
            Thread.interrupted();
            String msg = String.format("Interrupted while attempting to read end offsets for topic '%s' on brokers at %s", partition.topic(), bootstrapServers());
            throw new RetriableException(msg, e);
        }
    }
    return result;
}

Example 72 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class TopicAdmin method createOrFindTopics.

/**
 * Attempt to create the topics described by the given definitions, returning all of the names of those topics that
 * were created by this request. Any existing topics with the same name are unchanged, and the names of such topics
 * are excluded from the result.
 * <p>
 * If multiple topic definitions have the same topic name, the last one with that name will be used.
 * <p>
 * Apache Kafka added support for creating topics in 0.10.1.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.1.0, this method is unable to create topics and always returns an empty set.
 *
 * @param topics the specifications of the topics
 * @return the {@link TopicCreationResponse} with the names of the newly created and existing topics;
 *         never null but possibly empty
 * @throws ConnectException if an error occurs, the operation takes too long, or the thread is interrupted while
 *                          attempting to perform this operation
 */
public TopicCreationResponse createOrFindTopics(NewTopic... topics) {
    Map<String, NewTopic> topicsByName = new HashMap<>();
    if (topics != null) {
        for (NewTopic topic : topics) {
            if (topic != null)
                topicsByName.put(topic.name(), topic);
        }
    }
    if (topicsByName.isEmpty())
        return EMPTY_CREATION;
    String bootstrapServers = bootstrapServers();
    String topicNameList = Utils.join(topicsByName.keySet(), "', '");
    // Attempt to create any missing topics
    CreateTopicsOptions args = new CreateTopicsOptions().validateOnly(false);
    Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
    // Iterate over each future so that we can handle individual failures like when some topics already exist
    Set<String> newlyCreatedTopicNames = new HashSet<>();
    Set<String> existingTopicNames = new HashSet<>();
    for (Map.Entry<String, KafkaFuture<Void>> entry : newResults.entrySet()) {
        String topic = entry.getKey();
        try {
            entry.getValue().get();
            if (logCreation) {
                log.info("Created topic {} on brokers at {}", topicsByName.get(topic), bootstrapServers);
            }
            newlyCreatedTopicNames.add(topic);
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof TopicExistsException) {
                log.debug("Found existing topic '{}' on the brokers at {}", topic, bootstrapServers);
                existingTopicNames.add(topic);
                continue;
            }
            if (cause instanceof UnsupportedVersionException) {
                log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof ClusterAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof TopicAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof InvalidConfigurationException) {
                throw new ConnectException("Unable to create topic(s) '" + topicNameList + "': " + cause.getMessage(), cause);
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new ConnectException("Timed out while checking for or creating topic(s) '" + topicNameList + "'." + " This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create.", cause);
            }
            throw new ConnectException("Error while attempting to create/find topic(s) '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new ConnectException("Interrupted while attempting to create/find topic(s) '" + topicNameList + "'", e);
        }
    }
    return new TopicCreationResponse(newlyCreatedTopicNames, existingTopicNames);
}

Example 73 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class TopicAdmin method describeTopics.

/**
 * Attempt to fetch the descriptions of the given topics
 * Apache Kafka added support for describing topics in 0.10.0.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.0.0, this method is unable to describe topics and always returns an empty set.
 *
 * @param topics the topics to describe
 * @return a map of topic names to topic descriptions of the topics that were requested; never null but possibly empty
 * @throws RetriableException if a retriable error occurs, the operation takes too long, or the
 * thread is interrupted while attempting to perform this operation
 * @throws ConnectException if a non retriable error occurs
 */
public Map<String, TopicDescription> describeTopics(String... topics) {
    if (topics == null) {
        return Collections.emptyMap();
    }
    String bootstrapServers = bootstrapServers();
    String topicNameList = String.join(", ", topics);
    Map<String, KafkaFuture<TopicDescription>> newResults = admin.describeTopics(Arrays.asList(topics), new DescribeTopicsOptions()).topicNameValues();
    // Iterate over each future so that we can handle individual failures like when some topics don't exist
    Map<String, TopicDescription> existingTopics = new HashMap<>();
    newResults.forEach((topic, desc) -> {
        try {
            existingTopics.put(topic, desc.get());
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof UnknownTopicOrPartitionException) {
                log.debug("Topic '{}' does not exist on the brokers at {}", topic, bootstrapServers);
                return;
            }
            if (cause instanceof ClusterAuthorizationException || cause instanceof TopicAuthorizationException) {
                String msg = String.format("Not authorized to describe topic(s) '%s' on the brokers %s", topicNameList, bootstrapServers);
                throw new ConnectException(msg, cause);
            }
            if (cause instanceof UnsupportedVersionException) {
                String msg = String.format("Unable to describe topic(s) '%s' since the brokers " + "at %s do not support the DescribeTopics API.", topicNameList, bootstrapServers);
                throw new ConnectException(msg, cause);
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new RetriableException("Timed out while describing topics '" + topicNameList + "'", cause);
            }
            throw new ConnectException("Error while attempting to describe topics '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new RetriableException("Interrupted while attempting to describe topics '" + topicNameList + "'", e);
        }
    });
    return existingTopics;
}

Example 74 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class RestServer method initializeResources.

public void initializeResources(Herder herder) {
    log.info("Initializing REST resources");
    ResourceConfig resourceConfig = new ResourceConfig();
    resourceConfig.register(new JacksonJsonProvider());
    resourceConfig.register(new RootResource(herder));
    resourceConfig.register(new ConnectorsResource(herder, config));
    resourceConfig.register(new ConnectorPluginsResource(herder));
    resourceConfig.register(ConnectExceptionMapper.class);
    resourceConfig.property(ServerProperties.WADL_FEATURE_DISABLE, true);
    registerRestExtensions(herder, resourceConfig);
    List<String> adminListeners = config.getList(WorkerConfig.ADMIN_LISTENERS_CONFIG);
    ResourceConfig adminResourceConfig;
    if (adminListeners == null) {
        log.info("Adding admin resources to main listener");
        adminResourceConfig = resourceConfig;
        adminResourceConfig.register(new LoggingResource());
    } else if (adminListeners.size() > 0) {
        // TODO: we need to check if these listeners are same as 'listeners'
        // TODO: the following code assumes that they are different
        log.info("Adding admin resources to admin listener");
        adminResourceConfig = new ResourceConfig();
        adminResourceConfig.register(new JacksonJsonProvider());
        adminResourceConfig.register(new LoggingResource());
        adminResourceConfig.register(ConnectExceptionMapper.class);
    } else {
        log.info("Skipping adding admin resources");
        // set up adminResource but add no handlers to it
        adminResourceConfig = resourceConfig;
    }
    ServletContainer servletContainer = new ServletContainer(resourceConfig);
    ServletHolder servletHolder = new ServletHolder(servletContainer);
    List<Handler> contextHandlers = new ArrayList<>();
    ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
    context.setContextPath("/");
    context.addServlet(servletHolder, "/*");
    contextHandlers.add(context);
    ServletContextHandler adminContext = null;
    if (adminResourceConfig != resourceConfig) {
        adminContext = new ServletContextHandler(ServletContextHandler.SESSIONS);
        ServletHolder adminServletHolder = new ServletHolder(new ServletContainer(adminResourceConfig));
        adminContext.setContextPath("/");
        adminContext.addServlet(adminServletHolder, "/*");
        adminContext.setVirtualHosts(new String[] { "@" + ADMIN_SERVER_CONNECTOR_NAME });
        contextHandlers.add(adminContext);
    }
    String allowedOrigins = config.getString(WorkerConfig.ACCESS_CONTROL_ALLOW_ORIGIN_CONFIG);
    if (!Utils.isBlank(allowedOrigins)) {
        FilterHolder filterHolder = new FilterHolder(new CrossOriginFilter());
        filterHolder.setName("cross-origin");
        filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, allowedOrigins);
        String allowedMethods = config.getString(WorkerConfig.ACCESS_CONTROL_ALLOW_METHODS_CONFIG);
        if (!Utils.isBlank(allowedMethods)) {
            filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, allowedMethods);
        }
        context.addFilter(filterHolder, "/*", EnumSet.of(DispatcherType.REQUEST));
    }
    String headerConfig = config.getString(WorkerConfig.RESPONSE_HTTP_HEADERS_CONFIG);
    if (!Utils.isBlank(headerConfig)) {
        configureHttpResponsHeaderFilter(context);
    }
    RequestLogHandler requestLogHandler = new RequestLogHandler();
    Slf4jRequestLogWriter slf4jRequestLogWriter = new Slf4jRequestLogWriter();
    slf4jRequestLogWriter.setLoggerName(RestServer.class.getCanonicalName());
    CustomRequestLog requestLog = new CustomRequestLog(slf4jRequestLogWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT + " %{ms}T");
    requestLogHandler.setRequestLog(requestLog);
    contextHandlers.add(new DefaultHandler());
    contextHandlers.add(requestLogHandler);
    handlers.setHandlers(contextHandlers.toArray(new Handler[0]));
    try {
        context.start();
    } catch (Exception e) {
        throw new ConnectException("Unable to initialize REST resources", e);
    }
    if (adminResourceConfig != resourceConfig) {
        try {
            log.debug("Starting admin context");
            adminContext.start();
        } catch (Exception e) {
            throw new ConnectException("Unable to initialize Admin REST resources", e);
        }
    }
    log.info("REST resources initialized; server is started and ready to handle requests");
}

Example 75 with ConnectException

use of org.apache.kafka.connect.errors.ConnectException in project kafka by apache.

the class KafkaBasedLog method stop.

public void stop() {
    log.info("Stopping KafkaBasedLog for topic " + topic);
    synchronized (this) {
        stopRequested = true;
    }
    consumer.wakeup();
    try {
        thread.join();
    } catch (InterruptedException e) {
        throw new ConnectException("Failed to stop KafkaBasedLog. Exiting without cleanly shutting " + "down it's producer and consumer.", e);
    }
    try {
        producer.close();
    } catch (KafkaException e) {
        log.error("Failed to stop KafkaBasedLog producer", e);
    }
    try {
        consumer.close();
    } catch (KafkaException e) {
        log.error("Failed to stop KafkaBasedLog consumer", e);
    }
    // do not close the admin client, since we don't own it
    admin = null;
    log.info("Stopped KafkaBasedLog for topic " + topic);
}