Search in sources :

Example 1 with CloseableConnectorContext

use of org.apache.kafka.connect.runtime.CloseableConnectorContext in project kafka by apache.

the class DistributedHerder method startConnector.

// Helper for starting a connector with the given name, which will extract & parse the config, generate connector
// context and add to the worker. This needs to be called from within the main worker thread for this herder.
// The callback is invoked after the connector has finished startup and generated task configs, or failed in the process.
private void startConnector(String connectorName, Callback<Void> callback) {
    log.info("Starting connector {}", connectorName);
    final Map<String, String> configProps = configState.connectorConfig(connectorName);
    final CloseableConnectorContext ctx = new HerderConnectorContext(this, connectorName);
    final TargetState initialState = configState.targetState(connectorName);
    final Callback<TargetState> onInitialStateChange = (error, newState) -> {
        if (error != null) {
            callback.onCompletion(new ConnectException("Failed to start connector: " + connectorName, error), null);
            return;
        }
        // Use newState here in case the connector has been paused right after being created
        if (newState == TargetState.STARTED) {
            addRequest(() -> {
                // Request configuration since this could be a brand new connector. However, also only update those
                // task configs if they are actually different from the existing ones to avoid unnecessary updates when this is
                // just restoring an existing connector.
                reconfigureConnectorTasksWithRetry(time.milliseconds(), connectorName);
                callback.onCompletion(null, null);
                return null;
            }, forwardErrorCallback(callback));
        } else {
            callback.onCompletion(null, null);
        }
    };
    worker.startConnector(connectorName, configProps, ctx, this, initialState, onInitialStateChange);
}
Also used : TargetState(org.apache.kafka.connect.runtime.TargetState) Worker(org.apache.kafka.connect.runtime.Worker) SinkUtils(org.apache.kafka.connect.util.SinkUtils) Arrays(java.util.Arrays) TimeoutException(java.util.concurrent.TimeoutException) AlreadyExistsException(org.apache.kafka.connect.errors.AlreadyExistsException) KeyGenerator(javax.crypto.KeyGenerator) SessionKey(org.apache.kafka.connect.runtime.SessionKey) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) RestartRequest(org.apache.kafka.connect.runtime.RestartRequest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LogContext(org.apache.kafka.common.utils.LogContext) Map(java.util.Map) UriBuilder(javax.ws.rs.core.UriBuilder) ThreadUtils(org.apache.kafka.common.utils.ThreadUtils) RestClient(org.apache.kafka.connect.runtime.rest.RestClient) ConfigDef(org.apache.kafka.common.config.ConfigDef) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) Sensor(org.apache.kafka.common.metrics.Sensor) Time(org.apache.kafka.common.utils.Time) WakeupException(org.apache.kafka.common.errors.WakeupException) Collection(java.util.Collection) CumulativeSum(org.apache.kafka.common.metrics.stats.CumulativeSum) Set(java.util.Set) NavigableSet(java.util.NavigableSet) ConfigValue(org.apache.kafka.common.config.ConfigValue) Collectors(java.util.stream.Collectors) AbstractHerder(org.apache.kafka.connect.runtime.AbstractHerder) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) CONNECT_PROTOCOL_V0(org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0) ConnectorStateInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo) CloseableConnectorContext(org.apache.kafka.connect.runtime.CloseableConnectorContext) List(java.util.List) EAGER(org.apache.kafka.connect.runtime.distributed.ConnectProtocolCompatibility.EAGER) ConnectMetrics(org.apache.kafka.connect.runtime.ConnectMetrics) SourceConnectorConfig(org.apache.kafka.connect.runtime.SourceConnectorConfig) Response(javax.ws.rs.core.Response) Optional(java.util.Optional) TargetState(org.apache.kafka.connect.runtime.TargetState) SecretKey(javax.crypto.SecretKey) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) TaskStatus(org.apache.kafka.connect.runtime.TaskStatus) Max(org.apache.kafka.common.metrics.stats.Max) Connector(org.apache.kafka.connect.connector.Connector) Exit(org.apache.kafka.common.utils.Exit) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) NotFoundException(org.apache.kafka.connect.errors.NotFoundException) ConfigBackingStore(org.apache.kafka.connect.storage.ConfigBackingStore) AtomicReference(java.util.concurrent.atomic.AtomicReference) TOPIC_TRACKING_ENABLE_CONFIG(org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_TRACKING_ENABLE_CONFIG) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) CONNECT_PROTOCOL_V2(org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2) CONNECT_PROTOCOL_V1(org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1) MetricGroup(org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup) NoSuchElementException(java.util.NoSuchElementException) ExecutorService(java.util.concurrent.ExecutorService) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) Utils(org.apache.kafka.common.utils.Utils) HerderConnectorContext(org.apache.kafka.connect.runtime.HerderConnectorContext) Callback(org.apache.kafka.connect.util.Callback) RestartPlan(org.apache.kafka.connect.runtime.RestartPlan) Logger(org.slf4j.Logger) InternalRequestSignature(org.apache.kafka.connect.runtime.rest.InternalRequestSignature) HerderRequest(org.apache.kafka.connect.runtime.HerderRequest) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) ConnectRestException(org.apache.kafka.connect.runtime.rest.errors.ConnectRestException) ConnectMetricsRegistry(org.apache.kafka.connect.runtime.ConnectMetricsRegistry) TaskInfo(org.apache.kafka.connect.runtime.rest.entities.TaskInfo) StatusBackingStore(org.apache.kafka.connect.storage.StatusBackingStore) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) ConcurrentSkipListSet(java.util.concurrent.ConcurrentSkipListSet) Avg(org.apache.kafka.common.metrics.stats.Avg) ConnectException(org.apache.kafka.connect.errors.ConnectException) LinkedBlockingDeque(java.util.concurrent.LinkedBlockingDeque) Collections(java.util.Collections) CloseableConnectorContext(org.apache.kafka.connect.runtime.CloseableConnectorContext) HerderConnectorContext(org.apache.kafka.connect.runtime.HerderConnectorContext) ConnectException(org.apache.kafka.connect.errors.ConnectException)

Aggregations

ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 NavigableSet (java.util.NavigableSet)1 NoSuchElementException (java.util.NoSuchElementException)1 Objects (java.util.Objects)1 Optional (java.util.Optional)1 Set (java.util.Set)1 Callable (java.util.concurrent.Callable)1 ConcurrentSkipListSet (java.util.concurrent.ConcurrentSkipListSet)1 ExecutorService (java.util.concurrent.ExecutorService)1 Executors (java.util.concurrent.Executors)1 LinkedBlockingDeque (java.util.concurrent.LinkedBlockingDeque)1 ThreadPoolExecutor (java.util.concurrent.ThreadPoolExecutor)1 TimeUnit (java.util.concurrent.TimeUnit)1