Search in sources :

Example 1 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class HBaseRowDataAsyncLookupFunction method open.

@Override
public void open(FunctionContext context) {
    LOG.info("start open ...");
    final ExecutorService threadPool = Executors.newFixedThreadPool(THREAD_POOL_SIZE, new ExecutorThreadFactory("hbase-async-lookup-worker", Threads.LOGGING_EXCEPTION_HANDLER));
    Configuration config = prepareRuntimeConfiguration();
    CompletableFuture<AsyncConnection> asyncConnectionFuture = ConnectionFactory.createAsyncConnection(config);
    try {
        asyncConnection = asyncConnectionFuture.get();
        table = asyncConnection.getTable(TableName.valueOf(hTableName), threadPool);
        this.cache = cacheMaxSize <= 0 || cacheExpireMs <= 0 ? null : CacheBuilder.newBuilder().recordStats().expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS).maximumSize(cacheMaxSize).build();
        if (cache != null && context != null) {
            context.getMetricGroup().gauge("lookupCacheHitRate", (Gauge<Double>) () -> cache.stats().hitRate());
        }
    } catch (InterruptedException | ExecutionException e) {
        LOG.error("Exception while creating connection to HBase.", e);
        throw new RuntimeException("Cannot create connection to HBase.", e);
    }
    this.serde = new HBaseSerde(hbaseTableSchema, nullStringLiteral);
    LOG.info("end open.");
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Configuration(org.apache.hadoop.conf.Configuration) ExecutorService(java.util.concurrent.ExecutorService) AsyncConnection(org.apache.hadoop.hbase.client.AsyncConnection) ExecutionException(java.util.concurrent.ExecutionException) HBaseSerde(org.apache.flink.connector.hbase.util.HBaseSerde)

Example 2 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class ClusterEntrypoint method initializeServices.

protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {
    LOG.info("Initializing cluster services.");
    synchronized (lock) {
        resourceId = configuration.getOptional(JobManagerOptions.JOB_MANAGER_RESOURCE_ID).map(value -> DeterminismEnvelope.deterministicValue(new ResourceID(value))).orElseGet(() -> DeterminismEnvelope.nondeterministicValue(ResourceID.generate()));
        LOG.debug("Initialize cluster entrypoint {} with resource id {}.", getClass().getSimpleName(), resourceId);
        workingDirectory = ClusterEntrypointUtils.createJobManagerWorkingDirectory(configuration, resourceId);
        LOG.info("Using working directory: {}.", workingDirectory);
        rpcSystem = RpcSystem.load(configuration);
        commonRpcService = RpcUtils.createRemoteRpcService(rpcSystem, configuration, configuration.getString(JobManagerOptions.ADDRESS), getRPCPortRange(configuration), configuration.getString(JobManagerOptions.BIND_HOST), configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));
        JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
        // update the configuration used to create the high availability services
        configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
        configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());
        ioExecutor = Executors.newFixedThreadPool(ClusterEntrypointUtils.getPoolSize(configuration), new ExecutorThreadFactory("cluster-io"));
        haServices = createHaServices(configuration, ioExecutor, rpcSystem);
        blobServer = BlobUtils.createBlobServer(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), haServices.createBlobStore());
        blobServer.start();
        configuration.setString(BlobServerOptions.PORT, String.valueOf(blobServer.getPort()));
        heartbeatServices = createHeartbeatServices(configuration);
        metricRegistry = createMetricRegistry(configuration, pluginManager, rpcSystem);
        final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress(), configuration.getString(JobManagerOptions.BIND_HOST), rpcSystem);
        metricRegistry.startQueryService(metricQueryServiceRpcService, null);
        final String hostname = RpcUtils.getHostname(commonRpcService);
        processMetricGroup = MetricUtils.instantiateProcessMetricGroup(metricRegistry, hostname, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
        executionGraphInfoStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
    }
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) RpcService(org.apache.flink.runtime.rpc.RpcService)

Example 3 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class NettyShuffleServiceFactory method createNettyShuffleEnvironment.

@VisibleForTesting
static NettyShuffleEnvironment createNettyShuffleEnvironment(NettyShuffleEnvironmentConfiguration config, ResourceID taskExecutorResourceId, TaskEventPublisher taskEventPublisher, ResultPartitionManager resultPartitionManager, MetricGroup metricGroup, Executor ioExecutor) {
    checkNotNull(config);
    checkNotNull(taskExecutorResourceId);
    checkNotNull(taskEventPublisher);
    checkNotNull(resultPartitionManager);
    checkNotNull(metricGroup);
    NettyConfig nettyConfig = config.nettyConfig();
    FileChannelManager fileChannelManager = new FileChannelManagerImpl(config.getTempDirs(), DIR_NAME_PREFIX);
    if (LOG.isInfoEnabled()) {
        LOG.info("Created a new {} for storing result partitions of BLOCKING shuffles. Used directories:\n\t{}", FileChannelManager.class.getSimpleName(), Arrays.stream(fileChannelManager.getPaths()).map(File::getAbsolutePath).collect(Collectors.joining("\n\t")));
    }
    ConnectionManager connectionManager = nettyConfig != null ? new NettyConnectionManager(resultPartitionManager, taskEventPublisher, nettyConfig, config.getMaxNumberOfConnections(), config.isConnectionReuseEnabled()) : new LocalConnectionManager();
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(config.numNetworkBuffers(), config.networkBufferSize(), config.getRequestSegmentsTimeout());
    // we create a separated buffer pool here for batch shuffle instead of reusing the network
    // buffer pool directly to avoid potential side effects of memory contention, for example,
    // dead lock or "insufficient network buffer" error
    BatchShuffleReadBufferPool batchShuffleReadBufferPool = new BatchShuffleReadBufferPool(config.batchShuffleReadMemoryBytes(), config.networkBufferSize());
    // we create a separated IO executor pool here for batch shuffle instead of reusing the
    // TaskManager IO executor pool directly to avoid the potential side effects of execution
    // contention, for example, too long IO or waiting time leading to starvation or timeout
    ExecutorService batchShuffleReadIOExecutor = Executors.newFixedThreadPool(Math.max(1, Math.min(batchShuffleReadBufferPool.getMaxConcurrentRequests(), 4 * Hardware.getNumberCPUCores())), new ExecutorThreadFactory("blocking-shuffle-io"));
    registerShuffleMetrics(metricGroup, networkBufferPool);
    ResultPartitionFactory resultPartitionFactory = new ResultPartitionFactory(resultPartitionManager, fileChannelManager, networkBufferPool, batchShuffleReadBufferPool, batchShuffleReadIOExecutor, config.getBlockingSubpartitionType(), config.networkBuffersPerChannel(), config.floatingNetworkBuffersPerGate(), config.networkBufferSize(), config.isBlockingShuffleCompressionEnabled(), config.getCompressionCodec(), config.getMaxBuffersPerChannel(), config.sortShuffleMinBuffers(), config.sortShuffleMinParallelism(), config.isSSLEnabled());
    SingleInputGateFactory singleInputGateFactory = new SingleInputGateFactory(taskExecutorResourceId, config, connectionManager, resultPartitionManager, taskEventPublisher, networkBufferPool);
    return new NettyShuffleEnvironment(taskExecutorResourceId, config, networkBufferPool, connectionManager, resultPartitionManager, fileChannelManager, resultPartitionFactory, singleInputGateFactory, ioExecutor, batchShuffleReadBufferPool, batchShuffleReadIOExecutor);
}
Also used : BatchShuffleReadBufferPool(org.apache.flink.runtime.io.disk.BatchShuffleReadBufferPool) NettyConfig(org.apache.flink.runtime.io.network.netty.NettyConfig) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) FileChannelManager(org.apache.flink.runtime.io.disk.FileChannelManager) FileChannelManagerImpl(org.apache.flink.runtime.io.disk.FileChannelManagerImpl) NettyConnectionManager(org.apache.flink.runtime.io.network.netty.NettyConnectionManager) ResultPartitionFactory(org.apache.flink.runtime.io.network.partition.ResultPartitionFactory) ExecutorService(java.util.concurrent.ExecutorService) SingleInputGateFactory(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateFactory) NettyConnectionManager(org.apache.flink.runtime.io.network.netty.NettyConnectionManager) File(java.io.File) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting)

Example 4 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class TaskManagerRunner method startTaskManagerRunnerServices.

private void startTaskManagerRunnerServices() throws Exception {
    synchronized (lock) {
        rpcSystem = RpcSystem.load(configuration);
        this.executor = Executors.newScheduledThreadPool(Hardware.getNumberCPUCores(), new ExecutorThreadFactory("taskmanager-future"));
        highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(configuration, executor, AddressResolution.NO_ADDRESS_RESOLUTION, rpcSystem, this);
        JMXService.startInstance(configuration.getString(JMXServerOptions.JMX_SERVER_PORT));
        rpcService = createRpcService(configuration, highAvailabilityServices, rpcSystem);
        this.resourceId = getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort());
        this.workingDirectory = ClusterEntrypointUtils.createTaskManagerWorkingDirectory(configuration, resourceId);
        LOG.info("Using working directory: {}", workingDirectory);
        HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);
        metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration, rpcSystem.getMaximumMessageSizeInBytes(configuration)), ReporterSetup.fromConfiguration(configuration, pluginManager));
        final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress(), configuration.getString(TaskManagerOptions.BIND_HOST), rpcSystem);
        metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId.unwrap());
        blobCacheService = BlobUtils.createBlobCacheService(configuration, Reference.borrowed(workingDirectory.unwrap().getBlobStorageDirectory()), highAvailabilityServices.createBlobStore(), null);
        final ExternalResourceInfoProvider externalResourceInfoProvider = ExternalResourceUtils.createStaticExternalResourceInfoProviderFromConfig(configuration, pluginManager);
        taskExecutorService = taskExecutorServiceFactory.createTaskExecutor(this.configuration, this.resourceId.unwrap(), rpcService, highAvailabilityServices, heartbeatServices, metricRegistry, blobCacheService, false, externalResourceInfoProvider, workingDirectory.unwrap(), this);
        handleUnexpectedTaskExecutorServiceTermination();
        MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture.thenAccept(ignored -> {
        }));
    }
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) TaskExecutorBlobService(org.apache.flink.runtime.blob.TaskExecutorBlobService) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Hardware(org.apache.flink.runtime.util.Hardware) RpcSystemUtils(org.apache.flink.runtime.rpc.RpcSystemUtils) LoggerFactory(org.slf4j.LoggerFactory) ExternalResourceUtils(org.apache.flink.runtime.externalresource.ExternalResourceUtils) LeaderRetrievalException(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) LeaderRetrievalUtils(org.apache.flink.runtime.util.LeaderRetrievalUtils) ReporterSetup(org.apache.flink.runtime.metrics.ReporterSetup) InetAddress(java.net.InetAddress) MemoryLogger(org.apache.flink.runtime.taskmanager.MemoryLogger) StateChangelogStorageLoader(org.apache.flink.runtime.state.changelog.StateChangelogStorageLoader) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) FunctionUtils(org.apache.flink.util.function.FunctionUtils) Duration(java.time.Duration) SecurityUtils(org.apache.flink.runtime.security.SecurityUtils) TaskManagerOptionsInternal(org.apache.flink.configuration.TaskManagerOptionsInternal) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) AkkaOptions(org.apache.flink.configuration.AkkaOptions) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) AbstractID(org.apache.flink.util.AbstractID) Collection(java.util.Collection) JvmShutdownSafeguard(org.apache.flink.runtime.util.JvmShutdownSafeguard) HighAvailabilityServicesUtils(org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils) MetricUtils(org.apache.flink.runtime.metrics.util.MetricUtils) GuardedBy(javax.annotation.concurrent.GuardedBy) SecurityConfiguration(org.apache.flink.runtime.security.SecurityConfiguration) EnvironmentInformation(org.apache.flink.runtime.util.EnvironmentInformation) StringUtils(org.apache.flink.util.StringUtils) Executors(java.util.concurrent.Executors) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) ExecutorUtils(org.apache.flink.util.ExecutorUtils) JMXService(org.apache.flink.management.jmx.JMXService) MetricGroup(org.apache.flink.metrics.MetricGroup) FileSystem(org.apache.flink.core.fs.FileSystem) PluginManager(org.apache.flink.core.plugin.PluginManager) ConfigurationParserUtils(org.apache.flink.runtime.util.ConfigurationParserUtils) ClusterEntrypointUtils(org.apache.flink.runtime.entrypoint.ClusterEntrypointUtils) Time(org.apache.flink.api.common.time.Time) TaskExecutorPartitionTrackerImpl(org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl) FlinkException(org.apache.flink.util.FlinkException) MetricRegistryImpl(org.apache.flink.runtime.metrics.MetricRegistryImpl) CompletableFuture(java.util.concurrent.CompletableFuture) ShutdownHookUtil(org.apache.flink.util.ShutdownHookUtil) ArrayList(java.util.ArrayList) MetricRegistryConfiguration(org.apache.flink.runtime.metrics.MetricRegistryConfiguration) TaskManagerOptions(org.apache.flink.configuration.TaskManagerOptions) RpcService(org.apache.flink.runtime.rpc.RpcService) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) FlinkSecurityManager(org.apache.flink.core.security.FlinkSecurityManager) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) ExecutorService(java.util.concurrent.ExecutorService) TaskManagerExceptionUtils(org.apache.flink.util.TaskManagerExceptionUtils) Logger(org.slf4j.Logger) AutoCloseableAsync(org.apache.flink.util.AutoCloseableAsync) Configuration(org.apache.flink.configuration.Configuration) TaskManagerMetricGroup(org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup) IOException(java.io.IOException) Reference(org.apache.flink.util.Reference) RpcUtils(org.apache.flink.runtime.rpc.RpcUtils) SignalHandler(org.apache.flink.runtime.util.SignalHandler) UndeclaredThrowableException(java.lang.reflect.UndeclaredThrowableException) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) DeterminismEnvelope(org.apache.flink.runtime.entrypoint.DeterminismEnvelope) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) PluginUtils(org.apache.flink.core.plugin.PluginUtils) WorkingDirectory(org.apache.flink.runtime.entrypoint.WorkingDirectory) JMXServerOptions(org.apache.flink.configuration.JMXServerOptions) BlobCacheService(org.apache.flink.runtime.blob.BlobCacheService) AddressResolution(org.apache.flink.runtime.rpc.AddressResolution) BlobUtils(org.apache.flink.runtime.blob.BlobUtils) FlinkParseException(org.apache.flink.runtime.entrypoint.FlinkParseException) RpcService(org.apache.flink.runtime.rpc.RpcService) MetricRegistryImpl(org.apache.flink.runtime.metrics.MetricRegistryImpl) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider)

Example 5 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class JdbcOutputFormat method open.

/**
 * Connects to the target database and initializes the prepared statement.
 *
 * @param taskNumber The number of the parallel instance.
 */
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    try {
        connectionProvider.getOrEstablishConnection();
    } catch (Exception e) {
        throw new IOException("unable to open JDBC writer", e);
    }
    jdbcStatementExecutor = createAndOpenStatementExecutor(statementExecutorFactory);
    if (executionOptions.getBatchIntervalMs() != 0 && executionOptions.getBatchSize() != 1) {
        this.scheduler = Executors.newScheduledThreadPool(1, new ExecutorThreadFactory("jdbc-upsert-output-format"));
        this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> {
            synchronized (JdbcOutputFormat.this) {
                if (!closed) {
                    try {
                        flush();
                    } catch (Exception e) {
                        flushException = e;
                    }
                }
            }
        }, executionOptions.getBatchIntervalMs(), executionOptions.getBatchIntervalMs(), TimeUnit.MILLISECONDS);
    }
}
Also used : ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) IOException(java.io.IOException) SQLException(java.sql.SQLException) IOException(java.io.IOException)

Aggregations

ExecutorThreadFactory (org.apache.flink.util.concurrent.ExecutorThreadFactory)12 ExecutorService (java.util.concurrent.ExecutorService)7 IOException (java.io.IOException)5 ExecutionException (java.util.concurrent.ExecutionException)3 RpcService (org.apache.flink.runtime.rpc.RpcService)3 InetSocketAddress (java.net.InetSocketAddress)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 VisibleForTesting (org.apache.flink.annotation.VisibleForTesting)2 Configuration (org.apache.flink.configuration.Configuration)2 IllegalConfigurationException (org.apache.flink.configuration.IllegalConfigurationException)2 MetricGroup (org.apache.flink.metrics.MetricGroup)2 ResourceID (org.apache.flink.runtime.clusterframework.types.ResourceID)2 ActorRef (akka.actor.ActorRef)1 File (java.io.File)1 UndeclaredThrowableException (java.lang.reflect.UndeclaredThrowableException)1 BindException (java.net.BindException)1 InetAddress (java.net.InetAddress)1 URL (java.net.URL)1 SQLException (java.sql.SQLException)1 Duration (java.time.Duration)1