Examples with ExecutorThreadFactory org.apache.flink.util.concurrent.ExecutorThreadFactory used on opensource projects

Example 6 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class JobManagerSharedServices method fromConfiguration.

// ------------------------------------------------------------------------
// Creating the components from a configuration
// ------------------------------------------------------------------------
public static JobManagerSharedServices fromConfiguration(Configuration config, BlobServer blobServer, FatalErrorHandler fatalErrorHandler) throws Exception {
    checkNotNull(config);
    checkNotNull(blobServer);
    final String classLoaderResolveOrder = config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);
    final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);
    final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
    final boolean checkClassLoaderLeak = config.getBoolean(CoreOptions.CHECK_LEAKED_CLASSLOADER);
    final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(blobServer, BlobLibraryCacheManager.defaultClassLoaderFactory(FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns, failOnJvmMetaspaceOomError ? fatalErrorHandler : null, checkClassLoaderLeak));
    final int numberCPUCores = Hardware.getNumberCPUCores();
    final int jobManagerFuturePoolSize = config.getInteger(JobManagerOptions.JOB_MANAGER_FUTURE_POOL_SIZE, numberCPUCores);
    final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(jobManagerFuturePoolSize, new ExecutorThreadFactory("jobmanager-future"));
    final int jobManagerIoPoolSize = config.getInteger(JobManagerOptions.JOB_MANAGER_IO_POOL_SIZE, numberCPUCores);
    final ExecutorService ioExecutor = Executors.newFixedThreadPool(jobManagerIoPoolSize, new ExecutorThreadFactory("jobmanager-io"));
    final ShuffleMasterContext shuffleMasterContext = new ShuffleMasterContextImpl(config, fatalErrorHandler);
    final ShuffleMaster<?> shuffleMaster = ShuffleServiceLoader.loadShuffleServiceFactory(config).createShuffleMaster(shuffleMasterContext);
    shuffleMaster.start();
    return new JobManagerSharedServices(futureExecutor, ioExecutor, libraryCacheManager, shuffleMaster, blobServer);
}

Example 7 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class AkkaRpcService method startSupervisorActor.

private Supervisor startSupervisorActor() {
    final ExecutorService terminationFutureExecutor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory("AkkaRpcService-Supervisor-Termination-Future-Executor"));
    final ActorRef actorRef = SupervisorActor.startSupervisorActor(actorSystem, withContextClassLoader(terminationFutureExecutor, flinkClassLoader));
    return Supervisor.create(actorRef, terminationFutureExecutor);
}

Example 8 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class Fabric8FlinkKubeClientTest method testIOExecutorShouldBeShutDownWhenFlinkKubeClientClosed.

@Test
public void testIOExecutorShouldBeShutDownWhenFlinkKubeClientClosed() {
    final ExecutorService executorService = Executors.newFixedThreadPool(2, new ExecutorThreadFactory("Testing-IO"));
    final FlinkKubeClient flinkKubeClient = new Fabric8FlinkKubeClient(flinkConfig, kubeClient, executorService);
    flinkKubeClient.close();
    assertThat(executorService.isShutdown(), is(true));
}

Example 9 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class HBaseSinkFunction method open.

@Override
public void open(Configuration parameters) throws Exception {
    LOG.info("start open ...");
    org.apache.hadoop.conf.Configuration config = prepareRuntimeConfiguration();
    try {
        this.mutationConverter.open();
        this.numPendingRequests = new AtomicLong(0);
        if (null == connection) {
            this.connection = ConnectionFactory.createConnection(config);
        }
        // create a parameter instance, set the table name and custom listener reference.
        BufferedMutatorParams params = new BufferedMutatorParams(TableName.valueOf(hTableName)).listener(this);
        if (bufferFlushMaxSizeInBytes > 0) {
            params.writeBufferSize(bufferFlushMaxSizeInBytes);
        }
        this.mutator = connection.getBufferedMutator(params);
        if (bufferFlushIntervalMillis > 0 && bufferFlushMaxMutations != 1) {
            this.executor = Executors.newScheduledThreadPool(1, new ExecutorThreadFactory("hbase-upsert-sink-flusher"));
            this.scheduledFuture = this.executor.scheduleWithFixedDelay(() -> {
                if (closed) {
                    return;
                }
                try {
                    flush();
                } catch (Exception e) {
                    // fail the sink and skip the rest of the items
                    // if the failure handler decides to throw an exception
                    failureThrowable.compareAndSet(null, e);
                }
            }, bufferFlushIntervalMillis, bufferFlushIntervalMillis, TimeUnit.MILLISECONDS);
        }
    } catch (TableNotFoundException tnfe) {
        LOG.error("The table " + hTableName + " not found ", tnfe);
        throw new RuntimeException("HBase table '" + hTableName + "' not found.", tnfe);
    } catch (IOException ioe) {
        LOG.error("Exception while creating connection to HBase.", ioe);
        throw new RuntimeException("Cannot create connection to HBase.", ioe);
    }
    LOG.info("end open.");
}

Example 10 with ExecutorThreadFactory

use of org.apache.flink.util.concurrent.ExecutorThreadFactory in project flink by apache.

the class RestServerEndpoint method start.

/**
 * Starts this REST server endpoint.
 *
 * @throws Exception if we cannot start the RestServerEndpoint
 */
public final void start() throws Exception {
    synchronized (lock) {
        Preconditions.checkState(state == State.CREATED, "The RestServerEndpoint cannot be restarted.");
        log.info("Starting rest endpoint.");
        final Router router = new Router();
        final CompletableFuture<String> restAddressFuture = new CompletableFuture<>();
        handlers = initializeHandlers(restAddressFuture);
        /* sort the handlers such that they are ordered the following:
             * /jobs
             * /jobs/overview
             * /jobs/:jobid
             * /jobs/:jobid/config
             * /:*
             */
        Collections.sort(handlers, RestHandlerUrlComparator.INSTANCE);
        checkAllEndpointsAndHandlersAreUnique(handlers);
        handlers.forEach(handler -> registerHandler(router, handler, log));
        ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {

            @Override
            protected void initChannel(SocketChannel ch) throws ConfigurationException {
                RouterHandler handler = new RouterHandler(router, responseHeaders);
                // SSL should be the first handler in the pipeline
                if (isHttpsEnabled()) {
                    ch.pipeline().addLast("ssl", new RedirectingSslHandler(restAddress, restAddressFuture, sslHandlerFactory));
                }
                ch.pipeline().addLast(new HttpServerCodec()).addLast(new FileUploadHandler(uploadDir)).addLast(new FlinkHttpObjectAggregator(maxContentLength, responseHeaders));
                for (InboundChannelHandlerFactory factory : inboundChannelHandlerFactories) {
                    Optional<ChannelHandler> channelHandler = factory.createHandler(configuration, responseHeaders);
                    if (channelHandler.isPresent()) {
                        ch.pipeline().addLast(channelHandler.get());
                    }
                }
                ch.pipeline().addLast(new ChunkedWriteHandler()).addLast(handler.getName(), handler).addLast(new PipelineErrorHandler(log, responseHeaders));
            }
        };
        NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new ExecutorThreadFactory("flink-rest-server-netty-boss"));
        NioEventLoopGroup workerGroup = new NioEventLoopGroup(0, new ExecutorThreadFactory("flink-rest-server-netty-worker"));
        bootstrap = new ServerBootstrap();
        bootstrap.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class).childHandler(initializer);
        Iterator<Integer> portsIterator;
        try {
            portsIterator = NetUtils.getPortRangeFromString(restBindPortRange);
        } catch (IllegalConfigurationException e) {
            throw e;
        } catch (Exception e) {
            throw new IllegalArgumentException("Invalid port range definition: " + restBindPortRange);
        }
        int chosenPort = 0;
        while (portsIterator.hasNext()) {
            try {
                chosenPort = portsIterator.next();
                final ChannelFuture channel;
                if (restBindAddress == null) {
                    channel = bootstrap.bind(chosenPort);
                } else {
                    channel = bootstrap.bind(restBindAddress, chosenPort);
                }
                serverChannel = channel.syncUninterruptibly().channel();
                break;
            } catch (final Exception e) {
                // otherwise
                if (!(e instanceof java.net.BindException)) {
                    throw e;
                }
            }
        }
        if (serverChannel == null) {
            throw new BindException("Could not start rest endpoint on any port in port range " + restBindPortRange);
        }
        log.debug("Binding rest endpoint to {}:{}.", restBindAddress, chosenPort);
        final InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress();
        final String advertisedAddress;
        if (bindAddress.getAddress().isAnyLocalAddress()) {
            advertisedAddress = this.restAddress;
        } else {
            advertisedAddress = bindAddress.getAddress().getHostAddress();
        }
        port = bindAddress.getPort();
        log.info("Rest endpoint listening at {}:{}", advertisedAddress, port);
        restBaseUrl = new URL(determineProtocol(), advertisedAddress, port, "").toString();
        restAddressFuture.complete(restBaseUrl);
        state = State.RUNNING;
        startInternal();
    }
}