Examples with ExecutorService java.util.concurrent.ExecutorService used on opensource projects

Example 26 with ExecutorService

use of java.util.concurrent.ExecutorService in project hbase by apache.

the class ThriftServerRunner method setupServer.

/**
   * Setting up the thrift TServer
   */
private void setupServer() throws Exception {
    // Construct correct ProtocolFactory
    TProtocolFactory protocolFactory;
    if (conf.getBoolean(COMPACT_CONF_KEY, false)) {
        LOG.debug("Using compact protocol");
        protocolFactory = new TCompactProtocol.Factory();
    } else {
        LOG.debug("Using binary protocol");
        protocolFactory = new TBinaryProtocol.Factory();
    }
    final TProcessor p = new Hbase.Processor<>(handler);
    ImplType implType = ImplType.getServerImpl(conf);
    TProcessor processor = p;
    // Construct correct TransportFactory
    TTransportFactory transportFactory;
    if (conf.getBoolean(FRAMED_CONF_KEY, false) || implType.isAlwaysFramed) {
        if (qop != null) {
            throw new RuntimeException("Thrift server authentication" + " doesn't work with framed transport yet");
        }
        transportFactory = new TFramedTransport.Factory(conf.getInt(MAX_FRAME_SIZE_CONF_KEY, 2) * 1024 * 1024);
        LOG.debug("Using framed transport");
    } else if (qop == null) {
        transportFactory = new TTransportFactory();
    } else {
        // Extract the name from the principal
        String name = SecurityUtil.getUserFromPrincipal(conf.get("hbase.thrift.kerberos.principal"));
        Map<String, String> saslProperties = new HashMap<>();
        saslProperties.put(Sasl.QOP, qop);
        TSaslServerTransport.Factory saslFactory = new TSaslServerTransport.Factory();
        saslFactory.addServerDefinition("GSSAPI", name, host, saslProperties, new SaslGssCallbackHandler() {

            @Override
            public void handle(Callback[] callbacks) throws UnsupportedCallbackException {
                AuthorizeCallback ac = null;
                for (Callback callback : callbacks) {
                    if (callback instanceof AuthorizeCallback) {
                        ac = (AuthorizeCallback) callback;
                    } else {
                        throw new UnsupportedCallbackException(callback, "Unrecognized SASL GSSAPI Callback");
                    }
                }
                if (ac != null) {
                    String authid = ac.getAuthenticationID();
                    String authzid = ac.getAuthorizationID();
                    if (!authid.equals(authzid)) {
                        ac.setAuthorized(false);
                    } else {
                        ac.setAuthorized(true);
                        String userName = SecurityUtil.getUserFromPrincipal(authzid);
                        LOG.info("Effective user: " + userName);
                        ac.setAuthorizedID(userName);
                    }
                }
            }
        });
        transportFactory = saslFactory;
        // Create a processor wrapper, to get the caller
        processor = new TProcessor() {

            @Override
            public boolean process(TProtocol inProt, TProtocol outProt) throws TException {
                TSaslServerTransport saslServerTransport = (TSaslServerTransport) inProt.getTransport();
                SaslServer saslServer = saslServerTransport.getSaslServer();
                String principal = saslServer.getAuthorizationID();
                hbaseHandler.setEffectiveUser(principal);
                return p.process(inProt, outProt);
            }
        };
    }
    if (conf.get(BIND_CONF_KEY) != null && !implType.canSpecifyBindIP) {
        LOG.error("Server types " + Joiner.on(", ").join(ImplType.serversThatCannotSpecifyBindIP()) + " don't support IP " + "address binding at the moment. See " + "https://issues.apache.org/jira/browse/HBASE-2155 for details.");
        throw new RuntimeException("-" + BIND_CONF_KEY + " not supported with " + implType);
    }
    // Thrift's implementation uses '0' as a placeholder for 'use the default.'
    int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
    if (implType == ImplType.HS_HA || implType == ImplType.NONBLOCKING || implType == ImplType.THREADED_SELECTOR) {
        InetAddress listenAddress = getBindAddress(conf);
        TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(new InetSocketAddress(listenAddress, listenPort));
        if (implType == ImplType.NONBLOCKING) {
            TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
            serverArgs.processor(processor).transportFactory(transportFactory).protocolFactory(protocolFactory);
            tserver = new TNonblockingServer(serverArgs);
        } else if (implType == ImplType.HS_HA) {
            THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
            CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(), metrics);
            ExecutorService executorService = createExecutor(callQueue, serverArgs.getMaxWorkerThreads(), serverArgs.getMaxWorkerThreads());
            serverArgs.executorService(executorService).processor(processor).transportFactory(transportFactory).protocolFactory(protocolFactory);
            tserver = new THsHaServer(serverArgs);
        } else {
            // THREADED_SELECTOR
            TThreadedSelectorServer.Args serverArgs = new HThreadedSelectorServerArgs(serverTransport, conf);
            CallQueue callQueue = new CallQueue(new LinkedBlockingQueue<>(), metrics);
            ExecutorService executorService = createExecutor(callQueue, serverArgs.getWorkerThreads(), serverArgs.getWorkerThreads());
            serverArgs.executorService(executorService).processor(processor).transportFactory(transportFactory).protocolFactory(protocolFactory);
            tserver = new TThreadedSelectorServer(serverArgs);
        }
        LOG.info("starting HBase " + implType.simpleClassName() + " server on " + Integer.toString(listenPort));
    } else if (implType == ImplType.THREAD_POOL) {
        // Thread pool server. Get the IP address to bind to.
        InetAddress listenAddress = getBindAddress(conf);
        int readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY, THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
        TServerTransport serverTransport = new TServerSocket(new TServerSocket.ServerSocketTransportArgs().bindAddr(new InetSocketAddress(listenAddress, listenPort)).backlog(backlog).clientTimeout(readTimeout));
        TBoundedThreadPoolServer.Args serverArgs = new TBoundedThreadPoolServer.Args(serverTransport, conf);
        serverArgs.processor(processor).transportFactory(transportFactory).protocolFactory(protocolFactory);
        LOG.info("starting " + ImplType.THREAD_POOL.simpleClassName() + " on " + listenAddress + ":" + Integer.toString(listenPort) + " with readTimeout " + readTimeout + "ms; " + serverArgs);
        TBoundedThreadPoolServer tserver = new TBoundedThreadPoolServer(serverArgs, metrics);
        this.tserver = tserver;
    } else {
        throw new AssertionError("Unsupported Thrift server implementation: " + implType.simpleClassName());
    }
    // A sanity check that we instantiated the right type of server.
    if (tserver.getClass() != implType.serverClass) {
        throw new AssertionError("Expected to create Thrift server class " + implType.serverClass.getName() + " but got " + tserver.getClass().getName());
    }
    registerFilters(conf);
}

Example 27 with ExecutorService

use of java.util.concurrent.ExecutorService in project hbase by apache.

the class ThriftServer method getTHsHaServer.

private static TServer getTHsHaServer(TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, int workerThreads, int maxCallQueueSize, InetSocketAddress inetSocketAddress, ThriftMetrics metrics) throws TTransportException {
    TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
    log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
    THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
    if (workerThreads > 0) {
        // Could support the min & max threads, avoiding to preserve existing functionality.
        serverArgs.minWorkerThreads(workerThreads).maxWorkerThreads(workerThreads);
    }
    ExecutorService executorService = createExecutor(workerThreads, maxCallQueueSize, metrics);
    serverArgs.executorService(executorService);
    serverArgs.processor(processor);
    serverArgs.transportFactory(transportFactory);
    serverArgs.protocolFactory(protocolFactory);
    return new THsHaServer(serverArgs);
}

Example 28 with ExecutorService

use of java.util.concurrent.ExecutorService in project crate by crate.

the class TestingHelpers method newMockedThreadPool.

public static ThreadPool newMockedThreadPool() {
    ThreadPool threadPool = Mockito.mock(ThreadPool.class);
    final ExecutorService executorService = Executors.newSingleThreadExecutor();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            executorService.shutdown();
            return null;
        }
    }).when(threadPool).shutdown();
    when(threadPool.executor(anyString())).thenReturn(executorService);
    try {
        doAnswer(new Answer() {

            @Override
            public Object answer(InvocationOnMock invocation) throws Throwable {
                executorService.awaitTermination(1, TimeUnit.SECONDS);
                return null;
            }
        }).when(threadPool).awaitTermination(anyLong(), any(TimeUnit.class));
    } catch (InterruptedException e) {
        throw Throwables.propagate(e);
    }
    return threadPool;
}

Example 29 with ExecutorService

use of java.util.concurrent.ExecutorService in project hive by apache.

the class Hive method trashFiles.

/**
   * Trashes or deletes all files under a directory. Leaves the directory as is.
   * @param fs FileSystem to use
   * @param statuses fileStatuses of files to be deleted
   * @param conf hive configuration
   * @return true if deletion successful
   * @throws IOException
   */
public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf) throws IOException {
    boolean result = true;
    if (statuses == null || statuses.length == 0) {
        return false;
    }
    final List<Future<Boolean>> futures = new LinkedList<>();
    final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ? Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Delete-Thread-%d").build()) : null;
    final SessionState parentSession = SessionState.get();
    for (final FileStatus status : statuses) {
        if (null == pool) {
            result &= FileUtils.moveToTrash(fs, status.getPath(), conf);
        } else {
            futures.add(pool.submit(new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    SessionState.setCurrentSessionState(parentSession);
                    return FileUtils.moveToTrash(fs, status.getPath(), conf);
                }
            }));
        }
    }
    if (null != pool) {
        pool.shutdown();
        for (Future<Boolean> future : futures) {
            try {
                result &= future.get();
            } catch (InterruptedException | ExecutionException e) {
                LOG.error("Failed to delete: ", e);
                pool.shutdownNow();
                throw new IOException(e);
            }
        }
    }
    return result;
}

Example 30 with ExecutorService

use of java.util.concurrent.ExecutorService in project hive by apache.

the class StatsNoJobTask method execute.

@Override
public int execute(DriverContext driverContext) {
    LOG.info("Executing stats (no job) task");
    String tableName = "";
    ExecutorService threadPool = null;
    Hive db = getHive();
    try {
        tableName = work.getTableSpecs().tableName;
        table = db.getTable(tableName);
        int numThreads = HiveConf.getIntVar(conf, ConfVars.HIVE_STATS_GATHER_NUM_THREADS);
        tableFullName = table.getDbName() + "." + table.getTableName();
        threadPool = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("StatsNoJobTask-Thread-%d").build());
        partUpdates = new MapMaker().concurrencyLevel(numThreads).makeMap();
        LOG.info("Initialized threadpool for stats computation with " + numThreads + " threads");
    } catch (HiveException e) {
        LOG.error("Cannot get table " + tableName, e);
        console.printError("Cannot get table " + tableName, e.toString());
    }
    return aggregateStats(threadPool, db);
}