Examples with TThreadPoolServer org.apache.thrift.server.TThreadPoolServer used on opensource projects

Example 21 with TThreadPoolServer

use of org.apache.thrift.server.TThreadPoolServer in project hive by apache.

the class HiveMetaStore method startMetaStore.

/**
 * Start Metastore based on a passed {@link HadoopThriftAuthBridge}.
 *
 * @param port The port on which the Thrift server will start to serve
 * @param bridge
 * @param conf Configuration overrides
 * @param startMetaStoreThreads Start the background threads (initiator, cleaner, statsupdater, etc.)
 * @param startedBackgroundThreads If startMetaStoreThreads is true, this AtomicBoolean will be switched to true,
 *  when all of the background threads are scheduled. Useful for testing purposes to wait
 *  until the MetaStore is fully initialized.
 * @throws Throwable
 */
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, Configuration conf, boolean startMetaStoreThreads, AtomicBoolean startedBackgroundThreads) throws Throwable {
    isMetaStoreRemote = true;
    // Server will create new threads up to max as necessary. After an idle
    // period, it will destroy threads to keep the number of threads in the
    // pool to min.
    long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE);
    int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS);
    int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS);
    boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE);
    boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
    boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
    HMSHandler baseHandler = new HMSHandler("new db based metaserver", conf, false);
    AuthFactory authFactory = new AuthFactory(bridge, conf, baseHandler);
    useSasl = authFactory.isSASLWithKerberizedHadoop();
    if (useSasl) {
        // we are in secure mode. Login using keytab
        String kerberosName = SecurityUtil.getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0");
        String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE);
        UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile);
        saslServer = authFactory.getSaslServer();
        delegationTokenManager = authFactory.getDelegationTokenManager();
    }
    TProcessor processor;
    TTransportFactory transFactory = authFactory.getAuthTransFactory(useSSL, conf);
    final TProtocolFactory protocolFactory;
    final TProtocolFactory inputProtoFactory;
    if (useCompactProtocol) {
        protocolFactory = new TCompactProtocol.Factory();
        inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize);
    } else {
        protocolFactory = new TBinaryProtocol.Factory();
        inputProtoFactory = new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize);
    }
    IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
    TServerSocket serverSocket;
    if (useSasl) {
        processor = saslServer.wrapProcessor(new ThriftHiveMetastore.Processor<>(handler));
        LOG.info("Starting DB backed MetaStore Server in Secure Mode");
    } else {
        // we are in unsecure mode.
        if (MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) {
            processor = new TUGIBasedProcessor<>(handler);
            LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
        } else {
            processor = new TSetIpAddressProcessor<>(handler);
            LOG.info("Starting DB backed MetaStore Server");
        }
    }
    msHost = MetastoreConf.getVar(conf, ConfVars.THRIFT_BIND_HOST);
    if (msHost != null && !msHost.trim().isEmpty()) {
        LOG.info("Binding host " + msHost + " for metastore server");
    }
    if (!useSSL) {
        serverSocket = SecurityUtils.getServerSocket(msHost, port);
    } else {
        String keyStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_PATH).trim();
        if (keyStorePath.isEmpty()) {
            throw new IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString() + " Not configured for SSL connection");
        }
        String keyStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
        String keyStoreType = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_TYPE).trim();
        String keyStoreAlgorithm = MetastoreConf.getVar(conf, ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
        // enable SSL support for HMS
        List<String> sslVersionBlacklist = new ArrayList<>();
        for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) {
            sslVersionBlacklist.add(sslVersion);
        }
        serverSocket = SecurityUtils.getServerSSLSocket(msHost, port, keyStorePath, keyStorePassword, keyStoreType, keyStoreAlgorithm, sslVersionBlacklist);
    }
    if (tcpKeepAlive) {
        serverSocket = new TServerSocketKeepAlive(serverSocket);
    }
    TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket).processor(processor).transportFactory(transFactory).protocolFactory(protocolFactory).inputProtocolFactory(inputProtoFactory).minWorkerThreads(minWorkerThreads).maxWorkerThreads(maxWorkerThreads);
    TServer tServer = new TThreadPoolServer(args);
    TServerEventHandler tServerEventHandler = new TServerEventHandler() {

        @Override
        public void preServe() {
        }

        @Override
        public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) {
            Metrics.getOpenConnectionsCounter().inc();
            return null;
        }

        @Override
        public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) {
            Metrics.getOpenConnectionsCounter().dec();
            // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already
            // cleaned up thread local RawStore. Otherwise, do it now.
            HMSHandler.cleanupHandlerContext();
        }

        @Override
        public void processContext(ServerContext serverContext, TTransport tTransport, TTransport tTransport1) {
        }
    };
    tServer.setServerEventHandler(tServerEventHandler);
    LOG.info("Started the new metaserver on port [" + port + "]...");
    LOG.info("Options.minWorkerThreads = " + minWorkerThreads);
    LOG.info("Options.maxWorkerThreads = " + maxWorkerThreads);
    LOG.info("TCP keepalive = " + tcpKeepAlive);
    LOG.info("Enable SSL = " + useSSL);
    logCompactionParameters(conf);
    boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);
    LOG.info("Direct SQL optimization = {}", directSqlEnabled);
    if (startMetaStoreThreads) {
        Lock metaStoreThreadsLock = new ReentrantLock();
        Condition startCondition = metaStoreThreadsLock.newCondition();
        AtomicBoolean startedServing = new AtomicBoolean();
        startMetaStoreThreads(conf, metaStoreThreadsLock, startCondition, startedServing, isMetastoreHousekeepingLeader(conf, getServerHostName()), startedBackgroundThreads);
        signalOtherThreadsToStart(tServer, metaStoreThreadsLock, startCondition, startedServing);
    }
    // If dynamic service discovery through ZooKeeper is enabled, add this server to the ZooKeeper.
    if (MetastoreConf.getVar(conf, ConfVars.THRIFT_SERVICE_DISCOVERY_MODE).equalsIgnoreCase("zookeeper")) {
        try {
            zooKeeperHelper = MetastoreConf.getZKConfig(conf);
            String serverInstanceURI = getServerInstanceURI(port);
            zooKeeperHelper.addServerInstanceToZooKeeper(serverInstanceURI, serverInstanceURI, null, new ZKDeRegisterWatcher(zooKeeperHelper));
            LOG.info("Metastore server instance with URL " + serverInstanceURI + " added to " + "the zookeeper");
        } catch (Exception e) {
            LOG.error("Error adding this metastore instance to ZooKeeper: ", e);
            throw e;
        }
    }
    tServer.serve();
}

Example 22 with TThreadPoolServer

use of org.apache.thrift.server.TThreadPoolServer in project jstorm by alibaba.

the class AMServer method simple.

public void simple(JstormAM.Processor processor) {
    try {
        TServerTransport serverTransport = new TServerSocket(port);
        TServer server = new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor));
        LOG.info("Starting the simple server...");
        server.serve();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 23 with TThreadPoolServer

use of org.apache.thrift.server.TThreadPoolServer in project metacat by Netflix.

the class AbstractThriftServer method startServing.

private void startServing(final ExecutorService executorService, final TServerTransport serverTransport) {
    if (!stopping.get()) {
        final TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport).processor(getProcessor()).executorService(executorService);
        server = new TThreadPoolServer(serverArgs);
        if (hasServerEventHandler()) {
            server.setServerEventHandler(getServerEventHandler());
        }
        final String threadName = getServerName() + "-thread-#" + serverThreadCount.incrementAndGet();
        new Thread(threadName) {

            @Override
            public void run() {
                log.debug("starting serving");
                try {
                    server.serve();
                } catch (Throwable t) {
                    if (!stopping.get()) {
                        log.error("Unexpected exception in {}. This probably " + "means that the worker pool was exhausted. " + "Increase 'metacat.thrift.server_max_worker_threads' " + "from {} or throttle the number of requests. " + "This server thread is not in a bad state so starting a new one.", getServerName(), config.getThriftServerMaxWorkerThreads(), t);
                        startServing(executorService, serverTransport);
                    } else {
                        log.debug("stopping serving");
                    }
                }
                log.debug("started serving");
            }
        }.start();
    }
}

Example 24 with TThreadPoolServer

use of org.apache.thrift.server.TThreadPoolServer in project rubix by qubole.

the class TestPoolingClient method startServer.

private static void startServer(final Configuration conf) throws TTransportException {
    if (server != null) {
        return;
    }
    Processor processor = new Processor();
    TServerTransport serverTransport = new TServerSocket(new TServerSocket.ServerSocketTransportArgs().bindAddr(new InetSocketAddress(PORT)).backlog(Integer.MAX_VALUE));
    server = new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(new TestingService.Processor(processor)).maxWorkerThreads(getServerMaxThreads(conf)));
    server.serve();
}

Example 25 with TThreadPoolServer

use of org.apache.thrift.server.TThreadPoolServer in project rubix by qubole.

the class BookKeeperServer method createThriftServer.

private void createThriftServer(Configuration conf, BookKeeper bookKeeper) {
    processor = new BookKeeperService.Processor(bookKeeper);
    log.info("Starting BookKeeperServer on port " + getBookKeeperServerPort(conf));
    try {
        TServerTransport serverTransport = new TServerSocket(new TServerSocket.ServerSocketTransportArgs().bindAddr(new InetSocketAddress(getBookKeeperServerPort(conf))).backlog(Integer.MAX_VALUE));
        server = new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor).maxWorkerThreads(getServerMaxThreads(conf)));
    } catch (TTransportException e) {
        throw new RuntimeException("Error starting BookKeeperServer", e);
    }
}