Search in sources :

Example 6 with NettyClientConnection

use of com.linkedin.pinot.transport.netty.NettyClientConnection in project pinot by linkedin.

the class BrokerServerBuilder method buildNetwork.

public void buildNetwork() throws ConfigurationException {
    // build transport
    Configuration transportConfigs = _config.subset(TRANSPORT_CONFIG_PREFIX);
    TransportClientConf conf = new TransportClientConf();
    conf.init(transportConfigs);
    _registry = new MetricsRegistry();
    MetricsHelper.initializeMetrics(_config.subset(METRICS_CONFIG_PREFIX));
    MetricsHelper.registerMetricsRegistry(_registry);
    _brokerMetrics = new BrokerMetrics(_registry, !emitTableLevelMetrics());
    _brokerMetrics.initializeGlobalMeters();
    _state.set(State.INIT);
    _eventLoopGroup = new NioEventLoopGroup();
    /**
     * Some of the client metrics uses histogram which is doing synchronous operation.
     * These are fixed overhead per request/response.
     * TODO: Measure the overhead of this.
     */
    final NettyClientMetrics clientMetrics = new NettyClientMetrics(_registry, "client_");
    // Setup Netty Connection Pool
    _resourceManager = new PooledNettyClientResourceManager(_eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    _poolTimeoutExecutor = new ScheduledThreadPoolExecutor(50);
    // _requestSenderPool = MoreExecutors.sameThreadExecutor();
    _requestSenderPool = Executors.newCachedThreadPool();
    final ConnectionPoolConfig connPoolCfg = conf.getConnPool();
    _connPool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(connPoolCfg.getMinConnectionsPerServer(), connPoolCfg.getMaxConnectionsPerServer(), connPoolCfg.getIdleTimeoutMs(), connPoolCfg.getMaxBacklogPerServer(), _resourceManager, _poolTimeoutExecutor, _requestSenderPool, _registry);
    // MoreExecutors.sameThreadExecutor(), _registry);
    _resourceManager.setPool(_connPool);
    // Setup Routing Table
    if (conf.getRoutingMode() == RoutingMode.CONFIG) {
        final CfgBasedRouting rt = new CfgBasedRouting();
        rt.init(conf.getCfgBasedRouting());
        _routingTable = rt;
    } else {
    // Helix based routing is already initialized.
    }
    // Setup ScatterGather
    _scatterGather = new ScatterGatherImpl(_connPool, _requestSenderPool);
    // Setup Broker Request Handler
    ReduceServiceRegistry reduceServiceRegistry = buildReduceServiceRegistry();
    _requestHandler = new BrokerRequestHandler(_routingTable, _timeBoundaryService, _scatterGather, reduceServiceRegistry, _brokerMetrics, _config);
    LOGGER.info("Network initialized !!");
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ConnectionPoolConfig(com.linkedin.pinot.transport.config.ConnectionPoolConfig) NettyClientMetrics(com.linkedin.pinot.transport.metrics.NettyClientMetrics) Configuration(org.apache.commons.configuration.Configuration) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) PooledNettyClientResourceManager(com.linkedin.pinot.transport.netty.PooledNettyClientResourceManager) HashedWheelTimer(io.netty.util.HashedWheelTimer) NettyClientConnection(com.linkedin.pinot.transport.netty.NettyClientConnection) CfgBasedRouting(com.linkedin.pinot.routing.CfgBasedRouting) ScatterGatherImpl(com.linkedin.pinot.transport.scattergather.ScatterGatherImpl) ReduceServiceRegistry(com.linkedin.pinot.common.query.ReduceServiceRegistry) TransportClientConf(com.linkedin.pinot.transport.conf.TransportClientConf) BrokerMetrics(com.linkedin.pinot.common.metrics.BrokerMetrics) ServerInstance(com.linkedin.pinot.common.response.ServerInstance) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup) BrokerRequestHandler(com.linkedin.pinot.broker.requesthandler.BrokerRequestHandler)

Aggregations

ServerInstance (com.linkedin.pinot.common.response.ServerInstance)6 NettyClientMetrics (com.linkedin.pinot.transport.metrics.NettyClientMetrics)6 NettyClientConnection (com.linkedin.pinot.transport.netty.NettyClientConnection)6 PooledNettyClientResourceManager (com.linkedin.pinot.transport.netty.PooledNettyClientResourceManager)6 MetricsRegistry (com.yammer.metrics.core.MetricsRegistry)6 NioEventLoopGroup (io.netty.channel.nio.NioEventLoopGroup)6 HashedWheelTimer (io.netty.util.HashedWheelTimer)6 ScheduledThreadPoolExecutor (java.util.concurrent.ScheduledThreadPoolExecutor)6 BrokerMetrics (com.linkedin.pinot.common.metrics.BrokerMetrics)5 ThreadPoolExecutor (java.util.concurrent.ThreadPoolExecutor)5 SegmentId (com.linkedin.pinot.transport.common.SegmentId)4 SegmentIdSet (com.linkedin.pinot.transport.common.SegmentIdSet)4 NettyTCPServer (com.linkedin.pinot.transport.netty.NettyTCPServer)4 KeyedPoolImpl (com.linkedin.pinot.transport.pool.KeyedPoolImpl)4 ByteBuf (io.netty.buffer.ByteBuf)4 EventLoopGroup (io.netty.channel.EventLoopGroup)4 HashMap (java.util.HashMap)4 ExecutorService (java.util.concurrent.ExecutorService)4 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)4 Test (org.testng.annotations.Test)4