Search in sources :

Example 36 with MetricsRegistry

use of com.yammer.metrics.core.MetricsRegistry in project pinot by linkedin.

the class BrokerServerBuilder method buildNetwork.

public void buildNetwork() throws ConfigurationException {
    // build transport
    Configuration transportConfigs = _config.subset(TRANSPORT_CONFIG_PREFIX);
    TransportClientConf conf = new TransportClientConf();
    conf.init(transportConfigs);
    _registry = new MetricsRegistry();
    MetricsHelper.initializeMetrics(_config.subset(METRICS_CONFIG_PREFIX));
    MetricsHelper.registerMetricsRegistry(_registry);
    _brokerMetrics = new BrokerMetrics(_registry, !emitTableLevelMetrics());
    _brokerMetrics.initializeGlobalMeters();
    _state.set(State.INIT);
    _eventLoopGroup = new NioEventLoopGroup();
    /**
     * Some of the client metrics uses histogram which is doing synchronous operation.
     * These are fixed overhead per request/response.
     * TODO: Measure the overhead of this.
     */
    final NettyClientMetrics clientMetrics = new NettyClientMetrics(_registry, "client_");
    // Setup Netty Connection Pool
    _resourceManager = new PooledNettyClientResourceManager(_eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    _poolTimeoutExecutor = new ScheduledThreadPoolExecutor(50);
    // _requestSenderPool = MoreExecutors.sameThreadExecutor();
    _requestSenderPool = Executors.newCachedThreadPool();
    final ConnectionPoolConfig connPoolCfg = conf.getConnPool();
    _connPool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(connPoolCfg.getMinConnectionsPerServer(), connPoolCfg.getMaxConnectionsPerServer(), connPoolCfg.getIdleTimeoutMs(), connPoolCfg.getMaxBacklogPerServer(), _resourceManager, _poolTimeoutExecutor, _requestSenderPool, _registry);
    // MoreExecutors.sameThreadExecutor(), _registry);
    _resourceManager.setPool(_connPool);
    // Setup Routing Table
    if (conf.getRoutingMode() == RoutingMode.CONFIG) {
        final CfgBasedRouting rt = new CfgBasedRouting();
        rt.init(conf.getCfgBasedRouting());
        _routingTable = rt;
    } else {
    // Helix based routing is already initialized.
    }
    // Setup ScatterGather
    _scatterGather = new ScatterGatherImpl(_connPool, _requestSenderPool);
    // Setup Broker Request Handler
    ReduceServiceRegistry reduceServiceRegistry = buildReduceServiceRegistry();
    _requestHandler = new BrokerRequestHandler(_routingTable, _timeBoundaryService, _scatterGather, reduceServiceRegistry, _brokerMetrics, _config);
    LOGGER.info("Network initialized !!");
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ConnectionPoolConfig(com.linkedin.pinot.transport.config.ConnectionPoolConfig) NettyClientMetrics(com.linkedin.pinot.transport.metrics.NettyClientMetrics) Configuration(org.apache.commons.configuration.Configuration) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) PooledNettyClientResourceManager(com.linkedin.pinot.transport.netty.PooledNettyClientResourceManager) HashedWheelTimer(io.netty.util.HashedWheelTimer) NettyClientConnection(com.linkedin.pinot.transport.netty.NettyClientConnection) CfgBasedRouting(com.linkedin.pinot.routing.CfgBasedRouting) ScatterGatherImpl(com.linkedin.pinot.transport.scattergather.ScatterGatherImpl) ReduceServiceRegistry(com.linkedin.pinot.common.query.ReduceServiceRegistry) TransportClientConf(com.linkedin.pinot.transport.conf.TransportClientConf) BrokerMetrics(com.linkedin.pinot.common.metrics.BrokerMetrics) ServerInstance(com.linkedin.pinot.common.response.ServerInstance) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup) BrokerRequestHandler(com.linkedin.pinot.broker.requesthandler.BrokerRequestHandler)

Example 37 with MetricsRegistry

use of com.yammer.metrics.core.MetricsRegistry in project pinot by linkedin.

the class RealtimeQueriesSentinelTest method setup.

@BeforeClass
public void setup() throws Exception {
    TableDataManagerProvider.setServerMetrics(new ServerMetrics(new MetricsRegistry()));
    PINOT_SCHEMA = getTestSchema();
    PINOT_SCHEMA.setSchemaName("realtimeSchema");
    AVRO_RECORD_TRANSFORMER = new AvroRecordToPinotRowGenerator(PINOT_SCHEMA);
    final IndexSegment indexSegment = getRealtimeSegment();
    setUpTestQueries("testTable");
    CONFIG_BUILDER = new TestingServerPropertiesBuilder("testTable");
    final PropertiesConfiguration serverConf = CONFIG_BUILDER.build();
    serverConf.setDelimiterParsingDisabled(false);
    final FileBasedInstanceDataManager instanceDataManager = FileBasedInstanceDataManager.getInstanceDataManager();
    instanceDataManager.init(new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance")));
    instanceDataManager.start();
    instanceDataManager.getTableDataManager("testTable");
    instanceDataManager.getTableDataManager("testTable").addSegment(indexSegment);
    QUERY_EXECUTOR = new ServerQueryExecutorV1Impl(false);
    QUERY_EXECUTOR.init(serverConf.subset("pinot.server.query.executor"), instanceDataManager, new ServerMetrics(new MetricsRegistry()));
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) AvroRecordToPinotRowGenerator(com.linkedin.pinot.core.realtime.impl.kafka.AvroRecordToPinotRowGenerator) IndexSegment(com.linkedin.pinot.core.indexsegment.IndexSegment) FileBasedInstanceDataManager(com.linkedin.pinot.core.data.manager.offline.FileBasedInstanceDataManager) ServerQueryExecutorV1Impl(com.linkedin.pinot.core.query.executor.ServerQueryExecutorV1Impl) ServerMetrics(com.linkedin.pinot.common.metrics.ServerMetrics) FileBasedInstanceDataManagerConfig(com.linkedin.pinot.core.data.manager.config.FileBasedInstanceDataManagerConfig) PropertiesConfiguration(org.apache.commons.configuration.PropertiesConfiguration) BeforeClass(org.testng.annotations.BeforeClass)

Example 38 with MetricsRegistry

use of com.yammer.metrics.core.MetricsRegistry in project pinot by linkedin.

the class HelixServerStarter method startServerInstance.

private void startServerInstance(Configuration moreConfigurations) throws Exception {
    Utils.logVersions();
    _serverConf = getInstanceServerConfig(moreConfigurations);
    setupHelixSystemProperties(moreConfigurations);
    if (_serverInstance == null) {
        _serverInstance = new ServerInstance();
        _serverInstance.init(_serverConf, new MetricsRegistry());
        _serverInstance.start();
        LOGGER.info("Started server instance");
    }
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ServerInstance(com.linkedin.pinot.server.starter.ServerInstance)

Example 39 with MetricsRegistry

use of com.yammer.metrics.core.MetricsRegistry in project pinot by linkedin.

the class HelixStarterTest method testSingleHelixServerStartAndTakingSegment.

@Test
public void testSingleHelixServerStartAndTakingSegment() throws Exception {
    Configuration pinotHelixProperties = new PropertiesConfiguration();
    pinotHelixProperties.addProperty("pinot.server.instance.id", "localhost:0000");
    ServerConf serverConf = DefaultHelixStarterServerConfig.getDefaultHelixServerConfig(pinotHelixProperties);
    ServerInstance serverInstance = new ServerInstance();
    serverInstance.init(serverConf, new MetricsRegistry());
    serverInstance.start();
    File segmentDir0 = new File(INDEX_DIR, "segment0");
    setupSegment(segmentDir0, "testTable0");
    File[] segment0Files = segmentDir0.listFiles();
    Assert.assertNotNull(segment0Files);
    File segmentDir1 = new File(INDEX_DIR, "segment1");
    setupSegment(segmentDir1, "testTable1");
    File[] segment1Files = segmentDir1.listFiles();
    Assert.assertNotNull(segment1Files);
    File segmentDir2 = new File(INDEX_DIR, "segment2");
    setupSegment(segmentDir2, "testTable2");
    File[] segment2Files = segmentDir2.listFiles();
    Assert.assertNotNull(segment2Files);
    DataManager instanceDataManager = serverInstance.getInstanceDataManager();
    instanceDataManager.addSegment(columnarSegmentMetadataLoader.loadIndexSegmentMetadataFromDir(segment0Files[0].getAbsolutePath()), null, null);
    instanceDataManager.addSegment(columnarSegmentMetadataLoader.loadIndexSegmentMetadataFromDir(segment1Files[0].getAbsolutePath()), null, null);
    instanceDataManager.addSegment(columnarSegmentMetadataLoader.loadIndexSegmentMetadataFromDir(segment2Files[0].getAbsolutePath()), null, null);
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) ServerConf(com.linkedin.pinot.server.conf.ServerConf) Configuration(org.apache.commons.configuration.Configuration) PropertiesConfiguration(org.apache.commons.configuration.PropertiesConfiguration) DataManager(com.linkedin.pinot.common.data.DataManager) ServerInstance(com.linkedin.pinot.server.starter.ServerInstance) PropertiesConfiguration(org.apache.commons.configuration.PropertiesConfiguration) File(java.io.File) Test(org.testng.annotations.Test)

Example 40 with MetricsRegistry

use of com.yammer.metrics.core.MetricsRegistry in project pinot by linkedin.

the class FileBasedServer method main.

public static void main(String[] args) throws Exception {
    //Process Command Line to get config and port
    processCommandLineArgs(args);
    LOGGER.info("Trying to build server config");
    MetricsRegistry metricsRegistry = new MetricsRegistry();
    ServerBuilder serverBuilder = new ServerBuilder(new File(_serverConfigPath), metricsRegistry);
    LOGGER.info("Trying to build InstanceDataManager");
    final DataManager instanceDataManager = serverBuilder.buildInstanceDataManager();
    LOGGER.info("Trying to start InstanceDataManager");
    instanceDataManager.start();
    //    bootstrapSegments(instanceDataManager);
    LOGGER.info("Trying to build QueryExecutor");
    final QueryExecutor queryExecutor = serverBuilder.buildQueryExecutor(instanceDataManager);
    final QueryScheduler queryScheduler = serverBuilder.buildQueryScheduler(queryExecutor);
    LOGGER.info("Trying to build RequestHandlerFactory");
    RequestHandlerFactory simpleRequestHandlerFactory = serverBuilder.buildRequestHandlerFactory(queryScheduler);
    LOGGER.info("Trying to build NettyServer");
    NettyServer nettyServer = new NettyTCPServer(_serverPort, simpleRequestHandlerFactory, null);
    Thread serverThread = new Thread(nettyServer);
    ShutdownHook shutdownHook = new ShutdownHook(nettyServer);
    serverThread.start();
    Runtime.getRuntime().addShutdownHook(shutdownHook);
}
Also used : MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) QueryScheduler(com.linkedin.pinot.core.query.scheduler.QueryScheduler) QueryExecutor(com.linkedin.pinot.common.query.QueryExecutor) RequestHandlerFactory(com.linkedin.pinot.transport.netty.NettyServer.RequestHandlerFactory) DataManager(com.linkedin.pinot.common.data.DataManager) NettyTCPServer(com.linkedin.pinot.transport.netty.NettyTCPServer) File(java.io.File) NettyServer(com.linkedin.pinot.transport.netty.NettyServer)

Aggregations

MetricsRegistry (com.yammer.metrics.core.MetricsRegistry)45 Test (org.testng.annotations.Test)20 ServerMetrics (com.linkedin.pinot.common.metrics.ServerMetrics)13 File (java.io.File)12 ArrayList (java.util.ArrayList)12 ControllerMetrics (com.linkedin.pinot.common.metrics.ControllerMetrics)10 ServerInstance (com.linkedin.pinot.common.response.ServerInstance)9 PropertiesConfiguration (org.apache.commons.configuration.PropertiesConfiguration)9 HelixAdmin (org.apache.helix.HelixAdmin)9 NettyClientMetrics (com.linkedin.pinot.transport.metrics.NettyClientMetrics)8 NioEventLoopGroup (io.netty.channel.nio.NioEventLoopGroup)8 HashedWheelTimer (io.netty.util.HashedWheelTimer)8 ScheduledThreadPoolExecutor (java.util.concurrent.ScheduledThreadPoolExecutor)8 IdealState (org.apache.helix.model.IdealState)8 BrokerMetrics (com.linkedin.pinot.common.metrics.BrokerMetrics)7 FileBasedInstanceDataManager (com.linkedin.pinot.core.data.manager.offline.FileBasedInstanceDataManager)6 NettyClientConnection (com.linkedin.pinot.transport.netty.NettyClientConnection)6 PooledNettyClientResourceManager (com.linkedin.pinot.transport.netty.PooledNettyClientResourceManager)6 EventLoopGroup (io.netty.channel.EventLoopGroup)6 HashMap (java.util.HashMap)6