Examples with ValidationManager com.linkedin.pinot.controller.validation.ValidationManager used on opensource projects

Example 1 with ValidationManager

use of com.linkedin.pinot.controller.validation.ValidationManager in project pinot by linkedin.

the class ControllerStarter method start.

public void start() {
    LOGGER.info("Starting Pinot controller");
    Utils.logVersions();
    component.getServers().add(Protocol.HTTP, Integer.parseInt(config.getControllerPort()));
    component.getClients().add(Protocol.FILE);
    component.getClients().add(Protocol.JAR);
    final Context applicationContext = component.getContext().createChildContext();
    LOGGER.info("Controller download url base: {}", config.generateVipUrl());
    LOGGER.info("Injecting configuration and resource manager to the API context");
    applicationContext.getAttributes().put(ControllerConf.class.toString(), config);
    applicationContext.getAttributes().put(PinotHelixResourceManager.class.toString(), helixResourceManager);
    MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
    connectionManager.getParams().setConnectionTimeout(config.getServerAdminRequestTimeoutSeconds());
    applicationContext.getAttributes().put(HttpConnectionManager.class.toString(), connectionManager);
    applicationContext.getAttributes().put(Executor.class.toString(), executorService);
    controllerRestApp.setContext(applicationContext);
    component.getDefaultHost().attach(controllerRestApp);
    MetricsHelper.initializeMetrics(config.subset("pinot.controller.metrics"));
    MetricsHelper.registerMetricsRegistry(_metricsRegistry);
    final ControllerMetrics controllerMetrics = new ControllerMetrics(_metricsRegistry);
    try {
        LOGGER.info("Starting Pinot Helix resource manager and connecting to Zookeeper");
        helixResourceManager.start();
        // Helix resource manager must be started in order to create PinotLLCRealtimeSegmentManager
        PinotLLCRealtimeSegmentManager.create(helixResourceManager, config, controllerMetrics);
        ValidationMetrics validationMetrics = new ValidationMetrics(_metricsRegistry);
        validationManager = new ValidationManager(validationMetrics, helixResourceManager, config, PinotLLCRealtimeSegmentManager.getInstance());
        LOGGER.info("Starting Pinot REST API component");
        component.start();
        LOGGER.info("Starting retention manager");
        retentionManager.start();
        LOGGER.info("Starting validation manager");
        validationManager.start();
        LOGGER.info("Starting realtime segment manager");
        realtimeSegmentsManager.start(controllerMetrics);
        PinotLLCRealtimeSegmentManager.getInstance().start();
        LOGGER.info("Starting segment status manager");
        segmentStatusChecker.start(controllerMetrics);
        LOGGER.info("Pinot controller ready and listening on port {} for API requests", config.getControllerPort());
        LOGGER.info("Controller services available at http://{}:{}/", config.getControllerHost(), config.getControllerPort());
    } catch (final Exception e) {
        LOGGER.error("Caught exception while starting controller", e);
        Utils.rethrowException(e);
        throw new AssertionError("Should not reach this");
    }
    controllerMetrics.addCallbackGauge("helix.connected", new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            return helixResourceManager.getHelixZkManager().isConnected() ? 1L : 0L;
        }
    });
    controllerMetrics.addCallbackGauge("helix.leader", new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            return helixResourceManager.getHelixZkManager().isLeader() ? 1L : 0L;
        }
    });
    controllerMetrics.addCallbackGauge("dataDir.exists", new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            return new File(config.getDataDir()).exists() ? 1L : 0L;
        }
    });
    controllerMetrics.addCallbackGauge("dataDir.fileOpLatencyMs", new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            File dataDir = new File(config.getDataDir());
            if (dataDir.exists()) {
                try {
                    long startTime = System.currentTimeMillis();
                    final File testFile = new File(dataDir, config.getControllerHost());
                    FileOutputStream outputStream = new FileOutputStream(testFile, false);
                    outputStream.write(Longs.toByteArray(System.currentTimeMillis()));
                    outputStream.flush();
                    outputStream.close();
                    FileUtils.deleteQuietly(testFile);
                    long endTime = System.currentTimeMillis();
                    return endTime - startTime;
                } catch (IOException e) {
                    LOGGER.warn("Caught exception while checking the data directory operation latency", e);
                    return DATA_DIRECTORY_EXCEPTION_VALUE;
                }
            } else {
                return DATA_DIRECTORY_MISSING_VALUE;
            }
        }
    });
    ServiceStatus.setServiceStatusCallback(new ServiceStatus.ServiceStatusCallback() {

        private boolean _isStarted = false;

        @Override
        public ServiceStatus.Status getServiceStatus() {
            if (_isStarted) {
                // If we've connected to Helix at some point, the instance status depends on being connected to ZK
                if (helixResourceManager.getHelixZkManager().isConnected()) {
                    return ServiceStatus.Status.GOOD;
                } else {
                    return ServiceStatus.Status.BAD;
                }
            }
            // Return starting until zk is connected
            if (!helixResourceManager.getHelixZkManager().isConnected()) {
                return ServiceStatus.Status.STARTING;
            } else {
                _isStarted = true;
                return ServiceStatus.Status.GOOD;
            }
        }
    });
    helixResourceManager.getHelixZkManager().addPreConnectCallback(new PreConnectCallback() {

        @Override
        public void onPreConnect() {
            controllerMetrics.addMeteredGlobalValue(ControllerMeter.HELIX_ZOOKEEPER_RECONNECTS, 1L);
        }
    });
    controllerMetrics.initializeGlobalMeters();
    ControllerRestApplication.setControllerMetrics(controllerMetrics);
}