Examples with LogPipelineLoader io.cdap.cdap.logging.framework.LogPipelineLoader used on opensource projects

Example 1 with LogPipelineLoader

use of io.cdap.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class LogPipelineCheck method run.

@Override
public void run() throws Exception {
    // Because the logging framework supports usage of addition jars to host appender classes,
    // for validations, we need to construct a new classloader using all system jars plus the configured
    // additional log library jars.
    // A new classloader is needed because the logback doesn't support context classloader and requires the
    // LoggerContext class needs to be loaded from the same classloader as all appender classes.
    // We need to use reflection to load the LogPipelineLoader class and call validate on it
    // Collects all URLS used by the CDAP system classloader
    List<URL> urls = ClassLoaders.getClassLoaderURLs(getClass().getClassLoader(), new ArrayList<URL>());
    for (File libJar : LoggingUtil.getExtensionJars(cConf)) {
        urls.add(libJar.toURI().toURL());
    }
    // Serialize the cConf to a String. This is needed because the cConf field is
    // loaded from the CDAP system classloader and cannot be passed directly to the
    // LogPipelineLoader class that loaded from the new ClassLoader constructed above.
    StringWriter writer = new StringWriter();
    this.cConf.writeXml(writer);
    // Create a new classloader and run the following code using reflection
    // 
    // CConfiguration cConf = CConfiguration.create();
    // cConf.clear();
    // cConf.addResource(inputStream);
    // new LogPipelineLoader(cConf).validate();
    ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), null);
    Class<?> cConfClass = classLoader.loadClass(CConfiguration.class.getName());
    Object cConf = cConfClass.getMethod("create").invoke(null);
    cConfClass.getMethod("clear").invoke(cConf);
    InputStream input = new ByteArrayInputStream(writer.toString().getBytes(StandardCharsets.UTF_8));
    cConfClass.getMethod("addResource", InputStream.class).invoke(cConf, input);
    Class<?> loaderClass = classLoader.loadClass(LogPipelineLoader.class.getName());
    Object loader = loaderClass.getConstructor(cConfClass).newInstance(cConf);
    try {
        loaderClass.getMethod("validate").invoke(loader);
    } catch (InvocationTargetException e) {
        // Translate the exception throw by the reflection call
        Throwable cause = e.getCause();
        // will be throw from the validate method.
        if (InvalidPipelineException.class.getName().equals(cause.getClass().getName())) {
            InvalidPipelineException ex = new InvalidPipelineException(cause.getMessage(), cause.getCause());
            ex.setStackTrace(cause.getStackTrace());
            throw ex;
        }
        Throwables.propagateIfPossible(cause, Exception.class);
        throw new RuntimeException(cause);
    }
    LOG.info("Log pipeline configurations verified.");
}

Example 2 with LogPipelineLoader

use of io.cdap.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class LogBufferService method loadLogPipelines.

/**
 * Load log buffer pipelines.
 */
@SuppressWarnings("unchecked")
private List<LogBufferProcessorPipeline> loadLogPipelines() {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    List<LogBufferProcessorPipeline> bufferPipelines = new ArrayList<>();
    // Create one LogBufferProcessorPipeline per spec
    for (LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        CConfiguration cConf = pipelineSpec.getConf();
        AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf);
        LogBufferPipelineConfig config = new LogBufferPipelineConfig(bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS), cConf.getLong(Constants.LogBuffer.LOG_BUFFER_PIPELINE_BATCH_SIZE, 1000));
        CheckpointManager checkpointManager = checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix(), CheckpointManagerFactory.Type.LOG_BUFFER);
        LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), config, checkpointManager, 0);
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> pipeline, retryStrategy));
        bufferPipelines.add(pipeline);
        checkpointManagers.add(checkpointManager);
    }
    return bufferPipelines;
}

Example 3 with LogPipelineLoader

use of io.cdap.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class LocalLogAppender method start.

@Override
public void start() {
    // Load and starts all configured log processing pipelines
    LogPipelineLoader pipelineLoader = new LogPipelineLoader(cConf);
    Map<String, LogPipelineSpecification<AppenderContext>> specs = pipelineLoader.load(() -> new LocalAppenderContext(transactionRunner, locationFactory, metricsCollectionService));
    // Use the event delay as the sync interval
    long syncIntervalMillis = cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    List<LocalLogProcessorPipeline> pipelines = new ArrayList<>();
    Set<Thread> pipelineThreads = Collections.newSetFromMap(new IdentityHashMap<>());
    for (LogPipelineSpecification<AppenderContext> spec : specs.values()) {
        LogProcessorPipelineContext context = new LogProcessorPipelineContext(cConf, spec.getName(), spec.getContext(), spec.getContext().getMetricsContext(), spec.getContext().getInstanceId());
        LocalLogProcessorPipeline pipeline = new LocalLogProcessorPipeline(context, syncIntervalMillis);
        pipeline.startAndWait();
        pipelineThreads.add(pipeline.getAppenderThread());
        pipelines.add(pipeline);
    }
    this.pipelines.getAndSet(pipelines).forEach(LocalLogProcessorPipeline::stopAndWait);
    this.pipelineThreads.set(pipelineThreads);
    super.start();
}

Example 4 with LogPipelineLoader

use of io.cdap.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
@SuppressWarnings("unchecked")
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix() + topic, CheckpointManagerFactory.Type.KAFKA), brokerService, config), retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::start));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::stop));
        }
    };
}

Example 5 with LogPipelineLoader

use of io.cdap.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class StandaloneMain method startUp.

/**
 * Start the service.
 */
public void startUp() throws Exception {
    // Workaround for release of file descriptors opened by URLClassLoader - https://issues.cask.co/browse/CDAP-2841
    URLConnections.setDefaultUseCaches(false);
    ConfigurationLogger.logImportantConfig(cConf);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    // TODO: CDAP-7688, remove next line after the issue is resolved
    injector.getInstance(MessagingHttpService.class).startAndWait();
    if (txService != null) {
        txService.startAndWait();
    }
    // Define all StructuredTable before starting any services that need StructuredTable
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    metadataStorage.createIndex();
    metricsCollectionService.startAndWait();
    datasetOpExecutorService.startAndWait();
    datasetService.startAndWait();
    serviceStore.startAndWait();
    remoteExecutionTwillRunnerService.start();
    metadataSubscriberService.startAndWait();
    // Validate the logging pipeline configuration.
    // Do it explicitly as Standalone doesn't have a separate master check phase as the distributed does.
    new LogPipelineLoader(cConf).validate();
    // It is recommended to initialize log appender after datasetService is started,
    // since log appender instantiates a dataset.
    logAppenderInitializer.initialize();
    runtimeServer.startAndWait();
    Service.State state = appFabricServer.startAndWait();
    if (state != Service.State.RUNNING) {
        throw new Exception("Failed to start Application Fabric");
    }
    previewHttpServer.startAndWait();
    previewRunnerManager.startAndWait();
    metricsQueryService.startAndWait();
    logQueryService.startAndWait();
    router.startAndWait();
    if (userInterfaceService != null) {
        userInterfaceService.startAndWait();
    }
    if (SecurityUtil.isManagedSecurity(cConf)) {
        externalAuthenticationServer.startAndWait();
    }
    if (exploreExecutorService != null) {
        exploreExecutorService.startAndWait();
    }
    metadataService.startAndWait();
    operationalStatsService.startAndWait();
    secureStoreService.startAndWait();
    supportBundleInternalService.startAndWait();
    appFabricHealthCheckService.startAndWait();
    String protocol = sslEnabled ? "https" : "http";
    int dashboardPort = sslEnabled ? cConf.getInt(Constants.Dashboard.SSL_BIND_PORT) : cConf.getInt(Constants.Dashboard.BIND_PORT);
    System.out.println("CDAP Sandbox started successfully.");
    System.out.printf("Connect to the CDAP UI at %s://%s:%d\n", protocol, "localhost", dashboardPort);
}