Examples with LogPipelineLoader co.cask.cdap.logging.framework.LogPipelineLoader used on opensource projects

Example 1 with LogPipelineLoader

use of co.cask.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class LocalLogAppender method start.

@Override
public void start() {
    if (!started.compareAndSet(false, true)) {
        return;
    }
    // Load and starts all configured log processing pipelines
    LogPipelineLoader pipelineLoader = new LogPipelineLoader(cConf);
    Map<String, LogPipelineSpecification<AppenderContext>> specs = pipelineLoader.load(new Provider<AppenderContext>() {

        @Override
        public AppenderContext get() {
            return new LocalAppenderContext(datasetFramework, txClient, locationFactory, metricsCollectionService);
        }
    });
    // Use the event delay as the sync interval
    long syncIntervalMillis = cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    for (LogPipelineSpecification<AppenderContext> spec : specs.values()) {
        LogProcessorPipelineContext context = new LogProcessorPipelineContext(cConf, spec.getName(), spec.getContext(), spec.getContext().getMetricsContext(), spec.getContext().getInstanceId());
        LocalLogProcessorPipeline pipeline = new LocalLogProcessorPipeline(context, syncIntervalMillis);
        pipeline.startAndWait();
        pipelines.add(pipeline);
    }
    super.start();
}

Example 2 with LogPipelineLoader

use of co.cask.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class LogPipelineCheck method run.

@Override
public void run() throws Exception {
    // Because the logging framework supports usage of addition jars to host appender classes,
    // for validations, we need to construct a new classloader using all system jars plus the configured
    // additional log library jars.
    // A new classloader is needed because the logback doesn't support context classloader and requires the
    // LoggerContext class needs to be loaded from the same classloader as all appender classes.
    // We need to use reflection to load the LogPipelineLoader class and call validate on it
    // Collects all URLS used by the CDAP system classloader
    List<URL> urls = ClassLoaders.getClassLoaderURLs(getClass().getClassLoader(), new ArrayList<URL>());
    for (File libJar : LoggingUtil.getExtensionJars(cConf)) {
        urls.add(libJar.toURI().toURL());
    }
    // Serialize the cConf to a String. This is needed because the cConf field is
    // loaded from the CDAP system classloader and cannot be passed directly to the
    // LogPipelineLoader class that loaded from the new ClassLoader constructed above.
    StringWriter writer = new StringWriter();
    this.cConf.writeXml(writer);
    // Create a new classloader and run the following code using reflection
    //
    // CConfiguration cConf = CConfiguration.create();
    // cConf.clear();
    // cConf.addResource(inputStream);
    // new LogPipelineLoader(cConf).validate();
    ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), null);
    Class<?> cConfClass = classLoader.loadClass(CConfiguration.class.getName());
    Object cConf = cConfClass.getMethod("create").invoke(null);
    cConfClass.getMethod("clear").invoke(cConf);
    InputStream input = new ByteArrayInputStream(writer.toString().getBytes(StandardCharsets.UTF_8));
    cConfClass.getMethod("addResource", InputStream.class).invoke(cConf, input);
    Class<?> loaderClass = classLoader.loadClass(LogPipelineLoader.class.getName());
    Object loader = loaderClass.getConstructor(cConfClass).newInstance(cConf);
    try {
        loaderClass.getMethod("validate").invoke(loader);
    } catch (InvocationTargetException e) {
        // Translate the exception throw by the reflection call
        Throwable cause = e.getCause();
        // will be throw from the validate method.
        if (InvalidPipelineException.class.getName().equals(cause.getClass().getName())) {
            InvalidPipelineException ex = new InvalidPipelineException(cause.getMessage(), cause.getCause());
            ex.setStackTrace(cause.getStackTrace());
            throw ex;
        }
        Throwables.propagateIfPossible(cause, Exception.class);
        throw new RuntimeException(cause);
    }
    LOG.info("Log pipeline configurations verified.");
}

Example 3 with LogPipelineLoader

use of co.cask.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(new Supplier<Service>() {

            @Override
            public Service get() {
                return new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(topic, pipelineSpec.getCheckpointPrefix()), brokerService, config);
            }
        }, retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.start();
                }
            }));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.stop();
                }
            }));
        }
    };
}

Example 4 with LogPipelineLoader

use of co.cask.cdap.logging.framework.LogPipelineLoader in project cdap by caskdata.

the class StandaloneMain method startUp.

/**
   * Start the service.
   */
public void startUp() throws Exception {
    // Workaround for release of file descriptors opened by URLClassLoader - https://issues.cask.co/browse/CDAP-2841
    URLConnections.setDefaultUseCaches(false);
    cleanupTempDir();
    ConfigurationLogger.logImportantConfig(cConf);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    // TODO: CDAP-7688, remove next line after the issue is resolved
    injector.getInstance(MessagingHttpService.class).startAndWait();
    // Authorization bootstrapping is a blocking call, because CDAP will not start successfully if it does not
    // succeed on an authorization-enabled cluster
    authorizationBootstrapper.run();
    txService.startAndWait();
    metricsCollectionService.startAndWait();
    datasetService.startAndWait();
    serviceStore.startAndWait();
    streamService.startAndWait();
    // Validate the logging pipeline configuration.
    // Do it explicitly as Standalone doesn't have a separate master check phase as the distributed does.
    new LogPipelineLoader(cConf).validate();
    // It is recommended to initialize log appender after datasetService is started,
    // since log appender instantiates a dataset.
    logAppenderInitializer.initialize();
    Service.State state = appFabricServer.startAndWait();
    if (state != Service.State.RUNNING) {
        throw new Exception("Failed to start Application Fabric");
    }
    metricsQueryService.startAndWait();
    router.startAndWait();
    if (userInterfaceService != null) {
        userInterfaceService.startAndWait();
    }
    if (securityEnabled) {
        externalAuthenticationServer.startAndWait();
    }
    if (exploreExecutorService != null) {
        exploreExecutorService.startAndWait();
    }
    metadataService.startAndWait();
    if (trackerAppCreationService != null) {
        trackerAppCreationService.startAndWait();
    }
    wranglerAppCreationService.startAndWait();
    remoteSystemOperationsService.startAndWait();
    operationalStatsService.startAndWait();
    String protocol = sslEnabled ? "https" : "http";
    int dashboardPort = sslEnabled ? cConf.getInt(Constants.Dashboard.SSL_BIND_PORT) : cConf.getInt(Constants.Dashboard.BIND_PORT);
    System.out.println("CDAP Sandbox started successfully.");
    System.out.printf("Connect to the CDAP UI at %s://%s:%d\n", protocol, "localhost", dashboardPort);
}