Examples with ServiceLoggingContext co.cask.cdap.common.logging.ServiceLoggingContext used on opensource projects

Example 6 with ServiceLoggingContext

use of co.cask.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class StreamHandlerRunnable method doInit.

@Override
protected Injector doInit(TwillContext context) {
    CConfiguration cConf = getCConfiguration();
    Configuration hConf = getConfiguration();
    // Set the host name to the one provided by Twill
    cConf.set(Constants.Stream.ADDRESS, context.getHost().getHostName());
    // Set the worker threads to number of cores * 2 available
    cConf.setInt(Constants.Stream.WORKER_THREADS, Runtime.getRuntime().availableProcessors() * 2);
    // Set the instance id
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCE_ID, context.getInstanceId());
    String txClientId = String.format("cdap.service.%s.%d", Constants.Service.STREAMS, context.getInstanceId());
    injector = createInjector(cConf, hConf, txClientId);
    injector.getInstance(LogAppenderInitializer.class).initialize();
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.STREAMS));
    return injector;
}

Example 7 with ServiceLoggingContext

use of co.cask.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class TransactionServiceTwillRunnable method doInit.

@Override
protected Injector doInit(TwillContext context) {
    getCConfiguration().set(Constants.Transaction.Container.ADDRESS, context.getHost().getCanonicalHostName());
    // Set the hostname of the machine so that cConf can be used to start internal services
    LOG.info("{} Setting host name to {}", name, context.getHost().getCanonicalHostName());
    String txClientId = String.format("cdap.service.%s.%d", Constants.Service.TRANSACTION, context.getInstanceId());
    injector = createGuiceInjector(getCConfiguration(), getConfiguration(), txClientId);
    injector.getInstance(LogAppenderInitializer.class).initialize();
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION));
    return injector;
}

Example 8 with ServiceLoggingContext

use of co.cask.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class LoggingContextHelper method getLoggingContext.

public static LoggingContext getLoggingContext(Map<String, String> tags) {
    // Tags are empty, cannot determine logging context.
    if (tags == null || tags.isEmpty()) {
        throw new IllegalArgumentException("Tags are empty, cannot determine logging context");
    }
    String namespaceId = getByNamespaceOrSystemID(tags);
    String applicationId = tags.get(ApplicationLoggingContext.TAG_APPLICATION_ID);
    String componentId = tags.get(ComponentLoggingContext.TAG_COMPONENT_ID);
    // No namespace id or either from application id or component id is not present
    if (namespaceId == null || (applicationId == null && componentId == null)) {
        throw new IllegalArgumentException("No namespace/application or system/component id present");
    }
    if (tags.containsKey(FlowletLoggingContext.TAG_FLOW_ID)) {
        if (!tags.containsKey(FlowletLoggingContext.TAG_FLOWLET_ID)) {
            return null;
        }
        return new FlowletLoggingContext(namespaceId, applicationId, tags.get(FlowletLoggingContext.TAG_FLOW_ID), tags.get(FlowletLoggingContext.TAG_FLOWLET_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID));
    } else if (tags.containsKey(WorkflowLoggingContext.TAG_WORKFLOW_ID)) {
        if (tags.containsKey(WorkflowProgramLoggingContext.TAG_WORKFLOW_MAP_REDUCE_ID)) {
            return new WorkflowProgramLoggingContext(namespaceId, applicationId, tags.get(WorkflowLoggingContext.TAG_WORKFLOW_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID), ProgramType.MAPREDUCE, tags.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_MAP_REDUCE_ID), tags.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_PROGRAM_RUN_ID));
        }
        if (tags.containsKey(WorkflowProgramLoggingContext.TAG_WORKFLOW_SPARK_ID)) {
            return new WorkflowProgramLoggingContext(namespaceId, applicationId, tags.get(WorkflowLoggingContext.TAG_WORKFLOW_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID), ProgramType.SPARK, tags.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_SPARK_ID), tags.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_PROGRAM_RUN_ID));
        }
        return new WorkflowLoggingContext(namespaceId, applicationId, tags.get(WorkflowLoggingContext.TAG_WORKFLOW_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID));
    } else if (tags.containsKey(MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID)) {
        return new MapReduceLoggingContext(namespaceId, applicationId, tags.get(MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID));
    } else if (tags.containsKey(SparkLoggingContext.TAG_SPARK_JOB_ID)) {
        return new SparkLoggingContext(namespaceId, applicationId, tags.get(SparkLoggingContext.TAG_SPARK_JOB_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID));
    } else if (tags.containsKey(UserServiceLoggingContext.TAG_USER_SERVICE_ID)) {
        if (!tags.containsKey(UserServiceLoggingContext.TAG_HANDLER_ID)) {
            return null;
        }
        return new UserServiceLoggingContext(namespaceId, applicationId, tags.get(UserServiceLoggingContext.TAG_USER_SERVICE_ID), tags.get(UserServiceLoggingContext.TAG_HANDLER_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID));
    } else if (tags.containsKey(ServiceLoggingContext.TAG_SERVICE_ID)) {
        return new ServiceLoggingContext(namespaceId, componentId, tags.get(ServiceLoggingContext.TAG_SERVICE_ID));
    } else if (tags.containsKey(WorkerLoggingContext.TAG_WORKER_ID)) {
        return new WorkerLoggingContext(namespaceId, applicationId, tags.get(WorkerLoggingContext.TAG_WORKER_ID), tags.get(ApplicationLoggingContext.TAG_RUN_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID));
    }
    throw new IllegalArgumentException("Unsupported logging context");
}

Example 9 with ServiceLoggingContext

use of co.cask.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class LoggingContextHelper method createFilter.

public static Filter createFilter(LoggingContext loggingContext) {
    if (loggingContext instanceof ServiceLoggingContext) {
        LoggingContext.SystemTag systemTag = getByNamespaceOrSystemID(loggingContext.getSystemTagsMap());
        if (systemTag == null) {
            throw new IllegalArgumentException("No namespace or system id present");
        }
        String systemId = systemTag.getValue();
        String componentId = loggingContext.getSystemTagsMap().get(ServiceLoggingContext.TAG_COMPONENT_ID).getValue();
        String tagName = ServiceLoggingContext.TAG_SERVICE_ID;
        String entityId = loggingContext.getSystemTagsMap().get(ServiceLoggingContext.TAG_SERVICE_ID).getValue();
        ImmutableList.Builder<Filter> filterBuilder = ImmutableList.builder();
        // In CDAP 3.5 we removed SystemLoggingContext which had tag .systemId and now we use .namespaceId but to
        // support backward compatibility have an or filter so that we can read old logs too. See CDAP-7482
        OrFilter namespaceFilter = new OrFilter(ImmutableList.of(new MdcExpression(NamespaceLoggingContext.TAG_NAMESPACE_ID, systemId), new MdcExpression(ServiceLoggingContext.TAG_SYSTEM_ID, systemId)));
        filterBuilder.add(namespaceFilter);
        filterBuilder.add(new MdcExpression(ServiceLoggingContext.TAG_COMPONENT_ID, componentId));
        filterBuilder.add(new MdcExpression(tagName, entityId));
        return new AndFilter(filterBuilder.build());
    } else {
        String namespaceId = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_NAMESPACE_ID).getValue();
        String applId = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_APPLICATION_ID).getValue();
        LoggingContext.SystemTag entityTag = getEntityId(loggingContext);
        ImmutableList.Builder<Filter> filterBuilder = ImmutableList.builder();
        // For backward compatibility: The old logs before namespace have .accountId and developer as value so we don't
        // want them to get filtered out if they belong to this application and entity
        OrFilter namespaceFilter = new OrFilter(ImmutableList.of(new MdcExpression(NamespaceLoggingContext.TAG_NAMESPACE_ID, namespaceId), new MdcExpression(ACCOUNT_ID, Constants.DEVELOPER_ACCOUNT)));
        filterBuilder.add(namespaceFilter);
        filterBuilder.add(new MdcExpression(ApplicationLoggingContext.TAG_APPLICATION_ID, applId));
        filterBuilder.add(new MdcExpression(entityTag.getName(), entityTag.getValue()));
        if (loggingContext instanceof WorkflowProgramLoggingContext) {
            // Program is started by Workflow. Add Program information to filter.
            Map<String, LoggingContext.SystemTag> systemTagsMap = loggingContext.getSystemTagsMap();
            LoggingContext.SystemTag programTag = systemTagsMap.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_MAP_REDUCE_ID);
            if (programTag != null) {
                filterBuilder.add(new MdcExpression(WorkflowProgramLoggingContext.TAG_WORKFLOW_MAP_REDUCE_ID, programTag.getValue()));
            }
            programTag = systemTagsMap.get(WorkflowProgramLoggingContext.TAG_WORKFLOW_SPARK_ID);
            if (programTag != null) {
                filterBuilder.add(new MdcExpression(WorkflowProgramLoggingContext.TAG_WORKFLOW_SPARK_ID, programTag.getValue()));
            }
        }
        // Add runid filter if required
        LoggingContext.SystemTag runId = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_RUN_ID);
        if (runId != null && runId.getValue() != null) {
            filterBuilder.add(new MdcExpression(ApplicationLoggingContext.TAG_RUN_ID, runId.getValue()));
        }
        return new AndFilter(filterBuilder.build());
    }
}

Example 10 with ServiceLoggingContext

use of co.cask.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class KafkaLogProcessorPipelineTest method testMetricsAppender.

@Test
public void testMetricsAppender() throws Exception {
    Injector injector = KAFKA_TESTER.getInjector();
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    collectionService.startAndWait();
    LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionSystemClient.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class));
    final File logDir = TEMP_FOLDER.newFolder();
    loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
    LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
    configurator.setContext(loggerContext);
    URL configURL = getClass().getClassLoader().getResource("pipeline-metric-appender.xml");
    Assert.assertNotNull(configURL);
    configurator.doConfigure(configURL);
    String topic = "metricsPipeline";
    TestCheckpointManager checkpointManager = new TestCheckpointManager();
    KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
    KAFKA_TESTER.createTopic(topic, 1);
    loggerContext.start();
    KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMetricAppender", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
    pipeline.startAndWait();
    // Publish some log messages to Kafka
    long now = System.currentTimeMillis();
    FlowletLoggingContext flowletLoggingContext = new FlowletLoggingContext("default", "app1", "flow1", "flowlet1", "run1", "instance1");
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.INFO, "0", now - 1000), createLoggingEvent("test.logger", Level.INFO, "2", now - 700), createLoggingEvent("test.logger", Level.INFO, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900), createLoggingEvent("test.logger", Level.DEBUG, "hidden", now - 600), createLoggingEvent("test.logger", Level.INFO, "4", now - 100)), flowletLoggingContext);
    WorkflowProgramLoggingContext workflowProgramLoggingContext = new WorkflowProgramLoggingContext("default", "app1", "wflow1", "run1", ProgramType.MAPREDUCE, "mr1", "mrun1");
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.WARN, "0", now - 1000), createLoggingEvent("test.logger", Level.WARN, "2", now - 700), createLoggingEvent("test.logger", Level.TRACE, "3", now - 500)), workflowProgramLoggingContext);
    ServiceLoggingContext serviceLoggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION);
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.ERROR, "0", now - 1000), createLoggingEvent("test.logger", Level.ERROR, "2", now - 700), createLoggingEvent("test.logger", Level.ERROR, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900)), serviceLoggingContext);
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    try {
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.info", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 5L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.debug", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.warn", AggregationFunction.SUM, // mapreduce metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.MAPREDUCE, "mr1", Constants.Metrics.Tag.RUN_ID, "mrun1"), new ArrayList<String>()), 2L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.trace", AggregationFunction.SUM, // workflow metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.WORKFLOW, "wflow1", Constants.Metrics.Tag.RUN_ID, "run1"), new ArrayList<String>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.services.log.error", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<String>()), 3L);
    } finally {
        pipeline.stopAndWait();
        loggerContext.stop();
        collectionService.stopAndWait();
    }
}