Examples with FlowletLoggingContext co.cask.cdap.logging.context.FlowletLoggingContext used on opensource projects

Example 1 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class MockLogReader method generateLogs.

public void generateLogs() throws InterruptedException {
    // Add logs for app testApp2, flow testFlow1
    generateLogs(new FlowletLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp2", "testFlow1", "testFlowlet1", "", ""), NamespaceId.DEFAULT.app("testApp2").flow("testFlow1"), ProgramRunStatus.RUNNING);
    // Add logs for app testApp3, mapreduce testMapReduce1
    generateLogs(new MapReduceLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp3", "testMapReduce1", ""), NamespaceId.DEFAULT.app("testApp3").mr("testMapReduce1"), ProgramRunStatus.SUSPENDED);
    // Add logs for app testApp1, service testService1
    generateLogs(new UserServiceLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp4", "testService1", "test1", "", ""), NamespaceId.DEFAULT.app("testApp4").service("testService1"), ProgramRunStatus.RUNNING);
    // Add logs for app testApp1, mapreduce testMapReduce1
    generateLogs(new MapReduceLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testTemplate1", "testMapReduce1", ""), TEST_NAMESPACE_ID.app("testTemplate1").mr("testMapReduce1"), ProgramRunStatus.COMPLETED);
    // Add logs for app testApp1, flow testFlow1 in testNamespace
    generateLogs(new FlowletLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testApp1", "testFlow1", "testFlowlet1", "", ""), TEST_NAMESPACE_ID.app("testApp1").flow("testFlow1"), ProgramRunStatus.COMPLETED);
    // Add logs for app testApp1, service testService1 in testNamespace
    generateLogs(new UserServiceLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testApp4", "testService1", "test1", "", ""), TEST_NAMESPACE_ID.app("testApp4").service("testService1"), ProgramRunStatus.KILLED);
    // Add logs for testWorkflow1 in testNamespace
    generateLogs(new WorkflowLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testTemplate1", "testWorkflow1", "testRun1"), TEST_NAMESPACE_ID.app("testTemplate1").workflow("testWorkflow1"), ProgramRunStatus.COMPLETED);
    // Add logs for testWorkflow1 in default namespace
    generateLogs(new WorkflowLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testTemplate1", "testWorkflow1", "testRun2"), NamespaceId.DEFAULT.app("testTemplate1").workflow("testWorkflow1"), ProgramRunStatus.COMPLETED);
    generateWorkflowLogs();
}

Example 2 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    // noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LoggingModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}

Example 3 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class KafkaLogProcessorPipelineTest method testMetricsAppender.

@Test
public void testMetricsAppender() throws Exception {
    Injector injector = KAFKA_TESTER.getInjector();
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    collectionService.startAndWait();
    LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionSystemClient.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class));
    final File logDir = TEMP_FOLDER.newFolder();
    loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
    LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
    configurator.setContext(loggerContext);
    URL configURL = getClass().getClassLoader().getResource("pipeline-metric-appender.xml");
    Assert.assertNotNull(configURL);
    configurator.doConfigure(configURL);
    String topic = "metricsPipeline";
    TestCheckpointManager checkpointManager = new TestCheckpointManager();
    KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
    KAFKA_TESTER.createTopic(topic, 1);
    loggerContext.start();
    KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMetricAppender", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
    pipeline.startAndWait();
    // Publish some log messages to Kafka
    long now = System.currentTimeMillis();
    FlowletLoggingContext flowletLoggingContext = new FlowletLoggingContext("default", "app1", "flow1", "flowlet1", "run1", "instance1");
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.INFO, "0", now - 1000), createLoggingEvent("test.logger", Level.INFO, "2", now - 700), createLoggingEvent("test.logger", Level.INFO, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900), createLoggingEvent("test.logger", Level.DEBUG, "hidden", now - 600), createLoggingEvent("test.logger", Level.INFO, "4", now - 100)), flowletLoggingContext);
    WorkflowProgramLoggingContext workflowProgramLoggingContext = new WorkflowProgramLoggingContext("default", "app1", "wflow1", "run1", ProgramType.MAPREDUCE, "mr1", "mrun1");
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.WARN, "0", now - 1000), createLoggingEvent("test.logger", Level.WARN, "2", now - 700), createLoggingEvent("test.logger", Level.TRACE, "3", now - 500)), workflowProgramLoggingContext);
    ServiceLoggingContext serviceLoggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION);
    publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.ERROR, "0", now - 1000), createLoggingEvent("test.logger", Level.ERROR, "2", now - 700), createLoggingEvent("test.logger", Level.ERROR, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900)), serviceLoggingContext);
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    try {
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.info", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 5L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.debug", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.warn", AggregationFunction.SUM, // mapreduce metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.MAPREDUCE, "mr1", Constants.Metrics.Tag.RUN_ID, "mrun1"), new ArrayList<String>()), 2L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.trace", AggregationFunction.SUM, // workflow metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.WORKFLOW, "wflow1", Constants.Metrics.Tag.RUN_ID, "run1"), new ArrayList<String>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.services.log.error", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<String>()), 3L);
    } finally {
        pipeline.stopAndWait();
        loggerContext.stop();
        collectionService.stopAndWait();
    }
}

Example 4 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestFileLogging method testGetLogPrev.

@Test
public void testGetLogPrev() throws Exception {
    LoggingContext loggingContext = new FlowletLoggingContext("TFL_NS_1", "APP_1", "FLOW_1", "", "RUN1", "INSTANCE1");
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    LoggingTester tester = new LoggingTester();
    tester.testGetPrev(logReader, loggingContext);
}

Example 5 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestFileLogging method testGetLogNext.

@Test
public void testGetLogNext() throws Exception {
    LoggingContext loggingContext = new FlowletLoggingContext("TFL_NS_1", "APP_1", "FLOW_1", "", "RUN1", "INSTANCE1");
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    LoggingTester tester = new LoggingTester();
    tester.testGetNext(logReader, loggingContext);
}