Examples with LogPathIdentifier io.cdap.cdap.logging.appender.system.LogPathIdentifier used on opensource projects

Example 6 with LogPathIdentifier

use of io.cdap.cdap.logging.appender.system.LogPathIdentifier in project cdap by cdapio.

the class FileMetadataCleanerTest method testFileMetadataWithCommonContextPrefix.

@Test
public void testFileMetadataWithCommonContextPrefix() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(transactionRunner);
    try {
        List<LogPathIdentifier> logPathIdentifiers = new ArrayList<>();
        // this should be able to scan and delete common prefix programs like testFlow1, testFlow10 during clenaup.
        for (int i = 1; i <= 20; i++) {
            logPathIdentifiers.add(new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", String.format("testFlow%s", i)));
        }
        LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
        Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
        long currentTime = System.currentTimeMillis();
        long newCurrentTime = currentTime + 100;
        for (int i = 1; i <= 20; i++) {
            LogPathIdentifier identifier = logPathIdentifiers.get(i - 1);
            for (int j = 0; j < 10; j++) {
                fileMetaDataWriter.writeMetaData(identifier, newCurrentTime + j, newCurrentTime + j, location.append("testFileNew" + Integer.toString(j)));
            }
        }
        List<LogLocation> locations;
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from currentTime (0..9)
            Assert.assertEquals(10, locations.size());
        }
        long tillTime = newCurrentTime + 4;
        List<FileMetadataCleaner.DeletedEntry> deleteEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // 20 context, 5 entries each
        Assert.assertEquals(100, deleteEntries.size());
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from time (5..9)
            Assert.assertEquals(5, locations.size());
            int startIndex = 5;
            for (LogLocation logLocation : locations) {
                Assert.assertEquals(String.format("testFileNew%s", startIndex), logLocation.getLocation().getName());
                startIndex++;
            }
        }
    } finally {
        // cleanup meta
        deleteAllMetaEntries(transactionRunner);
    }
}

Example 7 with LogPathIdentifier

use of io.cdap.cdap.logging.appender.system.LogPathIdentifier in project cdap by cdapio.

the class FileMetadataTest method testFileMetadataReadWriteAcrossFormats.

@Test
public void testFileMetadataReadWriteAcrossFormats() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    LogPathIdentifier logPathIdentifier = new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", "testFlow");
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
    long currentTime = System.currentTimeMillis();
    long eventTime = currentTime + 20;
    long newCurrentTime = currentTime + 100;
    // 10 files in new format
    for (int i = 1; i <= 10; i++) {
        fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTime + i, newCurrentTime + i, location.append("testFileNew" + Integer.toString(i)));
    }
    // reader test
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    // scan only in new files time range
    List<LogLocation> locations = fileMetadataReader.listFiles(logPathIdentifier, eventTime + 2, eventTime + 6);
    // should include files from currentTime (1..6)
    Assert.assertEquals(6, locations.size());
    for (LogLocation logLocation : locations) {
        Assert.assertEquals(LogLocation.VERSION_1, logLocation.getFrameworkVersion());
    }
    // scan time range across formats
    locations = fileMetadataReader.listFiles(logPathIdentifier, currentTime + 2, eventTime + 6);
    // should include files from new range (1..6)
    Assert.assertEquals(6, locations.size());
    for (int i = 0; i < locations.size(); i++) {
        Assert.assertEquals(LogLocation.VERSION_1, locations.get(i).getFrameworkVersion());
        Assert.assertEquals(location.append("testFileNew" + Integer.toString(i + 1)), locations.get(i).getLocation());
    }
}

Example 8 with LogPathIdentifier

use of io.cdap.cdap.logging.appender.system.LogPathIdentifier in project cdap by caskdata.

the class FileMetadataCleanerTest method testFileMetadataWithCommonContextPrefix.

@Test
public void testFileMetadataWithCommonContextPrefix() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(transactionRunner);
    try {
        List<LogPathIdentifier> logPathIdentifiers = new ArrayList<>();
        // this should be able to scan and delete common prefix programs like testFlow1, testFlow10 during clenaup.
        for (int i = 1; i <= 20; i++) {
            logPathIdentifiers.add(new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", String.format("testFlow%s", i)));
        }
        LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
        Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
        long currentTime = System.currentTimeMillis();
        long newCurrentTime = currentTime + 100;
        for (int i = 1; i <= 20; i++) {
            LogPathIdentifier identifier = logPathIdentifiers.get(i - 1);
            for (int j = 0; j < 10; j++) {
                fileMetaDataWriter.writeMetaData(identifier, newCurrentTime + j, newCurrentTime + j, location.append("testFileNew" + Integer.toString(j)));
            }
        }
        List<LogLocation> locations;
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from currentTime (0..9)
            Assert.assertEquals(10, locations.size());
        }
        long tillTime = newCurrentTime + 4;
        List<FileMetadataCleaner.DeletedEntry> deleteEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // 20 context, 5 entries each
        Assert.assertEquals(100, deleteEntries.size());
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from time (5..9)
            Assert.assertEquals(5, locations.size());
            int startIndex = 5;
            for (LogLocation logLocation : locations) {
                Assert.assertEquals(String.format("testFileNew%s", startIndex), logLocation.getLocation().getName());
                startIndex++;
            }
        }
    } finally {
        // cleanup meta
        deleteAllMetaEntries(transactionRunner);
    }
}

Example 9 with LogPathIdentifier

use of io.cdap.cdap.logging.appender.system.LogPathIdentifier in project cdap by caskdata.

the class FileMetadataCleanerTest method testScanAndDeleteNewMetadata.

@Test
@Ignore
public // TODO CDAP-14953 ignoring this test until this jira fixed
void testScanAndDeleteNewMetadata() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(transactionRunner);
    try {
        long currentTime = System.currentTimeMillis();
        long eventTimestamp = currentTime - 100;
        LogPathIdentifier logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testFlow");
        LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
        List<String> expected = new ArrayList<>();
        for (int i = 0; i < 100; i++) {
            Location location = locationFactory.create("testFlowFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        long tillTime = currentTime + 50;
        List<FileMetadataCleaner.DeletedEntry> deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51 rows, till time is inclusive
        Assert.assertEquals(51, deletedEntries.size());
        int count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now add 10 entries for spark
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testSpark");
        expected = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            Location location = locationFactory.create("testSparkFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        // lets keep the same till time - this should only delete the spark entries now
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51 rows, till time is inclusive
        Assert.assertEquals(10, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now add 10 entries in mr context in time range 60-70
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testMr");
        expected = new ArrayList<>();
        // flow should come up at the beginning in the expected list
        for (int i = 51; i <= 70; i++) {
            expected.add(locationFactory.create("testFlowFile" + i).toURI().getPath());
        }
        for (int i = 0; i < 10; i++) {
            Location location = locationFactory.create("testMrFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        List<String> nextExpected = new ArrayList<>();
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testCustomAction");
        for (int i = 90; i < 100; i++) {
            Location location = locationFactory.create("testActionFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            nextExpected.add(location.toURI().getPath());
        }
        tillTime = currentTime + 70;
        // lets delete till 70.
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51-70 files of flow and 0-9 files of spark files in that order and 0 files of action.
        Assert.assertEquals(30, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now delete till currentTime + 100, this should delete all remaining entries.
        // custom action should come first and then flow entries
        tillTime = currentTime + 100;
        // lets delete till 100.
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 90-99 of custom action(10) 71-99 (29) files of flow.
        for (int i = 71; i < 100; i++) {
            nextExpected.add(locationFactory.create("testFlowFile" + i).toURI().getPath());
        }
        Assert.assertEquals(39, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(nextExpected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now lets do a delete with till time  = currentTime + 1000, this should return empty result
        tillTime = currentTime + 1000;
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        Assert.assertEquals(0, deletedEntries.size());
    } finally {
        // cleanup meta
        deleteAllMetaEntries(transactionRunner);
    }
}

Example 10 with LogPathIdentifier

use of io.cdap.cdap.logging.appender.system.LogPathIdentifier in project cdap by caskdata.

the class DistributedLogFrameworkTest method testFramework.

@Test
public void testFramework() throws Exception {
    DistributedLogFramework framework = injector.getInstance(DistributedLogFramework.class);
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    framework.startAndWait();
    // Send some logs to Kafka.
    LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test");
    // Make sure all events get flushed in the same batch
    long eventTimeBase = System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    final int msgCount = 50;
    for (int i = 0; i < msgCount; i++) {
        // Publish logs in descending timestamp order
        publishLog(cConf.get(Constants.Logging.KAFKA_TOPIC), context, ImmutableList.of(createLoggingEvent("io.cdap.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i)));
    }
    // Read the logs back. They should be sorted by timestamp order.
    final FileMetaDataReader metaDataReader = injector.getInstance(FileMetaDataReader.class);
    Tasks.waitFor(true, () -> {
        List<LogLocation> locations = metaDataReader.listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, "test"), 0, Long.MAX_VALUE);
        if (locations.size() != 1) {
            return false;
        }
        LogLocation location = locations.get(0);
        int i = 0;
        try {
            try (CloseableIterator<LogEvent> iter = location.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) {
                while (iter.hasNext()) {
                    String expectedMsg = "Testing " + (msgCount - i - 1);
                    LogEvent event = iter.next();
                    if (!expectedMsg.equals(event.getLoggingEvent().getMessage())) {
                        return false;
                    }
                    i++;
                }
                return i == msgCount;
            }
        } catch (Exception e) {
            // and the time when actual content are flushed to the file
            return false;
        }
    }, 10, TimeUnit.SECONDS, msgCount, TimeUnit.MILLISECONDS);
    framework.stopAndWait();
    String kafkaTopic = cConf.get(Constants.Logging.KAFKA_TOPIC);
    // Check the checkpoint is persisted correctly. Since all messages are processed,
    // the checkpoint should be the same as the message count.
    CheckpointManager<KafkaOffset> checkpointManager = getCheckpointManager(kafkaTopic);
    Checkpoint<KafkaOffset> checkpoint = checkpointManager.getCheckpoint(0);
    Assert.assertEquals(msgCount, checkpoint.getOffset().getNextOffset());
}