Examples with FileMetaDataWriter io.cdap.cdap.logging.meta.FileMetaDataWriter used on opensource projects

Example 6 with FileMetaDataWriter

use of io.cdap.cdap.logging.meta.FileMetaDataWriter in project cdap by caskdata.

the class FileMetadataCleanerTest method testScanAndDeleteNewMetadata.

@Test
@Ignore
public // TODO CDAP-14953 ignoring this test until this jira fixed
void testScanAndDeleteNewMetadata() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(transactionRunner);
    try {
        long currentTime = System.currentTimeMillis();
        long eventTimestamp = currentTime - 100;
        LogPathIdentifier logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testFlow");
        LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
        List<String> expected = new ArrayList<>();
        for (int i = 0; i < 100; i++) {
            Location location = locationFactory.create("testFlowFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        long tillTime = currentTime + 50;
        List<FileMetadataCleaner.DeletedEntry> deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51 rows, till time is inclusive
        Assert.assertEquals(51, deletedEntries.size());
        int count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now add 10 entries for spark
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testSpark");
        expected = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            Location location = locationFactory.create("testSparkFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        // lets keep the same till time - this should only delete the spark entries now
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51 rows, till time is inclusive
        Assert.assertEquals(10, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now add 10 entries in mr context in time range 60-70
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testMr");
        expected = new ArrayList<>();
        // flow should come up at the beginning in the expected list
        for (int i = 51; i <= 70; i++) {
            expected.add(locationFactory.create("testFlowFile" + i).toURI().getPath());
        }
        for (int i = 0; i < 10; i++) {
            Location location = locationFactory.create("testMrFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            expected.add(location.toURI().getPath());
        }
        List<String> nextExpected = new ArrayList<>();
        logPathIdentifier = new LogPathIdentifier("testNs2", "testApp", "testCustomAction");
        for (int i = 90; i < 100; i++) {
            Location location = locationFactory.create("testActionFile" + i);
            // values : event time is 100ms behind current timestamp
            fileMetaDataWriter.writeMetaData(logPathIdentifier, eventTimestamp + i, currentTime + i, location);
            nextExpected.add(location.toURI().getPath());
        }
        tillTime = currentTime + 70;
        // lets delete till 70.
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 51-70 files of flow and 0-9 files of spark files in that order and 0 files of action.
        Assert.assertEquals(30, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(expected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now delete till currentTime + 100, this should delete all remaining entries.
        // custom action should come first and then flow entries
        tillTime = currentTime + 100;
        // lets delete till 100.
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        // we should have deleted 90-99 of custom action(10) 71-99 (29) files of flow.
        for (int i = 71; i < 100; i++) {
            nextExpected.add(locationFactory.create("testFlowFile" + i).toURI().getPath());
        }
        Assert.assertEquals(39, deletedEntries.size());
        count = 0;
        for (FileMetadataCleaner.DeletedEntry deletedEntry : deletedEntries) {
            Assert.assertEquals(nextExpected.get(count), deletedEntry.getPath());
            count += 1;
        }
        // now lets do a delete with till time  = currentTime + 1000, this should return empty result
        tillTime = currentTime + 1000;
        deletedEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, 100);
        Assert.assertEquals(0, deletedEntries.size());
    } finally {
        // cleanup meta
        deleteAllMetaEntries(transactionRunner);
    }
}

Example 7 with FileMetaDataWriter

use of io.cdap.cdap.logging.meta.FileMetaDataWriter in project cdap by caskdata.

the class CDAPLogAppender method start.

@Override
public void start() {
    // These should all passed. The settings are from the cdap-log-pipeline.xml and the context must be AppenderContext
    Preconditions.checkState(dirPermissions != null, "Property dirPermissions cannot be null");
    Preconditions.checkState(filePermissions != null, "Property filePermissions cannot be null");
    Preconditions.checkState(syncIntervalBytes > 0, "Property syncIntervalBytes must be > 0.");
    Preconditions.checkState(maxFileLifetimeMs > 0, "Property maxFileLifetimeMs must be > 0");
    Preconditions.checkState(maxFileSizeInBytes > 0, "Property maxFileSizeInBytes must be > 0");
    Preconditions.checkState(fileRetentionDurationDays > 0, "Property fileRetentionDurationDays must be > 0");
    Preconditions.checkState(logCleanupIntervalMins > 0, "Property logCleanupIntervalMins must be > 0");
    Preconditions.checkState(fileCleanupBatchSize > 0, "Property fileCleanupBatchSize must be > 0");
    if (context instanceof AppenderContext) {
        AppenderContext context = (AppenderContext) this.context;
        logFileManager = new LogFileManager(dirPermissions, filePermissions, maxFileLifetimeMs, maxFileSizeInBytes, syncIntervalBytes, new FileMetaDataWriter(context.getTransactionRunner()), context.getLocationFactory());
        if (context.getInstanceId() == 0) {
            scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(Threads.createDaemonThreadFactory("log-clean-up"));
            FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(context.getTransactionRunner());
            LogCleaner logCleaner = new LogCleaner(fileMetadataCleaner, context.getLocationFactory(), TimeUnit.DAYS.toMillis(fileRetentionDurationDays), fileCleanupBatchSize);
            scheduledExecutorService.scheduleAtFixedRate(logCleaner, 10, logCleanupIntervalMins, TimeUnit.MINUTES);
        }
    } else if (!Boolean.TRUE.equals(context.getObject(Constants.Logging.PIPELINE_VALIDATION))) {
        throw new IllegalStateException("Expected logger context instance of " + AppenderContext.class.getName() + " but get " + context.getClass().getName());
    }
    super.start();
}

Example 8 with FileMetaDataWriter

use of io.cdap.cdap.logging.meta.FileMetaDataWriter in project cdap by caskdata.

the class FileMetadataTest method testFileMetadataReadWrite.

@Test
public void testFileMetadataReadWrite() throws Exception {
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(transactionRunner);
    LogPathIdentifier logPathIdentifier = new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", "testFlow");
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
    long currentTime = System.currentTimeMillis();
    for (int i = 10; i <= 100; i += 10) {
        // i is the event time
        fileMetaDataWriter.writeMetaData(logPathIdentifier, i, currentTime, location.append(Integer.toString(i)));
    }
    // for the timestamp 80, add new new log path id with different current time.
    fileMetaDataWriter.writeMetaData(logPathIdentifier, 80, currentTime + 1, location.append("81"));
    fileMetaDataWriter.writeMetaData(logPathIdentifier, 80, currentTime + 2, location.append("82"));
    // reader test
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    Assert.assertEquals(12, fileMetadataReader.listFiles(logPathIdentifier, 0, 100).size());
    Assert.assertEquals(5, fileMetadataReader.listFiles(logPathIdentifier, 20, 50).size());
    Assert.assertEquals(2, fileMetadataReader.listFiles(logPathIdentifier, 100, 150).size());
    // should include the latest file with event start time 80.
    List<LogLocation> locationList = fileMetadataReader.listFiles(logPathIdentifier, 81, 85);
    Assert.assertEquals(1, locationList.size());
    Assert.assertEquals(80, locationList.get(0).getEventTimeMs());
    Assert.assertEquals(location.append("82"), locationList.get(0).getLocation());
    Assert.assertEquals(1, fileMetadataReader.listFiles(logPathIdentifier, 150, 1000).size());
}