Search in sources :

Example 6 with DatasetManager

use of co.cask.cdap.api.dataset.DatasetManager in project cdap by caskdata.

the class FileMetadataCleanerTest method testFileMetadataWithCommonContextPrefix.

@Test
public void testFileMetadataWithCommonContextPrefix() throws Exception {
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    DatasetManager datasetManager = new DefaultDatasetManager(datasetFramework, NamespaceId.SYSTEM, co.cask.cdap.common.service.RetryStrategies.noRetry());
    Transactional transactional = Transactions.createTransactionalWithRetry(Transactions.createTransactional(new MultiThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework), injector.getInstance(TransactionSystemClient.class), NamespaceId.SYSTEM, ImmutableMap.<String, String>of(), null, null)), RetryStrategies.retryOnConflict(20, 100));
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(datasetManager, transactional);
    FileMetaDataReader fileMetadataReader = injector.getInstance(FileMetaDataReader.class);
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(datasetManager, transactional);
    try {
        List<LogPathIdentifier> logPathIdentifiers = new ArrayList<>();
        // this should be able to scan and delete common prefix programs like testFlow1, testFlow10 during clenaup.
        for (int i = 1; i <= 20; i++) {
            logPathIdentifiers.add(new LogPathIdentifier(NamespaceId.DEFAULT.getNamespace(), "testApp", String.format("testFlow%s", i)));
        }
        LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
        Location location = locationFactory.create(TMP_FOLDER.newFolder().getPath()).append("/logs");
        long currentTime = System.currentTimeMillis();
        long newCurrentTime = currentTime + 100;
        for (int i = 1; i <= 20; i++) {
            LogPathIdentifier identifier = logPathIdentifiers.get(i - 1);
            for (int j = 0; j < 10; j++) {
                fileMetaDataWriter.writeMetaData(identifier, newCurrentTime + j, newCurrentTime + j, location.append("testFileNew" + Integer.toString(j)));
            }
        }
        List<LogLocation> locations;
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from currentTime (0..9)
            Assert.assertEquals(10, locations.size());
        }
        long tillTime = newCurrentTime + 4;
        List<FileMetadataCleaner.DeletedEntry> deleteEntries = fileMetadataCleaner.scanAndGetFilesToDelete(tillTime, TRANSACTION_TIMEOUT);
        // 20 context, 5 entries each
        Assert.assertEquals(100, deleteEntries.size());
        for (int i = 1; i <= 20; i++) {
            locations = fileMetadataReader.listFiles(logPathIdentifiers.get(i - 1), newCurrentTime, newCurrentTime + 10);
            // should include files from time (5..9)
            Assert.assertEquals(5, locations.size());
            int startIndex = 5;
            for (LogLocation logLocation : locations) {
                Assert.assertEquals(String.format("testFileNew%s", startIndex), logLocation.getLocation().getName());
                startIndex++;
            }
        }
    } finally {
        // cleanup meta
        cleanupMetadata(transactional, datasetManager);
    }
}
Also used : FileMetaDataWriter(co.cask.cdap.logging.meta.FileMetaDataWriter) DefaultDatasetManager(co.cask.cdap.data2.datafabric.dataset.DefaultDatasetManager) DatasetManager(co.cask.cdap.api.dataset.DatasetManager) ArrayList(java.util.ArrayList) DefaultDatasetManager(co.cask.cdap.data2.datafabric.dataset.DefaultDatasetManager) LocationFactory(org.apache.twill.filesystem.LocationFactory) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) MultiThreadDatasetCache(co.cask.cdap.data2.dataset2.MultiThreadDatasetCache) SystemDatasetInstantiator(co.cask.cdap.data.dataset.SystemDatasetInstantiator) LogLocation(co.cask.cdap.logging.write.LogLocation) FileMetaDataReader(co.cask.cdap.logging.meta.FileMetaDataReader) LogPathIdentifier(co.cask.cdap.logging.appender.system.LogPathIdentifier) Transactional(co.cask.cdap.api.Transactional) Location(org.apache.twill.filesystem.Location) LogLocation(co.cask.cdap.logging.write.LogLocation) Test(org.junit.Test)

Example 7 with DatasetManager

use of co.cask.cdap.api.dataset.DatasetManager in project cdap by caskdata.

the class LogCleanerTest method testLogCleanup.

@Test
public void testLogCleanup() throws Exception {
    // use file meta data manager to write meta data in old format
    // use file meta writer to write meta data in new format
    // scan for old files and make sure we only get the old meta data entries.
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    DatasetManager datasetManager = new DefaultDatasetManager(datasetFramework, NamespaceId.SYSTEM, co.cask.cdap.common.service.RetryStrategies.noRetry());
    Transactional transactional = Transactions.createTransactionalWithRetry(Transactions.createTransactional(new MultiThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework), injector.getInstance(TransactionSystemClient.class), NamespaceId.SYSTEM, ImmutableMap.<String, String>of(), null, null)), RetryStrategies.retryOnConflict(20, 100));
    FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(datasetManager, transactional);
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    long currentTime = System.currentTimeMillis();
    LogPathIdentifier logPathIdentifier = new LogPathIdentifier("testNs", "testApp", "testEntity");
    FileMetaDataWriter fileMetaDataWriter = new FileMetaDataWriter(datasetManager, transactional);
    long startTime = currentTime - 5000;
    Location dirLocation = locationFactory.create("logs");
    dirLocation.mkdirs();
    // create 20 files, add them in past time range
    for (int i = 0; i < 20; i++) {
        Location location = dirLocation.append("test" + i);
        location.createNew();
        fileMetaDataWriter.writeMetaData(logPathIdentifier, startTime + i, startTime + i, location);
    }
    Assert.assertEquals(20, dirLocation.list().size());
    LogCleaner logCleaner = new LogCleaner(fileMetadataCleaner, locationFactory, 100, 60);
    logCleaner.run();
    FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
    // all meta data should be deleted
    Assert.assertEquals(0, fileMetaDataReader.listFiles(logPathIdentifier, 0, System.currentTimeMillis()).size());
// we are not asserting file existence as the delete could fail and we don't guarantee file deletion.
}
Also used : FileMetaDataWriter(co.cask.cdap.logging.meta.FileMetaDataWriter) DefaultDatasetManager(co.cask.cdap.data2.datafabric.dataset.DefaultDatasetManager) DatasetManager(co.cask.cdap.api.dataset.DatasetManager) DefaultDatasetManager(co.cask.cdap.data2.datafabric.dataset.DefaultDatasetManager) LocationFactory(org.apache.twill.filesystem.LocationFactory) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) MultiThreadDatasetCache(co.cask.cdap.data2.dataset2.MultiThreadDatasetCache) SystemDatasetInstantiator(co.cask.cdap.data.dataset.SystemDatasetInstantiator) LogPathIdentifier(co.cask.cdap.logging.appender.system.LogPathIdentifier) FileMetaDataReader(co.cask.cdap.logging.meta.FileMetaDataReader) Transactional(co.cask.cdap.api.Transactional) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test)

Aggregations

Transactional (co.cask.cdap.api.Transactional)7 DatasetManager (co.cask.cdap.api.dataset.DatasetManager)7 SystemDatasetInstantiator (co.cask.cdap.data.dataset.SystemDatasetInstantiator)7 DefaultDatasetManager (co.cask.cdap.data2.datafabric.dataset.DefaultDatasetManager)7 DatasetFramework (co.cask.cdap.data2.dataset2.DatasetFramework)7 MultiThreadDatasetCache (co.cask.cdap.data2.dataset2.MultiThreadDatasetCache)7 FileMetaDataWriter (co.cask.cdap.logging.meta.FileMetaDataWriter)7 TransactionSystemClient (org.apache.tephra.TransactionSystemClient)7 LocationFactory (org.apache.twill.filesystem.LocationFactory)7 Test (org.junit.Test)7 LogPathIdentifier (co.cask.cdap.logging.appender.system.LogPathIdentifier)6 Location (org.apache.twill.filesystem.Location)6 FileMetaDataReader (co.cask.cdap.logging.meta.FileMetaDataReader)5 LogLocation (co.cask.cdap.logging.write.LogLocation)5 LoggingContext (co.cask.cdap.common.logging.LoggingContext)2 FileMetaDataManager (co.cask.cdap.logging.write.FileMetaDataManager)2 ArrayList (java.util.ArrayList)2 LoggingEvent (ch.qos.logback.classic.spi.LoggingEvent)1