Search in sources :

Example 1 with LocalCacheCleanerStats

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.

the class TestLocalCacheCleanup method testBasicCleanup.

@Test
public void testBasicCleanup() {
    ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    addResource(publicRsrc, "/pub-resource1.txt", 5, 20, 0);
    addResource(publicRsrc, "/pub-resource2.txt", 3, 20, 0);
    addResource(publicRsrc, "/pub-resource3.txt", 15, 20, 0);
    ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
    ConcurrentMap<LocalResourceRequest, LocalizedResource> user1rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    addResource(user1rsrcs, "/private-u1-resource4.txt", 1, 20, 0);
    LocalResourcesTracker user1Tracker = new StubbedLocalResourcesTrackerImpl("user1", user1rsrcs);
    privateRsrc.put("user1", user1Tracker);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> user2rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    addResource(user2rsrcs, "/private-u2-resource5.txt", 2, 20, 0);
    LocalResourcesTracker user2Tracker = new StubbedLocalResourcesTrackerImpl("user2", user2rsrcs);
    privateRsrc.put("user2", user2Tracker);
    ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 0);
    LocalCacheCleanerStats stats = rls.handleCacheCleanup();
    assertEquals(0, ((StubbedLocalResourcesTrackerImpl) rls.publicRsrc).getLocalRsrc().size());
    assertEquals(0, ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user1")).getLocalRsrc().size());
    assertEquals(0, ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user2")).getLocalRsrc().size());
    assertEquals(100, stats.getTotalDelSize());
    assertEquals(60, stats.getPublicDelSize());
    assertEquals(40, stats.getPrivateDelSize());
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LocalCacheCleanerStats(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats) Test(org.junit.Test)

Example 2 with LocalCacheCleanerStats

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.

the class ResourceLocalizationService method handleCacheCleanup.

@VisibleForTesting
LocalCacheCleanerStats handleCacheCleanup() {
    LocalCacheCleaner cleaner = new LocalCacheCleaner(delService, cacheTargetSize);
    cleaner.addResources(publicRsrc);
    for (LocalResourcesTracker t : privateRsrc.values()) {
        cleaner.addResources(t);
    }
    LocalCacheCleaner.LocalCacheCleanerStats stats = cleaner.cleanCache();
    if (LOG.isDebugEnabled()) {
        LOG.debug(stats.toStringDetailed());
    } else if (LOG.isInfoEnabled()) {
        LOG.info(stats.toString());
    }
    return stats;
}
Also used : LocalCacheCleanerStats(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 3 with LocalCacheCleanerStats

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.

the class TestLocalCacheCleanup method testLRUAcrossTrackers.

@Test
public void testLRUAcrossTrackers() {
    ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    LocalResourceRequest pubSurviver1 = addResource(publicRsrc, "/pub-resource1.txt", 8, 20, 0);
    LocalResourceRequest pubSurviver2 = addResource(publicRsrc, "/pub-resource2.txt", 7, 20, 0);
    addResource(publicRsrc, "/pub-resource3.txt", 1, 20, 0);
    ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
    ConcurrentMap<LocalResourceRequest, LocalizedResource> user1rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    LocalResourceRequest usr1Surviver1 = addResource(user1rsrcs, "/private-u1-resource1.txt", 6, 20, 0);
    addResource(user1rsrcs, "/private-u1-resource2.txt", 2, 20, 0);
    LocalResourcesTracker user1Tracker = new StubbedLocalResourcesTrackerImpl("user1", user1rsrcs);
    privateRsrc.put("user1", user1Tracker);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> user2rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    LocalResourceRequest usr2Surviver1 = addResource(user2rsrcs, "/private-u2-resource1.txt", 5, 20, 0);
    addResource(user2rsrcs, "/private-u2-resource2.txt", 3, 20, 0);
    addResource(user2rsrcs, "/private-u2-resource3.txt", 4, 20, 0);
    LocalResourcesTracker user2Tracker = new StubbedLocalResourcesTrackerImpl("user2", user2rsrcs);
    privateRsrc.put("user2", user2Tracker);
    ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 80);
    LocalCacheCleanerStats stats = rls.handleCacheCleanup();
    Map<LocalResourceRequest, LocalizedResource> pubLocalRsrc = ((StubbedLocalResourcesTrackerImpl) rls.publicRsrc).getLocalRsrc();
    assertEquals(2, pubLocalRsrc.size());
    assertTrue(pubLocalRsrc.containsKey(pubSurviver1));
    assertTrue(pubLocalRsrc.containsKey(pubSurviver2));
    Map<LocalResourceRequest, LocalizedResource> usr1LocalRsrc = ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user1")).getLocalRsrc();
    assertEquals(1, usr1LocalRsrc.size());
    assertTrue(usr1LocalRsrc.containsKey(usr1Surviver1));
    Map<LocalResourceRequest, LocalizedResource> usr2LocalRsrc = ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user2")).getLocalRsrc();
    assertEquals(1, usr2LocalRsrc.size());
    assertTrue(usr2LocalRsrc.containsKey(usr2Surviver1));
    assertEquals(80, stats.getTotalDelSize());
    assertEquals(20, stats.getPublicDelSize());
    assertEquals(60, stats.getPrivateDelSize());
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LocalCacheCleanerStats(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats) Test(org.junit.Test)

Example 4 with LocalCacheCleanerStats

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.

the class TestLocalCacheCleanup method testPositiveRefCount.

@Test
public void testPositiveRefCount() {
    ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    // Oldest resource with a positive ref count the other with a ref count
    // equal to 0.
    LocalResourceRequest survivor = addResource(publicRsrc, "/pub-resource1.txt", 1, 20, 1);
    addResource(publicRsrc, "/pub-resource2.txt", 5, 20, 0);
    ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
    ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 0);
    LocalCacheCleanerStats stats = rls.handleCacheCleanup();
    StubbedLocalResourcesTrackerImpl resources = (StubbedLocalResourcesTrackerImpl) rls.publicRsrc;
    assertEquals(1, resources.getLocalRsrc().size());
    assertTrue(resources.getLocalRsrc().containsKey(survivor));
    assertEquals(20, stats.getTotalDelSize());
    assertEquals(20, stats.getPublicDelSize());
    assertEquals(0, stats.getPrivateDelSize());
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LocalCacheCleanerStats(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats) Test(org.junit.Test)

Aggregations

LocalCacheCleanerStats (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats)4 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 Test (org.junit.Test)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1