use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.
the class TestLocalCacheCleanup method testBasicCleanup.
@Test
public void testBasicCleanup() {
ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
addResource(publicRsrc, "/pub-resource1.txt", 5, 20, 0);
addResource(publicRsrc, "/pub-resource2.txt", 3, 20, 0);
addResource(publicRsrc, "/pub-resource3.txt", 15, 20, 0);
ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
ConcurrentMap<LocalResourceRequest, LocalizedResource> user1rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
addResource(user1rsrcs, "/private-u1-resource4.txt", 1, 20, 0);
LocalResourcesTracker user1Tracker = new StubbedLocalResourcesTrackerImpl("user1", user1rsrcs);
privateRsrc.put("user1", user1Tracker);
ConcurrentMap<LocalResourceRequest, LocalizedResource> user2rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
addResource(user2rsrcs, "/private-u2-resource5.txt", 2, 20, 0);
LocalResourcesTracker user2Tracker = new StubbedLocalResourcesTrackerImpl("user2", user2rsrcs);
privateRsrc.put("user2", user2Tracker);
ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 0);
LocalCacheCleanerStats stats = rls.handleCacheCleanup();
assertEquals(0, ((StubbedLocalResourcesTrackerImpl) rls.publicRsrc).getLocalRsrc().size());
assertEquals(0, ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user1")).getLocalRsrc().size());
assertEquals(0, ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user2")).getLocalRsrc().size());
assertEquals(100, stats.getTotalDelSize());
assertEquals(60, stats.getPublicDelSize());
assertEquals(40, stats.getPrivateDelSize());
}
use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.
the class ResourceLocalizationService method handleCacheCleanup.
@VisibleForTesting
LocalCacheCleanerStats handleCacheCleanup() {
LocalCacheCleaner cleaner = new LocalCacheCleaner(delService, cacheTargetSize);
cleaner.addResources(publicRsrc);
for (LocalResourcesTracker t : privateRsrc.values()) {
cleaner.addResources(t);
}
LocalCacheCleaner.LocalCacheCleanerStats stats = cleaner.cleanCache();
if (LOG.isDebugEnabled()) {
LOG.debug(stats.toStringDetailed());
} else if (LOG.isInfoEnabled()) {
LOG.info(stats.toString());
}
return stats;
}
use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.
the class TestLocalCacheCleanup method testLRUAcrossTrackers.
@Test
public void testLRUAcrossTrackers() {
ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
LocalResourceRequest pubSurviver1 = addResource(publicRsrc, "/pub-resource1.txt", 8, 20, 0);
LocalResourceRequest pubSurviver2 = addResource(publicRsrc, "/pub-resource2.txt", 7, 20, 0);
addResource(publicRsrc, "/pub-resource3.txt", 1, 20, 0);
ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
ConcurrentMap<LocalResourceRequest, LocalizedResource> user1rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
LocalResourceRequest usr1Surviver1 = addResource(user1rsrcs, "/private-u1-resource1.txt", 6, 20, 0);
addResource(user1rsrcs, "/private-u1-resource2.txt", 2, 20, 0);
LocalResourcesTracker user1Tracker = new StubbedLocalResourcesTrackerImpl("user1", user1rsrcs);
privateRsrc.put("user1", user1Tracker);
ConcurrentMap<LocalResourceRequest, LocalizedResource> user2rsrcs = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
LocalResourceRequest usr2Surviver1 = addResource(user2rsrcs, "/private-u2-resource1.txt", 5, 20, 0);
addResource(user2rsrcs, "/private-u2-resource2.txt", 3, 20, 0);
addResource(user2rsrcs, "/private-u2-resource3.txt", 4, 20, 0);
LocalResourcesTracker user2Tracker = new StubbedLocalResourcesTrackerImpl("user2", user2rsrcs);
privateRsrc.put("user2", user2Tracker);
ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 80);
LocalCacheCleanerStats stats = rls.handleCacheCleanup();
Map<LocalResourceRequest, LocalizedResource> pubLocalRsrc = ((StubbedLocalResourcesTrackerImpl) rls.publicRsrc).getLocalRsrc();
assertEquals(2, pubLocalRsrc.size());
assertTrue(pubLocalRsrc.containsKey(pubSurviver1));
assertTrue(pubLocalRsrc.containsKey(pubSurviver2));
Map<LocalResourceRequest, LocalizedResource> usr1LocalRsrc = ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user1")).getLocalRsrc();
assertEquals(1, usr1LocalRsrc.size());
assertTrue(usr1LocalRsrc.containsKey(usr1Surviver1));
Map<LocalResourceRequest, LocalizedResource> usr2LocalRsrc = ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user2")).getLocalRsrc();
assertEquals(1, usr2LocalRsrc.size());
assertTrue(usr2LocalRsrc.containsKey(usr2Surviver1));
assertEquals(80, stats.getTotalDelSize());
assertEquals(20, stats.getPublicDelSize());
assertEquals(60, stats.getPrivateDelSize());
}
use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats in project hadoop by apache.
the class TestLocalCacheCleanup method testPositiveRefCount.
@Test
public void testPositiveRefCount() {
ConcurrentMap<LocalResourceRequest, LocalizedResource> publicRsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
// Oldest resource with a positive ref count the other with a ref count
// equal to 0.
LocalResourceRequest survivor = addResource(publicRsrc, "/pub-resource1.txt", 1, 20, 1);
addResource(publicRsrc, "/pub-resource2.txt", 5, 20, 0);
ConcurrentMap<String, LocalResourcesTracker> privateRsrc = new ConcurrentHashMap<String, LocalResourcesTracker>();
ResourceLocalizationService rls = createLocService(publicRsrc, privateRsrc, 0);
LocalCacheCleanerStats stats = rls.handleCacheCleanup();
StubbedLocalResourcesTrackerImpl resources = (StubbedLocalResourcesTrackerImpl) rls.publicRsrc;
assertEquals(1, resources.getLocalRsrc().size());
assertTrue(resources.getLocalRsrc().containsKey(survivor));
assertEquals(20, stats.getTotalDelSize());
assertEquals(20, stats.getPublicDelSize());
assertEquals(0, stats.getPrivateDelSize());
}
Aggregations