Examples with UseSharedCacheResourceRequest org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest used on opensource projects

Example 1 with UseSharedCacheResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest in project hadoop by apache.

the class TestClientSCMProtocolService method testUse_ExistingEntry_NoAppIds.

@Test
public void testUse_ExistingEntry_NoAppIds() throws Exception {
    // Pre-populate the SCM with one cache entry
    store.addResource("key1", "foo.jar");
    long hits = ClientSCMMetrics.getInstance().getCacheHits();
    UseSharedCacheResourceRequest request = recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class);
    request.setResourceKey("key1");
    request.setAppId(createAppId(2, 2L));
    // Expecting default depth of 3 and under the shared cache root dir
    String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar";
    assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
    assertEquals(1, store.getResourceReferences("key1").size());
    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics.getInstance().getCacheHits() - hits);
}

Example 2 with UseSharedCacheResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest in project hadoop by apache.

the class TestClientSCMProtocolService method testUse_ExistingEntry_OneId.

@Test
public void testUse_ExistingEntry_OneId() throws Exception {
    // Pre-populate the SCM with one cache entry
    store.addResource("key1", "foo.jar");
    store.addResourceReference("key1", new SharedCacheResourceReference(createAppId(1, 1L), "user"));
    assertEquals(1, store.getResourceReferences("key1").size());
    long hits = ClientSCMMetrics.getInstance().getCacheHits();
    // Add a new distinct appId
    UseSharedCacheResourceRequest request = recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class);
    request.setResourceKey("key1");
    request.setAppId(createAppId(2, 2L));
    // Expecting default depth of 3 under the shared cache root dir
    String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar";
    assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
    assertEquals(2, store.getResourceReferences("key1").size());
    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics.getInstance().getCacheHits() - hits);
}

Example 3 with UseSharedCacheResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest in project hadoop by apache.

the class TestClientSCMProtocolService method testUse_ExistingEntry_DupId.

@Test
public void testUse_ExistingEntry_DupId() throws Exception {
    // Pre-populate the SCM with one cache entry
    store.addResource("key1", "foo.jar");
    UserGroupInformation testUGI = UserGroupInformation.getCurrentUser();
    store.addResourceReference("key1", new SharedCacheResourceReference(createAppId(1, 1L), testUGI.getShortUserName()));
    assertEquals(1, store.getResourceReferences("key1").size());
    long hits = ClientSCMMetrics.getInstance().getCacheHits();
    // Add a new duplicate appId
    UseSharedCacheResourceRequest request = recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class);
    request.setResourceKey("key1");
    request.setAppId(createAppId(1, 1L));
    // Expecting default depth of 3 under the shared cache root dir
    String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar";
    assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
    assertEquals(1, store.getResourceReferences("key1").size());
    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics.getInstance().getCacheHits() - hits);
}

Example 4 with UseSharedCacheResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest in project hadoop by apache.

the class TestClientSCMProtocolService method testUse_MissingEntry.

@Test
public void testUse_MissingEntry() throws Exception {
    long misses = ClientSCMMetrics.getInstance().getCacheMisses();
    UseSharedCacheResourceRequest request = recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class);
    request.setResourceKey("key1");
    request.setAppId(createAppId(1, 1L));
    assertNull(clientSCMProxy.use(request).getPath());
    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics.getInstance().getCacheMisses() - misses);
}

Example 5 with UseSharedCacheResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest in project hadoop by apache.

the class SharedCacheClientImpl method use.

@Override
public Path use(ApplicationId applicationId, String resourceKey, String resourceName) throws YarnException {
    Path resourcePath = null;
    UseSharedCacheResourceRequest request = Records.newRecord(UseSharedCacheResourceRequest.class);
    request.setAppId(applicationId);
    request.setResourceKey(resourceKey);
    try {
        UseSharedCacheResourceResponse response = this.scmClient.use(request);
        if (response != null && response.getPath() != null) {
            resourcePath = new Path(response.getPath());
        }
    } catch (Exception e) {
        // We don't handle different exceptions separately at this point.
        throw new YarnException(e);
    }
    if (resourcePath != null) {
        if (resourcePath.getName().equals(resourceName)) {
            // so we skip generating the fragment to save space in the MRconfig.
            return resourcePath;
        } else {
            // We are using the shared cache, and a preferred name has been
            // specified that is different than the name of the resource in the
            // shared cache. We need to set the fragment portion of the URI to
            // preserve the desired name.
            URI pathURI = resourcePath.toUri();
            try {
                // We assume that there is no existing fragment in the URI since the
                // shared cache manager does not use fragments.
                pathURI = new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(), resourceName);
                resourcePath = new Path(pathURI);
            } catch (URISyntaxException e) {
                throw new YarnException("Could not create a new URI due to syntax errors: " + pathURI.toString(), e);
            }
        }
    }
    return resourcePath;
}