Examples with SCMUploaderProtocol org.apache.hadoop.yarn.server.api.SCMUploaderProtocol used on opensource projects

Example 1 with SCMUploaderProtocol

use of org.apache.hadoop.yarn.server.api.SCMUploaderProtocol in project hadoop by apache.

the class TestSharedCacheUploader method testVerifyAccessPublicResource.

/**
   * If resource is public, verifyAccess should succeed
   */
@Test
public void testVerifyAccessPublicResource() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
    LocalResource resource = mock(LocalResource.class);
    // give public visibility
    when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
    Path localPath = mock(Path.class);
    when(localPath.getName()).thenReturn("foo.jar");
    String user = "joe";
    SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
    FileSystem fs = mock(FileSystem.class);
    FileSystem localFs = FileSystem.getLocal(conf);
    SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs);
    assertTrue(spied.verifyAccess());
}

Example 2 with SCMUploaderProtocol

use of org.apache.hadoop.yarn.server.api.SCMUploaderProtocol in project hadoop by apache.

the class TestSharedCacheUploader method testSuccess.

/**
   * If verifyAccess, uploadFile, rename, and notification succeed, the upload
   * should succeed
   */
@Test
public void testSuccess() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
    LocalResource resource = mock(LocalResource.class);
    Path localPath = mock(Path.class);
    when(localPath.getName()).thenReturn("foo.jar");
    String user = "joe";
    SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
    SCMUploaderNotifyResponse response = mock(SCMUploaderNotifyResponse.class);
    when(response.getAccepted()).thenReturn(true);
    when(scmClient.notify(isA(SCMUploaderNotifyRequest.class))).thenReturn(response);
    FileSystem fs = mock(FileSystem.class);
    // return false when rename is called
    when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true);
    FileSystem localFs = FileSystem.getLocal(conf);
    SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs);
    // stub verifyAccess() to return true
    doReturn(true).when(spied).verifyAccess();
    // stub getActualPath()
    doReturn(localPath).when(spied).getActualPath();
    // stub computeChecksum()
    doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class));
    // stub uploadFile() to return true
    doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class));
    // stub notifySharedCacheManager to return true
    doReturn(true).when(spied).notifySharedCacheManager(isA(String.class), isA(String.class));
    assertTrue(spied.call());
}

Example 3 with SCMUploaderProtocol

use of org.apache.hadoop.yarn.server.api.SCMUploaderProtocol in project hadoop by apache.

the class SharedCacheUploadService method createSCMClient.

private SCMUploaderProtocol createSCMClient(Configuration conf) {
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress scmAddress = conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);
    return (SCMUploaderProtocol) rpc.getProxy(SCMUploaderProtocol.class, scmAddress, conf);
}

Example 4 with SCMUploaderProtocol

use of org.apache.hadoop.yarn.server.api.SCMUploaderProtocol in project hadoop by apache.

the class TestSharedCacheUploader method testRenameFail.

/**
   * If rename fails, the upload should fail
   */
@Test
public void testRenameFail() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
    LocalResource resource = mock(LocalResource.class);
    Path localPath = mock(Path.class);
    when(localPath.getName()).thenReturn("foo.jar");
    String user = "joe";
    SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
    SCMUploaderNotifyResponse response = mock(SCMUploaderNotifyResponse.class);
    when(response.getAccepted()).thenReturn(true);
    when(scmClient.notify(isA(SCMUploaderNotifyRequest.class))).thenReturn(response);
    FileSystem fs = mock(FileSystem.class);
    // return false when rename is called
    when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(false);
    FileSystem localFs = FileSystem.getLocal(conf);
    SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs);
    // stub verifyAccess() to return true
    doReturn(true).when(spied).verifyAccess();
    // stub getActualPath()
    doReturn(localPath).when(spied).getActualPath();
    // stub computeChecksum()
    doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class));
    // stub uploadFile() to return true
    doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class));
    assertFalse(spied.call());
}

Example 5 with SCMUploaderProtocol

use of org.apache.hadoop.yarn.server.api.SCMUploaderProtocol in project hadoop by apache.

the class TestSharedCacheUploader method testGetActualPath.

/**
   * If the localPath does not exists, getActualPath should get to one level
   * down
   */
@Test
public void testGetActualPath() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
    LocalResource resource = mock(LocalResource.class);
    // give public visibility
    when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
    Path localPath = new Path("foo.jar");
    String user = "joe";
    SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
    FileSystem fs = mock(FileSystem.class);
    FileSystem localFs = mock(FileSystem.class);
    // stub it to return a status that indicates a directory
    FileStatus status = mock(FileStatus.class);
    when(status.isDirectory()).thenReturn(true);
    when(localFs.getFileStatus(localPath)).thenReturn(status);
    SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs);
    Path actualPath = spied.getActualPath();
    assertEquals(actualPath.getName(), localPath.getName());
    assertEquals(actualPath.getParent().getName(), localPath.getName());
}