Examples with MockFileSystem org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem used on opensource projects

Example 11 with MockFileSystem

use of org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem in project hadoop by apache.

the class TestFileSystemTokens method testFsWithToken.

@Test
public void testFsWithToken() throws Exception {
    Text service = new Text("singleTokenFs");
    MockFileSystem fs = createFileSystemForServiceName(service);
    Credentials credentials = new Credentials();
    fs.addDelegationTokens(renewer, credentials);
    verifyTokenFetch(fs, true);
    assertEquals(1, credentials.numberOfTokens());
    assertNotNull(credentials.getToken(service));
}

Example 12 with MockFileSystem

use of org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem in project hadoop by apache.

the class TestFileSystemTokens method createFileSystemForServiceName.

public static MockFileSystem createFileSystemForServiceName(final Text service, final FileSystem... children) throws IOException {
    final MockFileSystem fs = new MockFileSystem();
    final MockFileSystem mockFs = fs.getRawFileSystem();
    if (service != null) {
        when(mockFs.getCanonicalServiceName()).thenReturn(service.toString());
        when(mockFs.getDelegationToken(any(String.class))).thenAnswer(new Answer<Token<?>>() {

            @Override
            public Token<?> answer(InvocationOnMock invocation) throws Throwable {
                Token<?> token = new Token<TokenIdentifier>();
                token.setService(service);
                return token;
            }
        });
    }
    when(mockFs.getChildFileSystems()).thenReturn(children);
    return fs;
}

Example 13 with MockFileSystem

use of org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem in project hadoop by apache.

the class TestTokenCache method testBinaryCredentials.

private void testBinaryCredentials(boolean hasScheme) throws Exception {
    Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "test/build/data"));
    // ick, but need fq path minus file:/
    String binaryTokenFile = hasScheme ? FileSystem.getLocal(conf).makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toString() : FileSystem.getLocal(conf).makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
    MockFileSystem fs1 = createFileSystemForServiceName("service1");
    MockFileSystem fs2 = createFileSystemForServiceName("service2");
    MockFileSystem fs3 = createFileSystemForServiceName("service3");
    // get the tokens for fs1 & fs2 and write out to binary creds file
    Credentials creds = new Credentials();
    Token<?> token1 = fs1.getDelegationToken(renewer);
    Token<?> token2 = fs2.getDelegationToken(renewer);
    creds.addToken(token1.getService(), token1);
    creds.addToken(token2.getService(), token2);
    // wait to set, else the obtain tokens call above will fail with FNF
    conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
    creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
    // re-init creds and add a newer token for fs1
    creds = new Credentials();
    Token<?> newerToken1 = fs1.getDelegationToken(renewer);
    assertNotSame(newerToken1, token1);
    creds.addToken(newerToken1.getService(), newerToken1);
    checkToken(creds, newerToken1);
    // get token for fs1, see that fs2's token was loaded 
    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
    checkToken(creds, newerToken1, token2);
    // get token for fs2, nothing should change since already present
    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
    checkToken(creds, newerToken1, token2);
    // get token for fs3, should only add token for fs3
    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
    Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName()));
    assertTrue(token3 != null);
    checkToken(creds, newerToken1, token2, token3);
    // be paranoid, check one last time that nothing changes
    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
    TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
    TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
    checkToken(creds, newerToken1, token2, token3);
}

Example 14 with MockFileSystem

use of org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem in project hadoop by apache.

the class TestTokenCache method testGetTokensForNamenodes.

@SuppressWarnings("deprecation")
@Test
public void testGetTokensForNamenodes() throws IOException, URISyntaxException {
    Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "test/build/data"));
    // ick, but need fq path minus file:/
    String binaryTokenFile = FileSystem.getLocal(conf).makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
    MockFileSystem fs1 = createFileSystemForServiceName("service1");
    Credentials creds = new Credentials();
    Token<?> token1 = fs1.getDelegationToken(renewer);
    creds.addToken(token1.getService(), token1);
    // wait to set, else the obtain tokens call above will fail with FNF
    conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
    creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
    TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
    String fs_addr = fs1.getCanonicalServiceName();
    Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
    assertNotNull("Token for nn is null", nnt);
}

Example 15 with MockFileSystem

use of org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem in project hadoop by apache.

the class TestTokenCache method createFileSystemForServiceName.

private MockFileSystem createFileSystemForServiceName(final String service) throws IOException {
    MockFileSystem mockFs = new MockFileSystem();
    when(mockFs.getCanonicalServiceName()).thenReturn(service);
    when(mockFs.getDelegationToken(any(String.class))).thenAnswer(new Answer<Token<?>>() {

        int unique = 0;

        @Override
        public Token<?> answer(InvocationOnMock invocation) throws Throwable {
            Token<?> token = new Token<TokenIdentifier>();
            token.setService(new Text(service));
            // use unique value so when we restore from token storage, we can
            // tell if it's really the same token
            token.setKind(new Text("token" + unique++));
            return token;
        }
    });
    return mockFs;
}