Examples with AbstractFileSystem org.apache.hadoop.fs.AbstractFileSystem used on opensource projects

Example 1 with AbstractFileSystem

use of org.apache.hadoop.fs.AbstractFileSystem in project hadoop by apache.

the class TestNonAggregatingLogHandler method testLogDeletion.

@Test
public void testLogDeletion() throws IOException {
    File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
    String localLogDirsString = localLogDirs[0].getAbsolutePath() + "," + localLogDirs[1].getAbsolutePath();
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);
    dirsHandler.init(conf);
    NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler, new NMNullStateStoreService());
    NonAggregatingLogHandler logHandler = spy(rawLogHandler);
    AbstractFileSystem spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    FileContext lfs = FileContext.getFileContext(spylfs, conf);
    doReturn(lfs).when(logHandler).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(localLogDirs[0].getAbsolutePath()));
    doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));
    logHandler.init(conf);
    logHandler.start();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
    logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    Path[] localAppLogDirs = new Path[2];
    localAppLogDirs[0] = new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
    localAppLogDirs[1] = new Path(localLogDirs[1].getAbsolutePath(), appId.toString());
    testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
    logHandler.close();
    for (int i = 0; i < localLogDirs.length; i++) {
        FileUtils.deleteDirectory(localLogDirs[i]);
    }
}

Example 2 with AbstractFileSystem

use of org.apache.hadoop.fs.AbstractFileSystem in project hadoop by apache.

the class TestWasbUriAndConfiguration method testWasbAsDefaultFileSystemHasNoPort.

@Test
public void testWasbAsDefaultFileSystemHasNoPort() throws Exception {
    try {
        testAccount = AzureBlobStorageTestAccount.createMock();
        Configuration conf = testAccount.getFileSystem().getConf();
        String authority = testAccount.getFileSystem().getUri().getAuthority();
        URI defaultUri = new URI("wasb", authority, null, null, null);
        conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
        conf.addResource("azure-test.xml");
        FileSystem fs = FileSystem.get(conf);
        assertTrue(fs instanceof NativeAzureFileSystem);
        assertEquals(-1, fs.getUri().getPort());
        AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem();
        assertTrue(afs instanceof Wasb);
        assertEquals(-1, afs.getUri().getPort());
    } finally {
        testAccount.cleanup();
        FileSystem.closeAll();
    }
}

Example 3 with AbstractFileSystem

use of org.apache.hadoop.fs.AbstractFileSystem in project hadoop by apache.

the class TestWasbUriAndConfiguration method testAbstractFileSystemImplementationForWasbsScheme.

/**
   * Tests the cases when the scheme specified is 'wasbs'.
   */
@Test
public void testAbstractFileSystemImplementationForWasbsScheme() throws Exception {
    try {
        testAccount = AzureBlobStorageTestAccount.createMock();
        Configuration conf = testAccount.getFileSystem().getConf();
        String authority = testAccount.getFileSystem().getUri().getAuthority();
        URI defaultUri = new URI("wasbs", authority, null, null, null);
        conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
        conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs");
        conf.addResource("azure-test.xml");
        FileSystem fs = FileSystem.get(conf);
        assertTrue(fs instanceof NativeAzureFileSystem);
        assertEquals("wasbs", fs.getScheme());
        AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem();
        assertTrue(afs instanceof Wasbs);
        assertEquals(-1, afs.getUri().getPort());
        assertEquals("wasbs", afs.getUri().getScheme());
    } finally {
        testAccount.cleanup();
        FileSystem.closeAll();
    }
}

Example 4 with AbstractFileSystem

use of org.apache.hadoop.fs.AbstractFileSystem in project hadoop by apache.

the class TestContainerLocalizer method testMain.

@Test
public void testMain() throws Exception {
    ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
    ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
    Random random = wrapper.random;
    List<Path> localDirs = wrapper.localDirs;
    Path tokenPath = wrapper.tokenPath;
    LocalizationProtocol nmProxy = wrapper.nmProxy;
    AbstractFileSystem spylfs = wrapper.spylfs;
    mockOutDownloads(localizer);
    // verify created cache
    List<Path> privCacheList = new ArrayList<Path>();
    List<Path> appCacheList = new ArrayList<Path>();
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        privCacheList.add(privcache);
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        appCacheList.add(appcache);
    }
    // mock heartbeat responses from NM
    ResourceLocalizationSpec rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcC = getMockRsrc(random, LocalResourceVisibility.APPLICATION, appCacheList.get(0));
    ResourceLocalizationSpec rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    when(nmProxy.heartbeat(isA(LocalizerStatus.class))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcA))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcB))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcC))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcD))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.<ResourceLocalizationSpec>emptyList())).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, null));
    LocalResource tRsrcA = rsrcA.getResource();
    LocalResource tRsrcB = rsrcB.getResource();
    LocalResource tRsrcC = rsrcC.getResource();
    LocalResource tRsrcD = rsrcD.getResource();
    doReturn(new FakeDownload(rsrcA.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcA), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcB.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcB), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcC.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcC), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcD.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcD), isA(UserGroupInformation.class));
    // run localization
    localizer.runLocalization(nmAddr);
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        // $x/usercache/$user/filecache
        verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        // $x/usercache/$user/appcache/$appId/filecache
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
    }
    // verify tokens read at expected location
    verify(spylfs).open(tokenPath);
    // verify downloaded resources reported to NM
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD.getResource())));
    // verify all HB use localizerID provided
    verify(nmProxy, never()).heartbeat(argThat(new ArgumentMatcher<LocalizerStatus>() {

        @Override
        public boolean matches(Object o) {
            LocalizerStatus status = (LocalizerStatus) o;
            return !containerId.equals(status.getLocalizerId());
        }
    }));
}

Example 5 with AbstractFileSystem

use of org.apache.hadoop.fs.AbstractFileSystem in project hadoop by apache.

the class TestContainerLocalizer method testLocalizerTokenIsGettingRemoved.

@Test
@SuppressWarnings("unchecked")
public void testLocalizerTokenIsGettingRemoved() throws Exception {
    ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
    ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
    Path tokenPath = wrapper.tokenPath;
    AbstractFileSystem spylfs = wrapper.spylfs;
    mockOutDownloads(localizer);
    doNothing().when(localizer).localizeFiles(any(LocalizationProtocol.class), any(CompletionService.class), any(UserGroupInformation.class));
    localizer.runLocalization(nmAddr);
    verify(spylfs, times(1)).delete(tokenPath, false);
}