Examples with DataOutputBuffer org.apache.hadoop.io.DataOutputBuffer used on opensource projects

Example 91 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class TestContainerManagerRecovery method testContainerResizeRecovery.

@Test
public void testContainerResizeRecovery() throws Exception {
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
    NMStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    Context context = createContext(conf, stateStore);
    ContainerManagerImpl cm = createContainerManager(context, delSrvc);
    cm.init(conf);
    cm.start();
    // add an application by starting a container
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cid = ContainerId.newContainerId(attemptId, 1);
    Map<String, String> containerEnv = Collections.emptyMap();
    Map<String, ByteBuffer> serviceData = Collections.emptyMap();
    Credentials containerCreds = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    containerCreds.writeTokenStorageToStream(dob);
    ByteBuffer containerTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    Map<ApplicationAccessType, String> acls = Collections.emptyMap();
    File tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    if (Shell.WINDOWS) {
        fileWriter.println("@ping -n 100 127.0.0.1 >nul");
    } else {
        fileWriter.write("\numask 0");
        fileWriter.write("\nexec sleep 100");
    }
    fileWriter.close();
    FileContext localFS = FileContext.getLocalFSFileContext();
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(destinationFile, rsrc_alpha);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(localResources, containerEnv, commands, serviceData, containerTokens, acls);
    StartContainersResponse startResponse = startContainer(context, cm, cid, clc, null);
    assertTrue(startResponse.getFailedRequests().isEmpty());
    assertEquals(1, context.getApplications().size());
    Application app = context.getApplications().get(appId);
    assertNotNull(app);
    // make sure the container reaches RUNNING state
    waitForNMContainerState(cm, cid, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING);
    Resource targetResource = Resource.newInstance(2048, 2);
    IncreaseContainersResourceResponse increaseResponse = increaseContainersResource(context, cm, cid, targetResource);
    assertTrue(increaseResponse.getFailedRequests().isEmpty());
    // check status
    ContainerStatus containerStatus = getContainerStatus(context, cm, cid);
    assertEquals(targetResource, containerStatus.getCapability());
    // restart and verify container is running and recovered
    // to the correct size
    cm.stop();
    context = createContext(conf, stateStore);
    cm = createContainerManager(context);
    cm.init(conf);
    cm.start();
    assertEquals(1, context.getApplications().size());
    app = context.getApplications().get(appId);
    assertNotNull(app);
    containerStatus = getContainerStatus(context, cm, cid);
    assertEquals(targetResource, containerStatus.getCapability());
}

Example 92 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class TestPBRecordImpl method testLocalResourceStatusSerDe.

@Test(timeout = 10000)
public void testLocalResourceStatusSerDe() throws Exception {
    LocalResourceStatus rsrcS = createLocalResourceStatus();
    assertTrue(rsrcS instanceof LocalResourceStatusPBImpl);
    LocalResourceStatusPBImpl rsrcPb = (LocalResourceStatusPBImpl) rsrcS;
    DataOutputBuffer out = new DataOutputBuffer();
    rsrcPb.getProto().writeDelimitedTo(out);
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), 0, out.getLength());
    LocalResourceStatusProto rsrcPbD = LocalResourceStatusProto.parseDelimitedFrom(in);
    assertNotNull(rsrcPbD);
    LocalResourceStatus rsrcD = new LocalResourceStatusPBImpl(rsrcPbD);
    assertEquals(rsrcS, rsrcD);
    assertEquals(createResource(), rsrcS.getResource());
    assertEquals(createResource(), rsrcD.getResource());
}

Example 93 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerHeartbeatWhenAppCleaningUp.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception {
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, lfs.makeQualified(new Path(basedir, 0 + "")).toString());
    // Start dispatcher.
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    dispatcher.register(ApplicationEventType.class, mock(EventHandler.class));
    dispatcher.register(ContainerEventType.class, mock(EventHandler.class));
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    // Start resource localization service.
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        // Init application resources.
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(1234567890L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        when(app.toString()).thenReturn(appId.toString());
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 46, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getAppMockedResource(r);
        final LocalResource resource2 = getAppMockedResource(r);
        // Send localization requests for container.
        // 2 resources generated with APPLICATION visibility.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> appResourceList = Arrays.asList(req1, req2);
        rsrcs.put(LocalResourceVisibility.APPLICATION, appResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        dispatcher.await();
        // Wait for localization to begin.
        exec.waitForLocalizers(1);
        final String containerIdStr = c.getContainerId().toString();
        LocalizerRunner locRunnerForContainer = spyService.getLocalizerRunner(containerIdStr);
        // Heartbeats from container localizer
        LocalResourceStatus rsrcSuccess = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrcSuccess.getResource()).thenReturn(resource1);
        when(rsrcSuccess.getLocalSize()).thenReturn(4344L);
        when(rsrcSuccess.getLocalPath()).thenReturn(getPath("/some/path"));
        when(rsrcSuccess.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to be LIVE in Heartbeat.", LocalizerAction.LIVE, response.getLocalizerAction());
        // Cleanup application.
        spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app));
        dispatcher.await();
        try {
            // Directly send heartbeat to introduce race as app is being cleaned up.
            locRunnerForContainer.processHeartbeat(Collections.singletonList(rsrcSuccess));
        } catch (Exception e) {
            fail("Exception should not have been thrown on processing heartbeat");
        }
        // Send another heartbeat.
        response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to DIE in Heartbeat.", LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
    } finally {
        spyService.stop();
        dispatcher.stop();
    }
}

Example 94 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizationHeartbeat.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizationHeartbeat() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[1];
    // Making sure that we have only one local disk so that it will only be
    // selected for consecutive resource localization calls.  This is required
    // to test LocalCacheDirectoryManager.
    localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
    sDirs[0] = localDirs.get(0).toString();
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    // Adding configuration to make sure there is only one file per
    // directory
    conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
    final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE);
    final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE);
    final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR);
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0]));
    final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir);
    doAnswer(new Answer<FileStatus>() {

        @Override
        public FileStatus answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length > 0) {
                if (args[0].equals(userDir) || args[0].equals(fileDir)) {
                    return fs;
                }
            }
            return nmFs;
        }
    }).when(spylfs).getFileStatus(isA(Path.class));
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() {

            @Override
            public boolean matches(Object o) {
                ApplicationEvent evt = (ApplicationEvent) o;
                return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID();
            }
        };
        dispatcher.await();
        verify(applicationBus).handle(argThat(matchesAppInit));
        // init container rsrc, localizer
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getPrivateMockedResource(r);
        LocalResource resource2 = null;
        do {
            resource2 = getPrivateMockedResource(r);
        } while (resource2 == null || resource2.equals(resource1));
        LocalResource resource3 = null;
        do {
            resource3 = getPrivateMockedResource(r);
        } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2));
        // above call to make sure we don't get identical resources.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        privateResourceList.add(req2);
        privateResourceList.add(req3);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        // Sigh. Thread init of private localizer not accessible
        Thread.sleep(1000);
        dispatcher.await();
        String appStr = appId.toString();
        String ctnrStr = c.getContainerId().toString();
        ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor.forClass(LocalizerStartContext.class);
        verify(exec).startLocalizer(contextCaptor.capture());
        LocalizerStartContext context = contextCaptor.getValue();
        Path localizationTokenPath = context.getNmPrivateContainerTokens();
        assertEquals("user0", context.getUser());
        assertEquals(appStr, context.getAppId());
        assertEquals(ctnrStr, context.getLocId());
        // heartbeat from localizer
        LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc3success = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(ctnrStr);
        when(rsrc1success.getResource()).thenReturn(resource1);
        when(rsrc2pending.getResource()).thenReturn(resource2);
        when(rsrc2success.getResource()).thenReturn(resource2);
        when(rsrc3success.getResource()).thenReturn(resource3);
        when(rsrc1success.getLocalSize()).thenReturn(4344L);
        when(rsrc2success.getLocalSize()).thenReturn(2342L);
        when(rsrc3success.getLocalSize()).thenReturn(5345L);
        URL locPath = getPath("/cache/private/blah");
        when(rsrc1success.getLocalPath()).thenReturn(locPath);
        when(rsrc2success.getLocalPath()).thenReturn(locPath);
        when(rsrc3success.getLocalPath()).thenReturn(locPath);
        when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
        when(rsrc2success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc3success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        // Four heartbeats with sending:
        // 1 - empty
        // 2 - resource1 FETCH_SUCCESS
        // 3 - resource2 FETCH_PENDING
        // 4 - resource2 FETCH_SUCCESS, resource3 FETCH_SUCCESS
        List<LocalResourceStatus> rsrcs4 = new ArrayList<LocalResourceStatus>();
        rsrcs4.add(rsrc2success);
        rsrcs4.add(rsrc3success);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()).thenReturn(Collections.singletonList(rsrc1success)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(rsrcs4).thenReturn(Collections.<LocalResourceStatus>emptyList());
        String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + "user0" + Path.SEPARATOR + ContainerLocalizer.FILECACHE;
        // First heartbeat
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req1, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        URL localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        // Appending to local path unique number(10) generated as a part of
        // LocalResourcesTracker
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "10"));
        // Second heartbeat
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req2, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        // Resource's destination path should be now inside sub directory 0 as
        // LocalCacheDirectoryManager will be used and we have restricted number
        // of files per directory to 1.
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11"));
        // Third heartbeat
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req3, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "1" + Path.SEPARATOR + "12"));
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        spyService.handle(new ContainerLocalizationEvent(LocalizationEventType.CONTAINER_RESOURCES_LOCALIZED, c));
        // get shutdown after receive CONTAINER_RESOURCES_LOCALIZED event
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
        dispatcher.await();
        // verify container notification
        ArgumentMatcher<ContainerEvent> matchesContainerLoc = new ArgumentMatcher<ContainerEvent>() {

            @Override
            public boolean matches(Object o) {
                ContainerEvent evt = (ContainerEvent) o;
                return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c.getContainerId() == evt.getContainerID();
            }
        };
        // total 3 resource localzation calls. one for each resource.
        verify(containerBus, times(3)).handle(argThat(matchesContainerLoc));
        // Verify deletion of localization token.
        verify(delService).delete((String) isNull(), eq(localizationTokenPath));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 95 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class TestEditsDoubleBuffer method testDoubleBuffer.

@Test
public void testDoubleBuffer() throws IOException {
    EditsDoubleBuffer buf = new EditsDoubleBuffer(1024);
    assertTrue(buf.isFlushed());
    byte[] data = new byte[100];
    buf.writeRaw(data, 0, data.length);
    assertEquals("Should count new data correctly", data.length, buf.countBufferedBytes());
    assertTrue("Writing to current buffer should not affect flush state", buf.isFlushed());
    // Swap the buffers
    buf.setReadyToFlush();
    assertEquals("Swapping buffers should still count buffered bytes", data.length, buf.countBufferedBytes());
    assertFalse(buf.isFlushed());
    // Flush to a stream
    DataOutputBuffer outBuf = new DataOutputBuffer();
    buf.flushTo(outBuf);
    assertEquals(data.length, outBuf.getLength());
    assertTrue(buf.isFlushed());
    assertEquals(0, buf.countBufferedBytes());
    // Write some more
    buf.writeRaw(data, 0, data.length);
    assertEquals("Should count new data correctly", data.length, buf.countBufferedBytes());
    buf.setReadyToFlush();
    buf.flushTo(outBuf);
    assertEquals(data.length * 2, outBuf.getLength());
    assertEquals(0, buf.countBufferedBytes());
    outBuf.close();
}