Examples with LocalizedResourceProto org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto used on opensource projects

Example 1 with LocalizedResourceProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testFinishResourceLocalization.

@Test
public void testFinishResourceLocalization() throws IOException {
    String user = "somebody";
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    // start and finish a local resource for an application
    Path appRsrcPath = new Path("hdfs://some/app/resource");
    LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(appRsrcPath), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, 123L, 456L);
    LocalResourceProto appRsrcProto = rsrcPb.getProto();
    Path appRsrcLocalPath = new Path("/some/local/dir/for/apprsrc");
    stateStore.startResourceLocalization(user, appId, appRsrcProto, appRsrcLocalPath);
    LocalizedResourceProto appLocalizedProto = LocalizedResourceProto.newBuilder().setResource(appRsrcProto).setLocalPath(appRsrcLocalPath.toString()).setSize(1234567L).build();
    stateStore.finishResourceLocalization(user, appId, appLocalizedProto);
    // restart and verify only app resource is completed
    restartStateStore();
    RecoveredLocalizationState state = stateStore.loadLocalizationState();
    LocalResourceTrackerState pubts = state.getPublicTrackerState();
    assertTrue(pubts.getLocalizedResources().isEmpty());
    assertTrue(pubts.getInProgressResources().isEmpty());
    Map<String, RecoveredUserResources> userResources = state.getUserResources();
    assertEquals(1, userResources.size());
    RecoveredUserResources rur = userResources.get(user);
    LocalResourceTrackerState privts = rur.getPrivateTrackerState();
    assertNotNull(privts);
    assertTrue(privts.getLocalizedResources().isEmpty());
    assertTrue(privts.getInProgressResources().isEmpty());
    assertEquals(1, rur.getAppTrackerStates().size());
    LocalResourceTrackerState appts = rur.getAppTrackerStates().get(appId);
    assertNotNull(appts);
    assertTrue(appts.getInProgressResources().isEmpty());
    assertEquals(1, appts.getLocalizedResources().size());
    assertEquals(appLocalizedProto, appts.getLocalizedResources().iterator().next());
    // start some public and private resources
    Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(pubRsrcPath1), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 789L, 135L);
    LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
    Path pubRsrcLocalPath1 = new Path("/some/local/dir/for/pubrsrc1");
    stateStore.startResourceLocalization(null, null, pubRsrcProto1, pubRsrcLocalPath1);
    Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(pubRsrcPath2), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 789L, 135L);
    LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
    Path pubRsrcLocalPath2 = new Path("/some/local/dir/for/pubrsrc2");
    stateStore.startResourceLocalization(null, null, pubRsrcProto2, pubRsrcLocalPath2);
    Path privRsrcPath = new Path("hdfs://some/private/resource");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(privRsrcPath), LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE, 789L, 680L, "*pattern*");
    LocalResourceProto privRsrcProto = rsrcPb.getProto();
    Path privRsrcLocalPath = new Path("/some/local/dir/for/privrsrc");
    stateStore.startResourceLocalization(user, null, privRsrcProto, privRsrcLocalPath);
    // finish some of the resources
    LocalizedResourceProto pubLocalizedProto1 = LocalizedResourceProto.newBuilder().setResource(pubRsrcProto1).setLocalPath(pubRsrcLocalPath1.toString()).setSize(pubRsrcProto1.getSize()).build();
    stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
    LocalizedResourceProto privLocalizedProto = LocalizedResourceProto.newBuilder().setResource(privRsrcProto).setLocalPath(privRsrcLocalPath.toString()).setSize(privRsrcProto.getSize()).build();
    stateStore.finishResourceLocalization(user, null, privLocalizedProto);
    // restart and verify state
    restartStateStore();
    state = stateStore.loadLocalizationState();
    pubts = state.getPublicTrackerState();
    assertEquals(1, pubts.getLocalizedResources().size());
    assertEquals(pubLocalizedProto1, pubts.getLocalizedResources().iterator().next());
    assertEquals(1, pubts.getInProgressResources().size());
    assertEquals(pubRsrcLocalPath2, pubts.getInProgressResources().get(pubRsrcProto2));
    userResources = state.getUserResources();
    assertEquals(1, userResources.size());
    rur = userResources.get(user);
    privts = rur.getPrivateTrackerState();
    assertNotNull(privts);
    assertEquals(1, privts.getLocalizedResources().size());
    assertEquals(privLocalizedProto, privts.getLocalizedResources().iterator().next());
    assertTrue(privts.getInProgressResources().isEmpty());
    assertEquals(1, rur.getAppTrackerStates().size());
    appts = rur.getAppTrackerStates().get(appId);
    assertNotNull(appts);
    assertTrue(appts.getInProgressResources().isEmpty());
    assertEquals(1, appts.getLocalizedResources().size());
    assertEquals(appLocalizedProto, appts.getLocalizedResources().iterator().next());
}

Example 2 with LocalizedResourceProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testRemoveLocalizedResource.

@Test
public void testRemoveLocalizedResource() throws IOException {
    String user = "somebody";
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    // go through the complete lifecycle for an application local resource
    Path appRsrcPath = new Path("hdfs://some/app/resource");
    LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(appRsrcPath), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, 123L, 456L);
    LocalResourceProto appRsrcProto = rsrcPb.getProto();
    Path appRsrcLocalPath = new Path("/some/local/dir/for/apprsrc");
    stateStore.startResourceLocalization(user, appId, appRsrcProto, appRsrcLocalPath);
    LocalizedResourceProto appLocalizedProto = LocalizedResourceProto.newBuilder().setResource(appRsrcProto).setLocalPath(appRsrcLocalPath.toString()).setSize(1234567L).build();
    stateStore.finishResourceLocalization(user, appId, appLocalizedProto);
    stateStore.removeLocalizedResource(user, appId, appRsrcLocalPath);
    restartStateStore();
    verifyEmptyState();
    // remove an app resource that didn't finish
    stateStore.startResourceLocalization(user, appId, appRsrcProto, appRsrcLocalPath);
    stateStore.removeLocalizedResource(user, appId, appRsrcLocalPath);
    restartStateStore();
    verifyEmptyState();
    // add public and private resources and remove some
    Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(pubRsrcPath1), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 789L, 135L);
    LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
    Path pubRsrcLocalPath1 = new Path("/some/local/dir/for/pubrsrc1");
    stateStore.startResourceLocalization(null, null, pubRsrcProto1, pubRsrcLocalPath1);
    LocalizedResourceProto pubLocalizedProto1 = LocalizedResourceProto.newBuilder().setResource(pubRsrcProto1).setLocalPath(pubRsrcLocalPath1.toString()).setSize(789L).build();
    stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
    Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(pubRsrcPath2), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 789L, 135L);
    LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
    Path pubRsrcLocalPath2 = new Path("/some/local/dir/for/pubrsrc2");
    stateStore.startResourceLocalization(null, null, pubRsrcProto2, pubRsrcLocalPath2);
    LocalizedResourceProto pubLocalizedProto2 = LocalizedResourceProto.newBuilder().setResource(pubRsrcProto2).setLocalPath(pubRsrcLocalPath2.toString()).setSize(7654321L).build();
    stateStore.finishResourceLocalization(null, null, pubLocalizedProto2);
    stateStore.removeLocalizedResource(null, null, pubRsrcLocalPath2);
    Path privRsrcPath = new Path("hdfs://some/private/resource");
    rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(URL.fromPath(privRsrcPath), LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE, 789L, 680L, "*pattern*");
    LocalResourceProto privRsrcProto = rsrcPb.getProto();
    Path privRsrcLocalPath = new Path("/some/local/dir/for/privrsrc");
    stateStore.startResourceLocalization(user, null, privRsrcProto, privRsrcLocalPath);
    stateStore.removeLocalizedResource(user, null, privRsrcLocalPath);
    // restart and verify state
    restartStateStore();
    RecoveredLocalizationState state = stateStore.loadLocalizationState();
    LocalResourceTrackerState pubts = state.getPublicTrackerState();
    assertTrue(pubts.getInProgressResources().isEmpty());
    assertEquals(1, pubts.getLocalizedResources().size());
    assertEquals(pubLocalizedProto1, pubts.getLocalizedResources().iterator().next());
    Map<String, RecoveredUserResources> userResources = state.getUserResources();
    assertTrue(userResources.isEmpty());
}

Example 3 with LocalizedResourceProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreSuccessfulLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    DeletionService mockDelService = mock(DeletionService.class);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
        tracker.handle(rle1);
        dispatcher.await();
        ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class);
        verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture());
        LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(localizedProto.getResource())));
        Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath());
        LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
        Assert.assertNotNull(localizedRsrc1);
        // simulate release and retention processing
        tracker.handle(new ResourceReleaseEvent(lr1, cId1));
        dispatcher.await();
        boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
        Assert.assertTrue(removeResult);
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 4 with LocalizedResourceProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto in project hadoop by apache.

the class ResourceLocalizationService method recoverTrackerResources.

private void recoverTrackerResources(LocalResourcesTracker tracker, LocalResourceTrackerState state) throws URISyntaxException {
    for (LocalizedResourceProto proto : state.getLocalizedResources()) {
        LocalResource rsrc = new LocalResourcePBImpl(proto.getResource());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Recovering localized resource " + req + " at " + proto.getLocalPath());
        }
        tracker.handle(new ResourceRecoveredEvent(req, new Path(proto.getLocalPath()), proto.getSize()));
    }
    for (Map.Entry<LocalResourceProto, Path> entry : state.getInProgressResources().entrySet()) {
        LocalResource rsrc = new LocalResourcePBImpl(entry.getKey());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        Path localPath = entry.getValue();
        tracker.handle(new ResourceRecoveredEvent(req, localPath, 0));
        // delete any in-progress localizations, containers will request again
        LOG.info("Deleting in-progress localization for " + req + " at " + localPath);
        tracker.remove(tracker.getLocalizedResource(req), delService);
    }
// TODO: remove untracked directories in local filesystem
}

Example 5 with LocalizedResourceProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto in project hadoop by apache.

the class NMLeveldbStateStoreService method loadCompletedResources.

private List<LocalizedResourceProto> loadCompletedResources(LeveldbIterator iter, String keyPrefix) throws IOException {
    List<LocalizedResourceProto> rsrcs = new ArrayList<LocalizedResourceProto>();
    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.peekNext();
        String key = asString(entry.getKey());
        if (!key.startsWith(keyPrefix)) {
            break;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Loading completed resource from " + key);
        }
        rsrcs.add(LocalizedResourceProto.parseFrom(entry.getValue()));
        iter.next();
    }
    return rsrcs;
}