Examples with LocalResourcePBImpl org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl used on opensource projects

Example 6 with LocalResourcePBImpl

use of org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreFailedLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent(lr1, new Exception("Test").toString());
        tracker.handle(rfe1);
        dispatcher.await();
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 7 with LocalResourcePBImpl

use of org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl in project hadoop by apache.

the class TestPBRecordImpl method createResource.

static LocalResource createResource() {
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    assertTrue(ret instanceof LocalResourcePBImpl);
    ret.setResource(URL.fromPath(new Path("hdfs://y.ak:9820/foo/bar")));
    ret.setSize(4344L);
    ret.setTimestamp(3141592653589793L);
    ret.setVisibility(LocalResourceVisibility.PUBLIC);
    return ret;
}

Example 8 with LocalResourcePBImpl

use of org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl in project hadoop by apache.

the class ResourceLocalizationSpecPBImpl method getResource.

@Override
public LocalResource getResource() {
    ResourceLocalizationSpecProtoOrBuilder p = viaProto ? proto : builder;
    if (resource != null) {
        return resource;
    }
    if (!p.hasResource()) {
        return null;
    }
    resource = new LocalResourcePBImpl(p.getResource());
    return resource;
}

Example 9 with LocalResourcePBImpl

use of org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl in project hadoop by apache.

the class ResourceLocalizationSpecPBImpl method mergeLocalToBuilder.

private void mergeLocalToBuilder() {
    ResourceLocalizationSpecProtoOrBuilder l = viaProto ? proto : builder;
    if (this.resource != null && !(l.getResource().equals(((LocalResourcePBImpl) resource).getProto()))) {
        maybeInitBuilder();
        builder.setResource(((LocalResourcePBImpl) resource).getProto());
    }
    if (this.destinationDirectory != null && !(l.getDestinationDirectory().equals(((URLPBImpl) destinationDirectory).getProto()))) {
        maybeInitBuilder();
        builder.setDestinationDirectory(((URLPBImpl) destinationDirectory).getProto());
    }
}

Example 10 with LocalResourcePBImpl

use of org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl in project hadoop by apache.

the class ResourceLocalizationService method recoverTrackerResources.

private void recoverTrackerResources(LocalResourcesTracker tracker, LocalResourceTrackerState state) throws URISyntaxException {
    for (LocalizedResourceProto proto : state.getLocalizedResources()) {
        LocalResource rsrc = new LocalResourcePBImpl(proto.getResource());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Recovering localized resource " + req + " at " + proto.getLocalPath());
        }
        tracker.handle(new ResourceRecoveredEvent(req, new Path(proto.getLocalPath()), proto.getSize()));
    }
    for (Map.Entry<LocalResourceProto, Path> entry : state.getInProgressResources().entrySet()) {
        LocalResource rsrc = new LocalResourcePBImpl(entry.getKey());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        Path localPath = entry.getValue();
        tracker.handle(new ResourceRecoveredEvent(req, localPath, 0));
        // delete any in-progress localizations, containers will request again
        LOG.info("Deleting in-progress localization for " + req + " at " + localPath);
        tracker.remove(tracker.getLocalizedResource(req), delService);
    }
// TODO: remove untracked directories in local filesystem
}