Examples with ContainerManagerApplicationProto org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto used on opensource projects

Example 1 with ContainerManagerApplicationProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testApplicationStorage.

@Test
public void testApplicationStorage() throws IOException {
    // test empty when no state
    RecoveredApplicationsState state = stateStore.loadApplicationsState();
    assertTrue(state.getApplications().isEmpty());
    // store an application and verify recovered
    final ApplicationId appId1 = ApplicationId.newInstance(1234, 1);
    ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder();
    builder.setId(((ApplicationIdPBImpl) appId1).getProto());
    builder.setUser("user1");
    ContainerManagerApplicationProto appProto1 = builder.build();
    stateStore.storeApplication(appId1, appProto1);
    restartStateStore();
    state = stateStore.loadApplicationsState();
    assertEquals(1, state.getApplications().size());
    assertEquals(appProto1, state.getApplications().get(0));
    // add a new app
    final ApplicationId appId2 = ApplicationId.newInstance(1234, 2);
    builder = ContainerManagerApplicationProto.newBuilder();
    builder.setId(((ApplicationIdPBImpl) appId2).getProto());
    builder.setUser("user2");
    ContainerManagerApplicationProto appProto2 = builder.build();
    stateStore.storeApplication(appId2, appProto2);
    restartStateStore();
    state = stateStore.loadApplicationsState();
    assertEquals(2, state.getApplications().size());
    assertTrue(state.getApplications().contains(appProto1));
    assertTrue(state.getApplications().contains(appProto2));
    // test removing an application
    stateStore.removeApplication(appId2);
    restartStateStore();
    state = stateStore.loadApplicationsState();
    assertEquals(1, state.getApplications().size());
    assertEquals(appProto1, state.getApplications().get(0));
}

Example 2 with ContainerManagerApplicationProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto in project hadoop by apache.

the class ContainerManagerImpl method buildAppProto.

private ContainerManagerApplicationProto buildAppProto(ApplicationId appId, String user, Credentials credentials, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
    ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder();
    builder.setId(((ApplicationIdPBImpl) appId).getProto());
    builder.setUser(user);
    if (logAggregationContext != null) {
        builder.setLogAggregationContext(((LogAggregationContextPBImpl) logAggregationContext).getProto());
    }
    builder.clearCredentials();
    if (credentials != null) {
        DataOutputBuffer dob = new DataOutputBuffer();
        try {
            credentials.writeTokenStorageToStream(dob);
            builder.setCredentials(ByteString.copyFrom(dob.getData()));
        } catch (IOException e) {
            // should not occur
            LOG.error("Cannot serialize credentials", e);
        }
    }
    builder.clearAcls();
    if (appAcls != null) {
        for (Map.Entry<ApplicationAccessType, String> acl : appAcls.entrySet()) {
            ApplicationACLMapProto p = ApplicationACLMapProto.newBuilder().setAccessType(ProtoUtils.convertToProtoFormat(acl.getKey())).setAcl(acl.getValue()).build();
            builder.addAcls(p);
        }
    }
    return builder.build();
}

Example 3 with ContainerManagerApplicationProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto in project hadoop by apache.

the class ContainerManagerImpl method recover.

@SuppressWarnings("unchecked")
private void recover() throws IOException, URISyntaxException {
    NMStateStoreService stateStore = context.getNMStateStore();
    if (stateStore.canRecover()) {
        rsrcLocalizationSrvc.recoverLocalizedResources(stateStore.loadLocalizationState());
        RecoveredApplicationsState appsState = stateStore.loadApplicationsState();
        for (ContainerManagerApplicationProto proto : appsState.getApplications()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Recovering application with state: " + proto.toString());
            }
            recoverApplication(proto);
        }
        for (RecoveredContainerState rcs : stateStore.loadContainersState()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Recovering container with state: " + rcs);
            }
            recoverContainer(rcs);
        }
    } else {
        LOG.info("Not a recoverable state store. Nothing to recover.");
    }
}

Example 4 with ContainerManagerApplicationProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto in project hadoop by apache.

the class ApplicationImpl method buildAppProto.

static ContainerManagerApplicationProto buildAppProto(ApplicationImpl app) throws IOException {
    ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder();
    builder.setId(((ApplicationIdPBImpl) app.appId).getProto());
    builder.setUser(app.getUser());
    if (app.logAggregationContext != null) {
        builder.setLogAggregationContext(((LogAggregationContextPBImpl) app.logAggregationContext).getProto());
    }
    builder.clearCredentials();
    if (app.credentials != null) {
        DataOutputBuffer dob = new DataOutputBuffer();
        app.credentials.writeTokenStorageToStream(dob);
        builder.setCredentials(ByteString.copyFrom(dob.getData()));
    }
    builder.clearAcls();
    if (app.applicationACLs != null) {
        for (Map.Entry<ApplicationAccessType, String> acl : app.applicationACLs.entrySet()) {
            YarnProtos.ApplicationACLMapProto p = YarnProtos.ApplicationACLMapProto.newBuilder().setAccessType(ProtoUtils.convertToProtoFormat(acl.getKey())).setAcl(acl.getValue()).build();
            builder.addAcls(p);
        }
    }
    builder.setAppLogAggregationInitedTime(app.applicationLogInitedTimestamp);
    return builder.build();
}

Example 5 with ContainerManagerApplicationProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto in project hadoop by apache.

the class NMLeveldbStateStoreService method loadApplicationsState.

@Override
public RecoveredApplicationsState loadApplicationsState() throws IOException {
    RecoveredApplicationsState state = new RecoveredApplicationsState();
    state.applications = new ArrayList<ContainerManagerApplicationProto>();
    String keyPrefix = APPLICATIONS_KEY_PREFIX;
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(keyPrefix));
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String key = asString(entry.getKey());
            if (!key.startsWith(keyPrefix)) {
                break;
            }
            state.applications.add(ContainerManagerApplicationProto.parseFrom(entry.getValue()));
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    cleanupDeprecatedFinishedApps();
    return state;
}