Search in sources :

Example 96 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class ContainerManagerImpl method performContainerPreStartChecks.

private void performContainerPreStartChecks(NMTokenIdentifier nmTokenIdentifier, StartContainerRequest request, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken {
    /*
   * 1) It should save the NMToken into NMTokenSecretManager. This is done
   * here instead of RPC layer because at the time of opening/authenticating
   * the connection it doesn't know what all RPC calls user will make on it.
   * Also new NMToken is issued only at startContainer (once it gets
   * renewed).
   *
   * 2) It should validate containerToken. Need to check below things. a) It
   * is signed by correct master key (part of retrieve password). b) It
   * belongs to correct Node Manager (part of retrieve password). c) It has
   * correct RMIdentifier. d) It is not expired.
   */
    authorizeStartAndResourceIncreaseRequest(nmTokenIdentifier, containerTokenIdentifier, true);
    // update NMToken
    updateNMTokenIdentifier(nmTokenIdentifier);
    ContainerLaunchContext launchContext = request.getContainerLaunchContext();
    Map<String, ByteBuffer> serviceData = getAuxServiceMetaData();
    if (launchContext.getServiceData() != null && !launchContext.getServiceData().isEmpty()) {
        for (Entry<String, ByteBuffer> meta : launchContext.getServiceData().entrySet()) {
            if (null == serviceData.get(meta.getKey())) {
                throw new InvalidAuxServiceException("The auxService:" + meta.getKey() + " does not exist");
            }
        }
    }
}
Also used : ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ByteString(com.google.protobuf.ByteString) ByteBuffer(java.nio.ByteBuffer) InvalidAuxServiceException(org.apache.hadoop.yarn.exceptions.InvalidAuxServiceException)

Example 97 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class MockRM method submitApp.

public RMApp submitApp(Resource capability, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval, LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete, Priority priority, String amLabel, Map<ApplicationTimeoutType, Long> applicationTimeouts, ByteBuffer tokensConf) throws Exception {
    ApplicationId appId = isAppIdProvided ? applicationId : null;
    ApplicationClientProtocol client = getClientRMService();
    if (!isAppIdProvided) {
        GetNewApplicationResponse resp = client.getNewApplication(Records.newRecord(GetNewApplicationRequest.class));
        appId = resp.getApplicationId();
    }
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setKeepContainersAcrossApplicationAttempts(keepContainers);
    sub.setApplicationId(appId);
    sub.setApplicationName(name);
    sub.setMaxAppAttempts(maxAppAttempts);
    if (applicationTimeouts != null && applicationTimeouts.size() > 0) {
        sub.setApplicationTimeouts(applicationTimeouts);
    }
    if (unmanaged) {
        sub.setUnmanagedAM(true);
    }
    if (queue != null) {
        sub.setQueue(queue);
    }
    if (priority != null) {
        sub.setPriority(priority);
    }
    sub.setApplicationType(appType);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class);
    sub.setResource(capability);
    clc.setApplicationACLs(acls);
    if (ts != null && UserGroupInformation.isSecurityEnabled()) {
        DataOutputBuffer dob = new DataOutputBuffer();
        ts.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        clc.setTokens(securityTokens);
        clc.setTokensConf(tokensConf);
    }
    sub.setAMContainerSpec(clc);
    sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    if (logAggregationContext != null) {
        sub.setLogAggregationContext(logAggregationContext);
    }
    sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
    ResourceRequest amResourceRequest = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
    if (amLabel != null && !amLabel.isEmpty()) {
        amResourceRequest.setNodeLabelExpression(amLabel.trim());
    }
    sub.setAMContainerResourceRequest(amResourceRequest);
    req.setApplicationSubmissionContext(sub);
    UserGroupInformation fakeUser = UserGroupInformation.createUserForTesting(user, new String[] { "someGroup" });
    PrivilegedExceptionAction<SubmitApplicationResponse> action = new PrivilegedExceptionAction<SubmitApplicationResponse>() {

        ApplicationClientProtocol client;

        SubmitApplicationRequest req;

        @Override
        public SubmitApplicationResponse run() throws IOException, YarnException {
            try {
                return client.submitApplication(req);
            } catch (YarnException | IOException e) {
                e.printStackTrace();
                throw e;
            }
        }

        PrivilegedExceptionAction<SubmitApplicationResponse> setClientReq(ApplicationClientProtocol client, SubmitApplicationRequest req) {
            this.client = client;
            this.req = req;
            return this;
        }
    }.setClientReq(client, req);
    fakeUser.doAs(action);
    // make sure app is immediately available after submit
    if (waitForAccepted) {
        waitForState(appId, RMAppState.ACCEPTED);
    }
    RMApp rmApp = getRMContext().getRMApps().get(appId);
    // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
    if (waitForAccepted && !unmanaged) {
        waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.SCHEDULED);
    }
    return rmApp;
}
Also used : RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) GetNewApplicationResponse(org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) IOException(java.io.IOException) PrivilegedExceptionAction(java.security.PrivilegedExceptionAction) SubmitApplicationResponse(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse) ApplicationClientProtocol(org.apache.hadoop.yarn.api.ApplicationClientProtocol) ByteBuffer(java.nio.ByteBuffer) SubmitApplicationRequest(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) GetNewApplicationRequest(org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest) ApplicationSubmissionContext(org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext) DataOutputBuffer(org.apache.hadoop.io.DataOutputBuffer) ResourceRequest(org.apache.hadoop.yarn.api.records.ResourceRequest) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation)

Example 98 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class RegisterApplicationMasterResponsePBImpl method getClientToAMTokenMasterKey.

@Override
public ByteBuffer getClientToAMTokenMasterKey() {
    maybeInitBuilder();
    ByteBuffer key = ByteBuffer.wrap(builder.getClientToAmTokenMasterKey().toByteArray());
    return key;
}
Also used : ByteBuffer(java.nio.ByteBuffer)

Example 99 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class StartContainersResponsePBImpl method initServicesMetaData.

private void initServicesMetaData() {
    if (this.servicesMetaData != null) {
        return;
    }
    StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder;
    List<StringBytesMapProto> list = p.getServicesMetaDataList();
    this.servicesMetaData = new HashMap<String, ByteBuffer>();
    for (StringBytesMapProto c : list) {
        this.servicesMetaData.put(c.getKey(), convertFromProtoFormat(c.getValue()));
    }
}
Also used : StartContainersResponseProtoOrBuilder(org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProtoOrBuilder) StringBytesMapProto(org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto) ByteString(com.google.protobuf.ByteString) ByteBuffer(java.nio.ByteBuffer)

Example 100 with ByteBuffer

use of java.nio.ByteBuffer in project hadoop by apache.

the class AMLauncher method setupTokens.

@Private
@VisibleForTesting
protected void setupTokens(ContainerLaunchContext container, ContainerId containerID) throws IOException {
    Map<String, String> environment = container.getEnvironment();
    environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase());
    // Set AppSubmitTime to be consumable by the AM.
    ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
    environment.put(ApplicationConstants.APP_SUBMIT_TIME_ENV, String.valueOf(rmContext.getRMApps().get(applicationId).getSubmitTime()));
    Credentials credentials = new Credentials();
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    ByteBuffer tokens = container.getTokens();
    if (tokens != null) {
        // TODO: Don't do this kind of checks everywhere.
        dibb.reset(tokens);
        credentials.readTokenStorageStream(dibb);
        tokens.rewind();
    }
    // Add AMRMToken
    Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
    if (amrmToken != null) {
        credentials.addToken(amrmToken.getService(), amrmToken);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}
Also used : AMRMTokenIdentifier(org.apache.hadoop.yarn.security.AMRMTokenIdentifier) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) DataOutputBuffer(org.apache.hadoop.io.DataOutputBuffer) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) ByteBuffer(java.nio.ByteBuffer) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) Credentials(org.apache.hadoop.security.Credentials) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Private(org.apache.hadoop.classification.InterfaceAudience.Private)

Aggregations

ByteBuffer (java.nio.ByteBuffer)8537 Test (org.junit.Test)1905 IOException (java.io.IOException)972 ArrayList (java.util.ArrayList)430 File (java.io.File)216 FileChannel (java.nio.channels.FileChannel)204 MappedByteBuffer (java.nio.MappedByteBuffer)190 HashMap (java.util.HashMap)172 CharBuffer (java.nio.CharBuffer)144 ByteArrayOutputStream (java.io.ByteArrayOutputStream)138 InetSocketAddress (java.net.InetSocketAddress)137 Random (java.util.Random)119 InputStream (java.io.InputStream)115 FileInputStream (java.io.FileInputStream)114 Map (java.util.Map)113 WebSocketFrame (org.eclipse.jetty.websocket.common.WebSocketFrame)99 Test (org.testng.annotations.Test)96 IntBuffer (java.nio.IntBuffer)93 SocketChannel (java.nio.channels.SocketChannel)92 FileOutputStream (java.io.FileOutputStream)87