Examples with DataOutputBuffer org.apache.hadoop.io.DataOutputBuffer used on opensource projects

Example 86 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project kitten by cloudera.

the class ApplicationMasterServiceImpl method startUp.

@Override
protected void startUp() throws IOException {
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Create appSubmitterUgi and add original tokens to it
    String userName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(userName);
    appSubmitterUgi.addCredentials(credentials);
    this.resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, this);
    this.resourceManager.init(conf);
    this.resourceManager.start();
    RegisterApplicationMasterResponse registration;
    try {
        registration = resourceManager.registerApplicationMaster(parameters.getHostname(), parameters.getClientPort(), parameters.getTrackingUrl());
    } catch (Exception e) {
        LOG.error("Exception thrown registering application master", e);
        stop();
        return;
    }
    ContainerLaunchContextFactory factory = new ContainerLaunchContextFactory(registration.getMaximumResourceCapability(), tokens);
    for (ContainerLaunchParameters clp : parameters.getContainerLaunchParameters()) {
        ContainerTracker tracker = new ContainerTracker(clp);
        tracker.init(factory);
        trackers.add(tracker);
    }
    this.hasRunningContainers = true;
}

Example 87 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project SQLWindowing by hbutani.

the class DataType method cloneInstance.

public T cloneInstance(T instance) throws IOException {
    DataOutputBuffer obuffer = serializeBuffers.get();
    obuffer.reset();
    instance.write(obuffer);
    DataInputBuffer ibuffer = materializeBuffers.get();
    byte[] b = obuffer.getData();
    ibuffer.reset(b, 0, b.length);
    T nv = create();
    nv.readFields(ibuffer);
    return nv;
}

Example 88 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project jstorm by alibaba.

the class JstormOnYarn method run.

/**
     * Main run function for the client
     *
     * @return true if application completed successfully
     * @throws IOException
     * @throws YarnException
     */
public boolean run() throws IOException, YarnException {
    LOG.info("Running Client");
    jstormClientContext.yarnClient.start();
    YarnClusterMetrics clusterMetrics = jstormClientContext.yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());
    List<NodeReport> clusterNodeReports = jstormClientContext.yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM, count is " + String.valueOf(clusterNodeReports.size()));
    QueueInfo queueInfo = jstormClientContext.yarnClient.getQueueInfo(this.jstormClientContext.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size());
    List<QueueUserACLInfo> listAclInfo = jstormClientContext.yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name());
        }
    }
    if (jstormClientContext.domainId != null && jstormClientContext.domainId.length() > 0 && jstormClientContext.toCreateDomain) {
        prepareTimelineDomain();
    }
    // Get a new application id
    YarnClientApplication app = jstormClientContext.yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
    // A resource ask cannot exceed the max.
    if (jstormClientContext.amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + jstormClientContext.amMemory + ", max=" + maxMem);
        jstormClientContext.amMemory = maxMem;
    }
    int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
    if (jstormClientContext.amVCores > maxVCores) {
        LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + jstormClientContext.amVCores + ", max=" + maxVCores);
        jstormClientContext.amVCores = maxVCores;
    }
    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setKeepContainersAcrossApplicationAttempts(jstormClientContext.keepContainers);
    if (!StringUtils.isBlank(jstormClientContext.instanceName) && JOYConstants.CLIIENT_CLASS.equals(jstormClientContext.appName)) {
        appContext.setApplicationName(jstormClientContext.appName + JOYConstants.DOT + jstormClientContext.instanceName);
    } else {
        appContext.setApplicationName(jstormClientContext.appName);
    }
    if (jstormClientContext.attemptFailuresValidityInterval >= 0) {
        appContext.setAttemptFailuresValidityInterval(jstormClientContext.attemptFailuresValidityInterval);
    }
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    FileSystem fs = FileSystem.get(jstormClientContext.conf);
    addToLocalResources(fs, jstormClientContext.appMasterJar, JOYConstants.appMasterJarPath, appId.toString(), localResources, null);
    //add jstormOnYarn's configuration to resources which path is jstorm-yarn.xml
    if (jstormClientContext.confFile == null || jstormClientContext.confFile.isEmpty()) {
        addToLocalResources(fs, JOYConstants.CONF_NAME, JOYConstants.CONF_NAME, appId.toString(), localResources, null);
    } else {
        addToLocalResources(fs, jstormClientContext.confFile, JOYConstants.CONF_NAME, appId.toString(), localResources, null);
    }
    if (jstormClientContext.libJars != null && !jstormClientContext.libJars.isEmpty()) {
        for (String libPath : jstormClientContext.libJars.split(JOYConstants.COMMA)) {
            String[] strArr = libPath.split(JOYConstants.BACKLASH);
            String libName = strArr[strArr.length - 1];
            addToLocalResources(fs, libPath, libName, appId.toString(), localResources, null);
        }
    }
    // Set the log4j properties if needed
    if (!jstormClientContext.log4jPropFile.isEmpty()) {
        addToLocalResources(fs, jstormClientContext.log4jPropFile, JOYConstants.log4jPath, appId.toString(), localResources, null);
    }
    // The shell script has to be made available on the final container(s)
    // where it will be executed.
    String hdfsShellScriptLocation = JOYConstants.EMPTY;
    long hdfsShellScriptLen = 0;
    long hdfsShellScriptTimestamp = 0;
    if (!jstormClientContext.shellScriptPath.isEmpty()) {
        Path shellSrc = new Path(jstormClientContext.shellScriptPath);
        String shellPathSuffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId.toString() + JOYConstants.BACKLASH + JOYConstants.SCRIPT_PATH;
        Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix);
        fs.copyFromLocalFile(false, true, shellSrc, shellDst);
        hdfsShellScriptLocation = shellDst.toUri().toString();
        FileStatus shellFileStatus = fs.getFileStatus(shellDst);
        hdfsShellScriptLen = shellFileStatus.getLen();
        hdfsShellScriptTimestamp = shellFileStatus.getModificationTime();
    }
    if (!jstormClientContext.shellCommand.isEmpty()) {
        addToLocalResources(fs, null, JOYConstants.shellCommandPath, appId.toString(), localResources, jstormClientContext.shellCommand);
    }
    if (jstormClientContext.shellArgs.length > 0) {
        addToLocalResources(fs, null, JOYConstants.shellArgsPath, appId.toString(), localResources, StringUtils.join(jstormClientContext.shellArgs, JOYConstants.BLANK));
    }
    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    Path appMaterJar = new Path(fs.getHomeDirectory(), jstormClientContext.appName + JOYConstants.BACKLASH + appId.toString() + JOYConstants.BACKLASH + JOYConstants.appMasterJarPath);
    FileStatus jarFIleStatus = fs.getFileStatus(appMaterJar);
    // put location of shell script into env
    // using the env info, the application master will create the correct local resource for the
    // eventual containers that will be launched to execute the shell scripts
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp));
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen));
    if (jstormClientContext.domainId != null && jstormClientContext.domainId.length() > 0) {
        env.put(JOYConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN, jstormClientContext.domainId);
    }
    env.put(JOYConstants.APPMASTERJARSCRIPTLOCATION, appMaterJar.toUri().toString());
    env.put(JOYConstants.APPMASTERLEN, Long.toString(jarFIleStatus.getLen()));
    env.put(JOYConstants.APPMASTERTIMESTAMP, Long.toString(jarFIleStatus.getModificationTime()));
    env.put(JOYConstants.APPMASTERJARSCRIPTLOCATION, appMaterJar.toUri().toString());
    env.put(JOYConstants.BINARYFILEDEPLOYPATH, jstormClientContext.deployPath);
    env.put(JOYConstants.INSTANCENAME, jstormClientContext.instanceName);
    // Add AppMaster.jar location to classpath
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()).append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(JOYConstants.DOT + JOYConstants.BACKLASH + JOYConstants.ASTERISK);
    for (String c : jstormClientContext.conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(JOYConstants.DOT + JOYConstants.BACKLASH + JOYConstants.LOG_PROPERTIES);
    // add the runtime classpath needed for tests to work
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(JOYConstants.COLON);
        classPathEnv.append(System.getProperty(JOYConstants.JAVA_CLASS_PATH));
    }
    env.put(JOYConstants.CLASS_PATH, classPathEnv.toString());
    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);
    // Set java executable command
    vargs.add(Environment.JAVA_HOME.$$() + JOYConstants.JAVA);
    // Set Xmx based on am memory size
    vargs.add(JOYConstants.XMX + jstormClientContext.amMemory + JOYConstants.MB);
    // Set class name
    vargs.add(appMasterMainClass);
    // Set params for Application Master
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.CONTAINER_MEMORY + JOYConstants.BLANK + String.valueOf(jstormClientContext.containerMemory));
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.CONTAINER_VCORES + JOYConstants.BLANK + String.valueOf(jstormClientContext.containerVirtualCores));
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.NUM_CONTAINERS + JOYConstants.BLANK + String.valueOf(jstormClientContext.numContainers));
    if (null != jstormClientContext.nodeLabelExpression) {
        appContext.setNodeLabelExpression(jstormClientContext.nodeLabelExpression);
    }
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.PRIORITY + JOYConstants.BLANK + String.valueOf(jstormClientContext.shellCmdPriority));
    for (Map.Entry<String, String> entry : jstormClientContext.shellEnv.entrySet()) {
        vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.SHELL_ENV + JOYConstants.BLANK + entry.getKey() + JOYConstants.EQUAL + entry.getValue());
    }
    if (jstormClientContext.debugFlag) {
        vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.DEBUG);
    }
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + JOYConstants.APPMASTER_STDOUT);
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + JOYConstants.APPMASTER_STDERR);
    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(JOYConstants.BLANK);
    }
    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null);
    // Set up resource type requirements
    Resource capability = Resource.newInstance(jstormClientContext.amMemory, jstormClientContext.amVCores);
    appContext.setResource(capability);
    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
        Credentials credentials = new Credentials();
        String tokenRenewer = jstormClientContext.conf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }
        // For now, only getting tokens for the default file-system.
        final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }
    appContext.setAMContainerSpec(amContainer);
    // Set the priority for the application master
    Priority pri = Priority.newInstance(jstormClientContext.amPriority);
    appContext.setPriority(pri);
    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(jstormClientContext.amQueue);
    LOG.info("Submitting application to ASM");
    //check configuration
    if (JstormYarnUtils.isUnset(jstormClientContext.conf.get(JOYConstants.INSTANCE_NAME_KEY))) {
        throw new IOException(JOYConstants.INSTANCE_NAME_KEY + " is not set");
    }
    if (JstormYarnUtils.isUnset(jstormClientContext.conf.get(JOYConstants.INSTANCE_DEPLOY_DIR_KEY))) {
        throw new IOException(JOYConstants.INSTANCE_DEPLOY_DIR_KEY + " is not set");
    }
    jstormClientContext.yarnClient.submitApplication(appContext);
    // Monitor the application
    return monitorApplication(appId);
}

Example 89 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project ambrose by twitter.

the class JSONUtilTest method test.

public void test(Object obj, String expected) throws IOException {
    DataOutputBuffer out = new DataOutputBuffer();
    PrintWriter writer = new PrintWriter(out);
    JSONUtil.writeJson(writer, obj);
    String actual = new String(out.getData(), 0, out.getLength(), Charsets.UTF_8.name());
    assertEquals(expected, actual);
}

Example 90 with DataOutputBuffer

use of org.apache.hadoop.io.DataOutputBuffer in project hadoop by apache.

the class AMRMProxyService method processApplicationStartRequest.

/**
   * Callback from the ContainerManager implementation for initializing the
   * application request processing pipeline.
   *
   * @param request - encapsulates information for starting an AM
   * @throws IOException
   * @throws YarnException
   */
public void processApplicationStartRequest(StartContainerRequest request) throws IOException, YarnException {
    LOG.info("Callback received for initializing request " + "processing pipeline for an AM");
    ContainerTokenIdentifier containerTokenIdentifierForKey = BuilderUtils.newContainerTokenIdentifier(request.getContainerToken());
    ApplicationAttemptId appAttemptId = containerTokenIdentifierForKey.getContainerID().getApplicationAttemptId();
    Credentials credentials = YarnServerSecurityUtils.parseCredentials(request.getContainerLaunchContext());
    Token<AMRMTokenIdentifier> amrmToken = getFirstAMRMToken(credentials.getAllTokens());
    if (amrmToken == null) {
        throw new YarnRuntimeException("AMRMToken not found in the start container request for application:" + appAttemptId.toString());
    }
    // Substitute the existing AMRM Token with a local one. Keep the rest of the
    // tokens in the credentials intact.
    Token<AMRMTokenIdentifier> localToken = this.secretManager.createAndGetAMRMToken(appAttemptId);
    credentials.addToken(localToken.getService(), localToken);
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    request.getContainerLaunchContext().setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
    initializePipeline(containerTokenIdentifierForKey.getContainerID().getApplicationAttemptId(), containerTokenIdentifierForKey.getApplicationSubmitter(), amrmToken, localToken);
}