Examples with YarnClientApplication org.apache.hadoop.yarn.client.api.YarnClientApplication used on opensource projects

Example 6 with YarnClientApplication

use of org.apache.hadoop.yarn.client.api.YarnClientApplication in project apex-core by apache.

the class InlineAM method run.

public boolean run() throws Exception {
    LOG.info("Starting Client");
    // Connect to ResourceManager
    rmClient.start();
    try {
        // Get a new application id
        YarnClientApplication newApp = rmClient.createApplication();
        ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
        // Create launch context for app master
        LOG.info("Setting up application submission context for ASM");
        ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
        // set the application id
        appContext.setApplicationId(appId);
        // set the application name
        appContext.setApplicationName(appName);
        // Set the priority for the application master
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(amPriority);
        appContext.setPriority(pri);
        // Set the queue to which this application is to be submitted in the RM
        appContext.setQueue(amQueue);
        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        appContext.setAMContainerSpec(amContainer);
        // unmanaged AM
        appContext.setUnmanagedAM(true);
        LOG.info("Setting unmanaged AM");
        // Submit the application to the applications manager
        LOG.info("Submitting application to ASM");
        rmClient.submitApplication(appContext);
        // Monitor the application to wait for launch state
        ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED));
        ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
        LOG.info("Launching application with id: " + attemptId);
        // launch AM
        runAM(attemptId);
        // Monitor the application for end state
        appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        YarnApplicationState appState = appReport.getYarnApplicationState();
        FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
        LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);
        boolean success;
        if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
            LOG.info("Application has completed successfully.");
            success = true;
        } else {
            LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString());
            success = false;
        }
        return success;
    } finally {
        rmClient.stop();
    }
}

Example 7 with YarnClientApplication

use of org.apache.hadoop.yarn.client.api.YarnClientApplication in project asterixdb by apache.

the class AsterixYARNClient method startAction.

private static void startAction(AsterixYARNClient client) throws YarnException {
    YarnClientApplication app;
    List<DFSResourceCoordinate> res;
    ApplicationId appId;
    try {
        app = client.makeApplicationContext();
        res = client.deployConfig();
        res.addAll(client.distributeBinaries());
        appId = client.deployAM(app, res, client.mode);
        LOG.info("Asterix started up with Application ID: " + appId.toString());
        if (Utils.waitForLiveness(appId, "Waiting for AsterixDB instance to resume ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) {
            System.out.println("Asterix successfully deployed and is now running.");
        } else {
            LOG.fatal("AsterixDB appears to have failed to install and start");
            throw new YarnException("AsterixDB appears to have failed to install and start");
        }
    } catch (IOException e) {
        throw new YarnException(e);
    }
}

Example 8 with YarnClientApplication

use of org.apache.hadoop.yarn.client.api.YarnClientApplication in project asterixdb by apache.

the class AsterixYARNClient method installAction.

private static void installAction(AsterixYARNClient client) throws YarnException {
    YarnClientApplication app;
    List<DFSResourceCoordinate> res;
    ApplicationId appId;
    try {
        app = client.makeApplicationContext();
        client.installConfig();
        client.writeAsterixConfig(Utils.parseYarnClusterConfig(client.asterixConf));
        client.installAsterixConfig(false);
        res = client.deployConfig();
        res.addAll(client.distributeBinaries());
        appId = client.deployAM(app, res, client.mode);
        LOG.info("Asterix started up with Application ID: " + appId.toString());
        if (Utils.waitForLiveness(appId, "Waiting for new AsterixDB Instance to start ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) {
            System.out.println("Asterix successfully deployed and is now running.");
        } else {
            LOG.fatal("AsterixDB appears to have failed to install and start");
            throw new YarnException("AsterixDB appears to have failed to install and start");
        }
    } catch (IOException e) {
        LOG.fatal("Asterix failed to deploy on to cluster");
        throw new YarnException(e);
    }
}

Example 9 with YarnClientApplication

use of org.apache.hadoop.yarn.client.api.YarnClientApplication in project incubator-systemml by apache.

the class DMLYarnClient method launchDMLYarnAppmaster.

/**
	 * Method to launch the dml yarn app master and execute the given dml script
	 * with the given configuration and jar file.
	 * 
	 * NOTE: on launching the yarn app master, we do not explicitly probe if we
	 *	  are running on a yarn or MR1 cluster. In case of MR1, already the class 
	 *	  YarnConfiguration will not be found and raise a classnotfound. In case of any 
	 *	  exception we fall back to run CP directly in the client process.
	 * 
	 * @return true if dml program successfully executed as yarn app master
	 * @throws IOException if IOException occurs
	 * @throws DMLScriptException if DMLScriptException occurs
	 */
protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException {
    boolean ret = false;
    String hdfsWD = null;
    try {
        Timing time = new Timing(true);
        // load yarn configuration
        YarnConfiguration yconf = new YarnConfiguration();
        // create yarn client
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(yconf);
        yarnClient.start();
        // create application and get the ApplicationID
        YarnClientApplication app = yarnClient.createApplication();
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        LOG.debug("Created application (applicationID: " + appId + ")");
        // prepare hdfs working directory via ApplicationID
        // copy script, config, jar file to hdfs
        hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId);
        copyResourcesToHdfsWorkingDir(yconf, hdfsWD);
        //construct command line argument
        String command = constructAMCommand(_args, _dmlConfig);
        LOG.debug("Constructed application master command: \n" + command);
        // set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        amContainer.setCommands(Collections.singletonList(command));
        amContainer.setLocalResources(constructLocalResourceMap(yconf));
        amContainer.setEnvironment(constructEnvionmentMap(yconf));
        // Set up resource type requirements for ApplicationMaster
        int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM);
        int memAlloc = (int) computeMemoryAllocation(memHeap);
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(memAlloc);
        capability.setVirtualCores(NUM_CORES);
        LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES);
        // Finally, set-up ApplicationSubmissionContext for the application
        String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE);
        // application name
        appContext.setApplicationName(APPMASTER_NAME);
        appContext.setAMContainerSpec(amContainer);
        appContext.setResource(capability);
        // queue
        appContext.setQueue(qname);
        LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname);
        // submit application (non-blocking)
        yarnClient.submitApplication(appContext);
        // Check application status periodically (and output web ui address)
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        LOG.info("Application tracking-URL: " + appReport.getTrackingUrl());
        YarnApplicationState appState = appReport.getYarnApplicationState();
        YarnApplicationState oldState = appState;
        LOG.info("Application state: " + appState);
        while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) {
            //wait for 200ms
            Thread.sleep(APP_STATE_INTERVAL);
            appReport = yarnClient.getApplicationReport(appId);
            appState = appReport.getYarnApplicationState();
            if (appState != oldState) {
                oldState = appState;
                LOG.info("Application state: " + appState);
            }
        }
        //check final status (failed or succeeded)
        FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
        LOG.info("Application final status: " + finalState);
        //show application and total runtime
        double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000;
        LOG.info("Application runtime: " + appRuntime + " sec.");
        LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec.");
        //raised script-level error in case of failed final status
        if (finalState != FinalApplicationStatus.SUCCEEDED) {
            //propagate script-level stop call message
            String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId);
            if (stop_msg != null)
                throw new DMLScriptException(stop_msg);
            //generic failure message
            throw new DMLRuntimeException("DML yarn app master finished with final status: " + finalState + ".");
        }
        ret = true;
    } catch (DMLScriptException ex) {
        //rethrow DMLScriptException to propagate stop call
        throw ex;
    } catch (Exception ex) {
        LOG.error("Failed to run DML yarn app master.", ex);
        ret = false;
    } finally {
        //cleanup working directory
        if (hdfsWD != null)
            MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD);
    }
    return ret;
}

Example 10 with YarnClientApplication

use of org.apache.hadoop.yarn.client.api.YarnClientApplication in project weave by continuuity.

the class Hadoop21YarnAppClient method createLauncher.

@Override
public ProcessLauncher<ApplicationId> createLauncher(WeaveSpecification weaveSpec) throws Exception {
    // Request for new application
    YarnClientApplication application = yarnClient.createApplication();
    final GetNewApplicationResponse response = application.getNewApplicationResponse();
    final ApplicationId appId = response.getApplicationId();
    // Setup the context for application submission
    final ApplicationSubmissionContext appSubmissionContext = application.getApplicationSubmissionContext();
    appSubmissionContext.setApplicationId(appId);
    appSubmissionContext.setApplicationName(weaveSpec.getName());
    ApplicationSubmitter submitter = new ApplicationSubmitter() {

        @Override
        public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context, Resource capability) {
            ContainerLaunchContext launchContext = context.getLaunchContext();
            addRMToken(launchContext);
            appSubmissionContext.setAMContainerSpec(launchContext);
            appSubmissionContext.setResource(adjustMemory(response, capability));
            appSubmissionContext.setMaxAppAttempts(2);
            try {
                yarnClient.submitApplication(appSubmissionContext);
                return new ProcessControllerImpl(yarnClient, appId);
            } catch (Exception e) {
                LOG.error("Failed to submit application {}", appId, e);
                throw Throwables.propagate(e);
            }
        }
    };
    return new ApplicationMasterProcessLauncher(appId, submitter);
}