Examples with YarnApplicationReport com.continuuity.weave.internal.yarn.YarnApplicationReport used on opensource projects

Example 1 with YarnApplicationReport

use of com.continuuity.weave.internal.yarn.YarnApplicationReport in project weave by continuuity.

the class YarnWeaveController method kill.

@Override
public void kill() {
    if (processController != null) {
        YarnApplicationReport report = processController.getReport();
        LOG.info("Killing application {}", report.getApplicationId());
        processController.cancel();
    } else {
        LOG.warn("No process controller for application that is not submitted.");
    }
}

Example 2 with YarnApplicationReport

use of com.continuuity.weave.internal.yarn.YarnApplicationReport in project weave by continuuity.

the class YarnWeaveController method doStartUp.

@Override
protected void doStartUp() {
    super.doStartUp();
    // Submit and poll the status of the yarn application
    try {
        processController = startUp.call();
        YarnApplicationReport report = processController.getReport();
        LOG.debug("Application {} submit", report.getApplicationId());
        YarnApplicationState state = report.getYarnApplicationState();
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        stopWatch.split();
        long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS);
        LOG.info("Checking yarn application status");
        while (!hasRun(state) && stopWatch.getSplitTime() < maxTime) {
            report = processController.getReport();
            state = report.getYarnApplicationState();
            LOG.debug("Yarn application status: {}", state);
            TimeUnit.SECONDS.sleep(1);
            stopWatch.split();
        }
        LOG.info("Yarn application is in state {}", state);
        if (state != YarnApplicationState.RUNNING) {
            LOG.info("Yarn application is not in running state. Shutting down controller.", Constants.APPLICATION_MAX_START_SECONDS);
            forceShutDown();
        } else {
            try {
                URL resourceUrl = URI.create(String.format("http://%s:%d", report.getHost(), report.getRpcPort())).resolve(TrackerService.PATH).toURL();
                resourcesClient = new ResourceReportClient(resourceUrl);
            } catch (IOException e) {
                resourcesClient = null;
            }
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 3 with YarnApplicationReport

use of com.continuuity.weave.internal.yarn.YarnApplicationReport in project weave by continuuity.

the class YarnWeaveController method doShutDown.

@Override
protected void doShutDown() {
    if (processController == null) {
        LOG.warn("No process controller for application that is not submitted.");
        return;
    }
    // Wait for the stop message being processed
    try {
        Uninterruptibles.getUninterruptibly(getStopMessageFuture(), Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
    } catch (Exception e) {
        LOG.error("Failed to wait for stop message being processed.", e);
        // Kill the application through yarn
        kill();
    }
    // Poll application status from yarn
    try {
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        stopWatch.split();
        long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
        YarnApplicationReport report = processController.getReport();
        FinalApplicationStatus finalStatus = report.getFinalApplicationStatus();
        while (finalStatus == FinalApplicationStatus.UNDEFINED && stopWatch.getSplitTime() < maxTime) {
            LOG.debug("Yarn application final status for {} {}", report.getApplicationId(), finalStatus);
            TimeUnit.SECONDS.sleep(1);
            stopWatch.split();
            finalStatus = processController.getReport().getFinalApplicationStatus();
        }
        LOG.debug("Yarn application final status is {}", finalStatus);
        // Application not finished after max stop time, kill the application
        if (finalStatus == FinalApplicationStatus.UNDEFINED) {
            kill();
        }
    } catch (Exception e) {
        LOG.warn("Exception while waiting for application report: {}", e.getMessage(), e);
        kill();
    }
    super.doShutDown();
}

Example 4 with YarnApplicationReport

use of com.continuuity.weave.internal.yarn.YarnApplicationReport in project weave by continuuity.

the class YarnWeavePreparer method start.

@Override
public WeaveController start() {
    try {
        final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(user, weaveSpec);
        final ApplicationId appId = launcher.getContainerInfo();
        Callable<ProcessController<YarnApplicationReport>> submitTask = new Callable<ProcessController<YarnApplicationReport>>() {

            @Override
            public ProcessController<YarnApplicationReport> call() throws Exception {
                String fsUser = locationFactory.getHomeLocation().getName();
                // Local files needed by AM
                Map<String, LocalFile> localFiles = Maps.newHashMap();
                // Local files declared by runnables
                Multimap<String, LocalFile> runnableLocalFiles = HashMultimap.create();
                String vmOpts = jvmOpts.get();
                createAppMasterJar(createBundler(), localFiles);
                createContainerJar(createBundler(), localFiles);
                populateRunnableLocalFiles(weaveSpec, runnableLocalFiles);
                saveWeaveSpec(weaveSpec, runnableLocalFiles, localFiles);
                saveLogback(localFiles);
                saveLauncher(localFiles);
                saveKafka(localFiles);
                saveVmOptions(vmOpts, localFiles);
                saveArguments(new Arguments(arguments, runnableArgs), localFiles);
                saveLocalFiles(localFiles, ImmutableSet.of(Constants.Files.WEAVE_SPEC, Constants.Files.LOGBACK_TEMPLATE, Constants.Files.CONTAINER_JAR, Constants.Files.LAUNCHER_JAR, Constants.Files.ARGUMENTS));
                LOG.debug("Submit AM container spec: {}", appId);
                //     false
                return launcher.prepareLaunch(ImmutableMap.<String, String>builder().put(EnvKeys.WEAVE_FS_USER, fsUser).put(EnvKeys.WEAVE_APP_DIR, getAppLocation().toURI().toASCIIString()).put(EnvKeys.WEAVE_ZK_CONNECT, zkClient.getConnectString()).put(EnvKeys.WEAVE_RUN_ID, runId.getId()).put(EnvKeys.WEAVE_RESERVED_MEMORY_MB, Integer.toString(reservedMemory)).put(EnvKeys.WEAVE_APP_NAME, weaveSpec.getName()).build(), localFiles.values(), credentials).noResources().noEnvironment().withCommands().add("java", "-Djava.io.tmpdir=tmp", "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, "-Dweave.app=$" + EnvKeys.WEAVE_APP_NAME, "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", "-Xmx" + (Constants.APP_MASTER_MEMORY_MB - Constants.APP_MASTER_RESERVED_MEMORY_MB) + "m", vmOpts, WeaveLauncher.class.getName(), Constants.Files.APP_MASTER_JAR, ApplicationMasterMain.class.getName(), Boolean.FALSE.toString()).redirectOutput(Constants.STDOUT).redirectError(Constants.STDERR).launch();
            }
        };
        YarnWeaveController controller = controllerFactory.create(runId, logHandlers, submitTask);
        controller.start();
        return controller;
    } catch (Exception e) {
        LOG.error("Failed to submit application {}", weaveSpec.getName(), e);
        throw Throwables.propagate(e);
    }
}