Examples with YarnContainerInfo com.continuuity.weave.internal.yarn.YarnContainerInfo used on opensource projects

Example 1 with YarnContainerInfo

use of com.continuuity.weave.internal.yarn.YarnContainerInfo in project weave by continuuity.

the class ApplicationMasterService method launchRunnable.

/**
   * Launches runnables in the provisioned containers.
   */
private void launchRunnable(List<ProcessLauncher<YarnContainerInfo>> launchers, Queue<ProvisionRequest> provisioning) {
    for (ProcessLauncher<YarnContainerInfo> processLauncher : launchers) {
        LOG.info("Got container {}", processLauncher.getContainerInfo().getId());
        ProvisionRequest provisionRequest = provisioning.peek();
        if (provisionRequest == null) {
            continue;
        }
        String runnableName = provisionRequest.getRuntimeSpec().getName();
        LOG.info("Starting runnable {} with {}", runnableName, processLauncher);
        int containerCount = expectedContainers.getExpected(runnableName);
        ProcessLauncher.PrepareLaunchContext launchContext = processLauncher.prepareLaunch(ImmutableMap.<String, String>builder().put(EnvKeys.WEAVE_APP_DIR, System.getenv(EnvKeys.WEAVE_APP_DIR)).put(EnvKeys.WEAVE_FS_USER, System.getenv(EnvKeys.WEAVE_FS_USER)).put(EnvKeys.WEAVE_APP_RUN_ID, runId.getId()).put(EnvKeys.WEAVE_APP_NAME, weaveSpec.getName()).put(EnvKeys.WEAVE_ZK_CONNECT, zkClient.getConnectString()).put(EnvKeys.WEAVE_LOG_KAFKA_ZK, getKafkaZKConnect()).build(), getLocalizeFiles(), credentials);
        WeaveContainerLauncher launcher = new WeaveContainerLauncher(weaveSpec.getRunnables().get(runnableName), launchContext, ZKClients.namespace(zkClient, getZKNamespace(runnableName)), containerCount, jvmOpts, reservedMemory, getSecureStoreLocation());
        runningContainers.start(runnableName, processLauncher.getContainerInfo(), launcher);
        // Need to call complete to workaround bug in YARN AMRMClient
        if (provisionRequest.containerAcquired()) {
            amClient.completeContainerRequest(provisionRequest.getRequestId());
        }
        if (expectedContainers.getExpected(runnableName) == runningContainers.count(runnableName)) {
            LOG.info("Runnable " + runnableName + " fully provisioned with " + containerCount + " instances.");
            provisioning.poll();
        }
    }
}

Example 2 with YarnContainerInfo

use of com.continuuity.weave.internal.yarn.YarnContainerInfo in project weave by continuuity.

the class ApplicationMasterService method doStop.

private void doStop() throws Exception {
    // This is just to clear the interrupt flag
    Thread.interrupted();
    LOG.info("Stop application master with spec: {}", WeaveSpecificationAdapter.create().toJson(weaveSpec));
    try {
        // call event handler destroy. If there is error, only log and not affected stop sequence.
        eventHandler.destroy();
    } catch (Throwable t) {
        LOG.warn("Exception when calling {}.destroy()", weaveSpec.getEventHandler().getClassName(), t);
    }
    instanceChangeExecutor.shutdownNow();
    // For checking if all containers are stopped.
    final Set<String> ids = Sets.newHashSet(runningContainers.getContainerIds());
    YarnAMClient.AllocateHandler handler = new YarnAMClient.AllocateHandler() {

        @Override
        public void acquired(List<ProcessLauncher<YarnContainerInfo>> launchers) {
        // no-op
        }

        @Override
        public void completed(List<YarnContainerStatus> completed) {
            for (YarnContainerStatus status : completed) {
                ids.remove(status.getContainerId());
            }
        }
    };
    runningContainers.stopAll();
    // Poll for 5 seconds to wait for containers to stop.
    int count = 0;
    while (!ids.isEmpty() && count++ < 5) {
        amClient.allocate(0.0f, handler);
        TimeUnit.SECONDS.sleep(1);
    }
    LOG.info("Stopping application master tracker server");
    try {
        trackerService.stopAndWait();
        LOG.info("Stopped application master tracker server");
    } catch (Exception e) {
        LOG.error("Failed to stop tracker service.", e);
    } finally {
        try {
            // App location cleanup
            cleanupDir(URI.create(System.getenv(EnvKeys.WEAVE_APP_DIR)));
            Loggings.forceFlush();
            // Sleep a short while to let kafka clients to have chance to fetch the log
            TimeUnit.SECONDS.sleep(1);
        } finally {
            kafkaServer.stopAndWait();
            LOG.info("Kafka server stopped");
        }
    }
}