Examples with ContainerInfo com.datatorrent.stram.webapp.ContainerInfo used on opensource projects

Example 1 with ContainerInfo

use of com.datatorrent.stram.webapp.ContainerInfo in project apex-core by apache.

the class FSStatsRecorder method recordContainers.

@Override
public void recordContainers(Map<String, StreamingContainerAgent> containerMap, long timestamp) throws IOException {
    for (Map.Entry<String, StreamingContainerAgent> entry : containerMap.entrySet()) {
        StreamingContainerAgent sca = entry.getValue();
        ContainerInfo containerInfo = sca.getContainerInfo();
        if (!containerInfo.state.equals("ACTIVE")) {
            continue;
        }
        int containerIndex;
        if (!knownContainers.containsKey(entry.getKey())) {
            containerIndex = knownContainers.size();
            knownContainers.put(entry.getKey(), containerIndex);
            Map<String, Object> fieldMap = extractRecordFields(containerInfo, "meta");
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            Slice f = streamCodec.toByteArray(fieldMap);
            bos.write((String.valueOf(containerIndex) + ":").getBytes());
            bos.write(f.buffer, f.offset, f.length);
            bos.write("\n".getBytes());
            queue.add(new WriteOperation(containersStorage, bos.toByteArray(), true));
        } else {
            containerIndex = knownContainers.get(entry.getKey());
        }
        Map<String, Object> fieldMap = extractRecordFields(containerInfo, "stats");
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        Slice f = streamCodec.toByteArray(fieldMap);
        bos.write((String.valueOf(containerIndex) + ":").getBytes());
        bos.write((String.valueOf(timestamp) + ":").getBytes());
        bos.write(f.buffer, f.offset, f.length);
        bos.write("\n".getBytes());
        queue.add(new WriteOperation(containersStorage, bos.toByteArray(), false));
    }
}

Example 2 with ContainerInfo

use of com.datatorrent.stram.webapp.ContainerInfo in project apex-core by apache.

the class StreamingContainerManager method getAppMasterContainerInfo.

public final ContainerInfo getAppMasterContainerInfo() {
    ContainerInfo ci = new ContainerInfo();
    ci.id = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
    String nmHost = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
    String nmPort = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
    String nmHttpPort = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());
    ci.state = "ACTIVE";
    ci.jvmName = ManagementFactory.getRuntimeMXBean().getName();
    ci.numOperators = 0;
    YarnConfiguration conf = new YarnConfiguration();
    if (nmHost != null) {
        if (nmPort != null) {
            ci.host = nmHost + ":" + nmPort;
        }
        if (nmHttpPort != null) {
            String nodeHttpAddress = nmHost + ":" + nmHttpPort;
            if (allocatedMemoryMB == 0) {
                String url = ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/ws/v1/node/containers/" + ci.id;
                WebServicesClient webServicesClient = new WebServicesClient();
                try {
                    String content = webServicesClient.process(url, String.class, new WebServicesClient.GetWebServicesHandler<String>());
                    JSONObject json = new JSONObject(content);
                    int totalMemoryNeededMB = json.getJSONObject("container").getInt("totalMemoryNeededMB");
                    if (totalMemoryNeededMB > 0) {
                        allocatedMemoryMB = totalMemoryNeededMB;
                    } else {
                        LOG.warn("Could not determine the memory allocated for the streaming application master.  Node manager is reporting {} MB from {}", totalMemoryNeededMB, url);
                    }
                } catch (Exception ex) {
                    LOG.warn("Could not determine the memory allocated for the streaming application master", ex);
                }
            }
            ci.containerLogsUrl = ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System.getenv(ApplicationConstants.Environment.USER.toString());
            ci.rawContainerLogsUrl = ConfigUtils.getRawContainerLogsUrl(conf, nodeHttpAddress, plan.getLogicalPlan().getAttributes().get(LogicalPlan.APPLICATION_ID), ci.id);
        }
    }
    ci.memoryMBAllocated = allocatedMemoryMB;
    ci.memoryMBFree = ((int) (Runtime.getRuntime().freeMemory() / (1024 * 1024)));
    ci.lastHeartbeat = -1;
    ci.startedTime = startTime;
    ci.finishedTime = -1;
    return ci;
}

Example 3 with ContainerInfo

use of com.datatorrent.stram.webapp.ContainerInfo in project apex-core by apache.

the class StreamingContainerAgent method getContainerInfo.

public ContainerInfo getContainerInfo() {
    ContainerInfo ci = new ContainerInfo();
    ci.id = container.getExternalId();
    ci.host = container.host;
    ci.state = container.getState().name();
    ci.jvmName = this.jvmName;
    ci.numOperators = container.getOperators().size();
    ci.operators = new TreeMap<>();
    for (PTOperator ptOperator : container.getOperators()) {
        ci.operators.put(ptOperator.getId(), ptOperator.getName());
    }
    ci.memoryMBAllocated = container.getAllocatedMemoryMB();
    ci.lastHeartbeat = lastHeartbeatMillis;
    ci.memoryMBFree = this.memoryMBFree;
    ci.gcCollectionCount = this.gcCollectionCount;
    ci.gcCollectionTime = this.gcCollectionTime;
    ci.startedTime = container.getStartedTime();
    ci.finishedTime = container.getFinishedTime();
    if (this.container.nodeHttpAddress != null) {
        YarnConfiguration conf = new YarnConfiguration();
        ci.containerLogsUrl = ConfigUtils.getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System.getenv(ApplicationConstants.Environment.USER.toString());
        ci.rawContainerLogsUrl = ConfigUtils.getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes().get(LogicalPlan.APPLICATION_ID), ci.id);
    }
    return ci;
}