Examples with Node com.yahoo.vespa.hosted.provision.Node used on opensource projects

Example 91 with Node

use of com.yahoo.vespa.hosted.provision.Node in project vespa by vespa-engine.

the class Preparer method prepare.

/**
 * Ensure sufficient nodes are reserved or active for the given application and cluster
 *
 * @return the list of nodes this cluster will have allocated if activated
 */
// Note: This operation may make persisted changes to the set of reserved and inactive nodes,
// but it may not change the set of active nodes, as the active nodes must stay in sync with the
// active config model which is changed on activate
public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) {
    List<Node> surplusNodes = findNodesInRemovableGroups(application, cluster, wantedGroups);
    MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster));
    List<Node> acceptedNodes = new ArrayList<>();
    for (int groupIndex = 0; groupIndex < wantedGroups; groupIndex++) {
        ClusterSpec clusterGroup = cluster.with(Optional.of(ClusterSpec.Group.from(groupIndex)));
        List<Node> accepted = groupPreparer.prepare(application, clusterGroup, requestedNodes.fraction(wantedGroups), surplusNodes, highestIndex, spareCount);
        replace(acceptedNodes, accepted);
    }
    moveToActiveGroup(surplusNodes, wantedGroups, cluster.group());
    replace(acceptedNodes, retire(surplusNodes));
    return acceptedNodes;
}

Example 92 with Node

use of com.yahoo.vespa.hosted.provision.Node in project vespa by vespa-engine.

the class NodeAclResponse method toSlime.

private void toSlime(String hostname, Cursor object) {
    Node node = nodeRepository.getNode(hostname).orElseGet(() -> nodeRepository.getConfigNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'")));
    Cursor trustedNodesArray = object.setArray("trustedNodes");
    nodeRepository.getNodeAcls(node, aclsForChildren).forEach(nodeAcl -> toSlime(nodeAcl, trustedNodesArray));
    Cursor trustedNetworksArray = object.setArray("trustedNetworks");
    nodeRepository.getNodeAcls(node, aclsForChildren).forEach(nodeAcl -> toSlime(nodeAcl.trustedNetworks(), nodeAcl.node(), trustedNetworksArray));
}

Example 93 with Node

use of com.yahoo.vespa.hosted.provision.Node in project vespa by vespa-engine.

the class NodesResponse method toSlime.

private void toSlime(Node node, boolean allFields, Cursor object) {
    object.setString("url", nodeParentUrl + node.hostname());
    if (!allFields)
        return;
    object.setString("id", node.id());
    object.setString("state", NodeStateSerializer.wireNameOf(node.state()));
    object.setString("type", node.type().name());
    object.setString("hostname", node.hostname());
    object.setString("type", toString(node.type()));
    if (node.parentHostname().isPresent()) {
        object.setString("parentHostname", node.parentHostname().get());
    }
    object.setString("openStackId", node.openStackId());
    object.setString("flavor", node.flavor().name());
    object.setString("canonicalFlavor", node.flavor().canonicalName());
    object.setDouble("minDiskAvailableGb", node.flavor().getMinDiskAvailableGb());
    object.setDouble("minMainMemoryAvailableGb", node.flavor().getMinMainMemoryAvailableGb());
    if (node.flavor().getDescription() != null && !node.flavor().getDescription().isEmpty())
        object.setString("description", node.flavor().getDescription());
    object.setDouble("minCpuCores", node.flavor().getMinCpuCores());
    if (node.flavor().cost() > 0)
        object.setLong("cost", node.flavor().cost());
    object.setBool("fastDisk", node.flavor().hasFastDisk());
    object.setString("environment", node.flavor().getType().name());
    if (node.allocation().isPresent()) {
        toSlime(node.allocation().get().owner(), object.setObject("owner"));
        toSlime(node.allocation().get().membership(), object.setObject("membership"));
        object.setLong("restartGeneration", node.allocation().get().restartGeneration().wanted());
        object.setLong("currentRestartGeneration", node.allocation().get().restartGeneration().current());
        object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(node.allocation().get().membership().cluster().vespaVersion()).asString());
        object.setString("wantedVespaVersion", node.allocation().get().membership().cluster().vespaVersion().toFullString());
        try {
            object.setBool("allowedToBeDown", orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN);
        } catch (HostNameNotFoundException e) {
        /* ok */
        }
    }
    object.setLong("rebootGeneration", node.status().reboot().wanted());
    object.setLong("currentRebootGeneration", node.status().reboot().current());
    node.status().vespaVersion().filter(version -> !version.isEmpty()).ifPresent(version -> {
        object.setString("vespaVersion", version.toFullString());
        object.setString("currentDockerImage", nodeRepository.dockerImage().withTag(version).asString());
        // TODO: Remove these when they are no longer read
        object.setString("hostedVersion", version.toFullString());
        object.setString("convergedStateVersion", version.toFullString());
    });
    object.setLong("failCount", node.status().failCount());
    object.setBool("hardwareFailure", node.status().hardwareFailureDescription().isPresent());
    node.status().hardwareFailureDescription().ifPresent(failure -> object.setString("hardwareFailureDescription", failure));
    object.setBool("wantToRetire", node.status().wantToRetire());
    object.setBool("wantToDeprovision", node.status().wantToDeprovision());
    toSlime(node.history(), object.setArray("history"));
    ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses"));
    ipAddressesToSlime(node.additionalIpAddresses(), object.setArray("additionalIpAddresses"));
    node.status().hardwareDivergence().ifPresent(hardwareDivergence -> object.setString("hardwareDivergence", hardwareDivergence));
}

Example 94 with Node

use of com.yahoo.vespa.hosted.provision.Node in project vespa by vespa-engine.

the class ServiceMonitorStub method getAllApplicationInstances.

@Override
public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
    // Convert apps information to the response payload to return
    Map<ApplicationInstanceReference, ApplicationInstance> status = new HashMap<>();
    for (Map.Entry<ApplicationId, MockDeployer.ApplicationContext> app : apps.entrySet()) {
        Set<ServiceInstance> serviceInstances = new HashSet<>();
        for (Node node : nodeRepository.getNodes(app.getValue().id(), Node.State.active)) {
            serviceInstances.add(new ServiceInstance(new ConfigId("configid"), new HostName(node.hostname()), getHostStatus(node.hostname())));
        }
        Set<ServiceCluster> serviceClusters = new HashSet<>();
        serviceClusters.add(new ServiceCluster(new ClusterId(app.getValue().clusterContexts().get(0).cluster().id().value()), new ServiceType("serviceType"), serviceInstances));
        TenantId tenantId = new TenantId(app.getKey().tenant().value());
        ApplicationInstanceId applicationInstanceId = new ApplicationInstanceId(app.getKey().application().value());
        status.put(new ApplicationInstanceReference(tenantId, applicationInstanceId), new ApplicationInstance(tenantId, applicationInstanceId, serviceClusters));
    }
    return status;
}

Example 95 with Node

use of com.yahoo.vespa.hosted.provision.Node in project vespa by vespa-engine.

the class NodeAgentImpl method converge.

// Public for testing
void converge() {
    final Optional<ContainerNodeSpec> nodeSpecOptional = nodeRepository.getContainerNodeSpec(hostname);
    // We just removed the node from node repo, so this is expected until NodeAdmin stop this NodeAgent
    if (!nodeSpecOptional.isPresent() && expectNodeNotInNodeRepo)
        return;
    final ContainerNodeSpec nodeSpec = nodeSpecOptional.orElseThrow(() -> new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname)));
    expectNodeNotInNodeRepo = false;
    Optional<Container> container = getContainer();
    if (!nodeSpec.equals(lastNodeSpec)) {
        // will change and we will be reporting duplicate metrics.
        if (container.map(c -> c.state.isRunning()).orElse(false)) {
            storageMaintainer.writeMetricsConfig(containerName, nodeSpec);
        }
        addDebugMessage("Loading new node spec: " + nodeSpec.toString());
        lastNodeSpec = nodeSpec;
    }
    switch(nodeSpec.nodeState) {
        case ready:
        case reserved:
        case parked:
        case failed:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            break;
        case active:
            storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false);
            storageMaintainer.getDiskUsageFor(containerName).map(diskUsage -> (double) diskUsage / BYTES_IN_GB / nodeSpec.minDiskAvailableGb).filter(diskUtil -> diskUtil >= 0.8).ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(containerName));
            scheduleDownLoadIfNeeded(nodeSpec);
            if (isDownloadingImage()) {
                addDebugMessage("Waiting for image to download " + imageBeingDownloaded.asString());
                return;
            }
            container = removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            if (!container.isPresent()) {
                storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false);
                containerState = STARTING;
                startContainer(nodeSpec);
                containerState = UNKNOWN;
            }
            runLocalResumeScriptIfNeeded(nodeSpec);
            // Because it's more important to stop a bad release from rolling out in prod,
            // we put the resume call last. So if we fail after updating the node repo attributes
            // but before resume, the app may go through the tenant pipeline but will halt in prod.
            // 
            // Note that this problem exists only because there are 2 different mechanisms
            // that should really be parts of a single mechanism:
            // - The content of node repo is used to determine whether a new Vespa+application
            // has been successfully rolled out.
            // - Slobrok and internal orchestrator state is used to determine whether
            // to allow upgrade (suspend).
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            logger.info("Call resume against Orchestrator");
            orchestrator.resume(hostname);
            break;
        case inactive:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            break;
        case provisioned:
            nodeRepository.markAsDirty(hostname);
            break;
        case dirty:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready");
            storageMaintainer.cleanupNodeStorage(containerName, nodeSpec);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            nodeRepository.markNodeAvailableForNewAllocation(hostname);
            expectNodeNotInNodeRepo = true;
            break;
        default:
            throw new RuntimeException("UNKNOWN STATE " + nodeSpec.nodeState.name());
    }
}