Examples with HostSpec com.yahoo.config.provision.HostSpec used on opensource projects

Example 26 with HostSpec

use of com.yahoo.config.provision.HostSpec in project vespa by vespa-engine.

the class ProvisioningTest method assertCorrectFlavorPreferences.

private void assertCorrectFlavorPreferences(boolean largeIsStock) {
    FlavorConfigBuilder b = new FlavorConfigBuilder();
    b.addFlavor("large", 4., 8., 100, Flavor.Type.BARE_METAL).cost(10).stock(largeIsStock);
    FlavorsConfig.Flavor.Builder largeVariant = b.addFlavor("large-variant", 3., 9., 101, Flavor.Type.BARE_METAL).cost(9);
    b.addReplaces("large", largeVariant);
    FlavorsConfig.Flavor.Builder largeVariantVariant = b.addFlavor("large-variant-variant", 4., 9., 101, Flavor.Type.BARE_METAL).cost(11);
    b.addReplaces("large-variant", largeVariantVariant);
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), b.build());
    // cost = 10
    tester.makeReadyNodes(6, "large");
    // cost = 9
    tester.makeReadyNodes(6, "large-variant");
    // cost = 11
    tester.makeReadyNodes(6, "large-variant-variant");
    ApplicationId applicationId = tester.makeApplicationId();
    ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false);
    ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false);
    List<HostSpec> containerNodes = tester.prepare(applicationId, containerClusterSpec, 5, 1, "large");
    List<HostSpec> contentNodes = tester.prepare(applicationId, contentClusterSpec, 10, 1, "large");
    if (largeIsStock) {
        // 'large' is replaced by 'large-variant' when possible, as it is cheaper
        tester.assertNumberOfNodesWithFlavor(containerNodes, "large-variant", 5);
        tester.assertNumberOfNodesWithFlavor(contentNodes, "large-variant", 1);
        tester.assertNumberOfNodesWithFlavor(contentNodes, "large", 6);
    } else {
        // 'large' is preferred when available, as it is what is exactly specified
        tester.assertNumberOfNodesWithFlavor(containerNodes, "large", 5);
        tester.assertNumberOfNodesWithFlavor(contentNodes, "large", 1);
        tester.assertNumberOfNodesWithFlavor(contentNodes, "large-variant", 6);
    }
    // in both cases the most expensive, never exactly specified is least preferred
    tester.assertNumberOfNodesWithFlavor(contentNodes, "large-variant-variant", 3);
}

Example 27 with HostSpec

use of com.yahoo.config.provision.HostSpec in project vespa by vespa-engine.

the class ProvisioningTest method application_deployment_variable_application_size.

@Test
public void application_deployment_variable_application_size() {
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
    ApplicationId application1 = tester.makeApplicationId();
    tester.makeReadyNodes(24, "default");
    // deploy
    SystemState state1 = prepare(application1, 2, 2, 3, 3, "default", tester);
    tester.activate(application1, state1.allHosts);
    // redeploy with increased sizes
    SystemState state2 = prepare(application1, 3, 4, 4, 5, "default", tester);
    state2.assertExtends(state1);
    assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size());
    tester.activate(application1, state2.allHosts);
    // decrease again
    SystemState state3 = prepare(application1, 2, 2, 3, 3, "default", tester);
    tester.activate(application1, state3.allHosts);
    assertEquals("Superfluous container nodes are deactivated", 3 - 2 + 4 - 2, tester.getNodes(application1, Node.State.inactive).size());
    assertEquals("Superfluous content nodes are retired", 4 - 3 + 5 - 3, tester.getNodes(application1, Node.State.active).retired().size());
    // increase even more, and remove one node before deploying
    SystemState state4 = prepare(application1, 4, 5, 5, 6, "default", tester);
    assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size());
    assertEquals("Earlier retired nodes are not unretired before activate", 4 - 3 + 5 - 3, tester.getNodes(application1, Node.State.active).retired().size());
    state4.assertExtends(state2);
    assertEquals("New and inactive nodes are reserved", 4 + 3, tester.getNodes(application1, Node.State.reserved).size());
    // Remove a retired host from one of the content clusters (which one is random depending on host names)
    HostSpec removed = state4.removeHost(tester.getNodes(application1, Node.State.active).retired().asList().get(0).hostname());
    tester.activate(application1, state4.allHosts);
    assertEquals("Retired active removed when activating became inactive", 1, tester.getNodes(application1, Node.State.inactive).asList().size());
    assertEquals(removed.hostname(), tester.getNodes(application1, Node.State.inactive).asList().get(0).hostname());
    assertEquals("Earlier retired nodes are unretired on activate", 0, tester.getNodes(application1, Node.State.active).retired().size());
    // decrease again
    SystemState state5 = prepare(application1, 2, 2, 3, 3, "default", tester);
    tester.activate(application1, state5.allHosts);
    assertEquals("Superfluous container nodes are also deactivated", 4 - 2 + 5 - 2 + 1, // 
    tester.getNodes(application1, Node.State.inactive).size());
    assertEquals("Superfluous content nodes are retired", 5 - 3 + 6 - 3 - 1, tester.getNodes(application1, Node.State.active).retired().size());
    // increase content slightly
    SystemState state6 = prepare(application1, 2, 2, 4, 3, "default", tester);
    tester.activate(application1, state6.allHosts);
    assertEquals("One content node is unretired", 5 - 4 + 6 - 3 - 1, tester.getNodes(application1, Node.State.active).retired().size());
    // Then reserve more
    SystemState state7 = prepare(application1, 8, 2, 2, 2, "default", tester);
    // delete app
    NestedTransaction removeTransaction = new NestedTransaction();
    tester.provisioner().remove(removeTransaction, application1);
    removeTransaction.commit();
    assertEquals(0, tester.getNodes(application1, Node.State.active).size());
    assertEquals(0, tester.getNodes(application1, Node.State.reserved).size());
}

Example 28 with HostSpec

use of com.yahoo.config.provision.HostSpec in project vespa by vespa-engine.

the class DockerProvisioningTest method get_specified_flavor_not_default_flavor_for_docker.

// In dev, test and staging you get nodes with default flavor, but we should get specified flavor for docker nodes
@Test
public void get_specified_flavor_not_default_flavor_for_docker() {
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("corp-us-east-1")));
    ApplicationId application1 = tester.makeApplicationId();
    tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost");
    List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false), 1, 1, dockerFlavor);
    tester.activate(application1, new HashSet<>(hosts));
    NodeList nodes = tester.getNodes(application1, Node.State.active);
    assertEquals(1, nodes.size());
    assertEquals(dockerFlavor, nodes.asList().get(0).flavor().canonicalName());
}

Example 29 with HostSpec

use of com.yahoo.config.provision.HostSpec in project vespa by vespa-engine.

the class DockerProvisioningTest method docker_application_deployment.

@Test
public void docker_application_deployment() {
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
    ApplicationId application1 = tester.makeApplicationId();
    for (int i = 1; i < 10; i++) tester.makeReadyDockerNodes(1, dockerFlavor, "dockerHost" + i);
    Version wantedVespaVersion = Version.fromString("6.39");
    int nodeCount = 7;
    List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), nodeCount, 1, dockerFlavor);
    tester.activate(application1, new HashSet<>(hosts));
    NodeList nodes = tester.getNodes(application1, Node.State.active);
    assertEquals(nodeCount, nodes.size());
    assertEquals(dockerFlavor, nodes.asList().get(0).flavor().canonicalName());
    // Upgrade Vespa version on nodes
    Version upgradedWantedVespaVersion = Version.fromString("6.40");
    List<HostSpec> upgradedHosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false), nodeCount, 1, dockerFlavor);
    tester.activate(application1, new HashSet<>(upgradedHosts));
    NodeList upgradedNodes = tester.getNodes(application1, Node.State.active);
    assertEquals(nodeCount, upgradedNodes.size());
    assertEquals(dockerFlavor, upgradedNodes.asList().get(0).flavor().canonicalName());
    assertEquals(hosts, upgradedHosts);
}

Example 30 with HostSpec

use of com.yahoo.config.provision.HostSpec in project vespa by vespa-engine.

the class NodeTypeProvisioningTest method retire_multiple_proxy_simultaneously.

@Test
public void retire_multiple_proxy_simultaneously() {
    MockDeployer deployer = new MockDeployer(tester.provisioner(), Collections.singletonMap(application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
    RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database()));
    final int numNodesToRetire = 5;
    {
        // Deploy
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals("Reserved all proxies", 11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals("Activated all proxies", 11, nodes.size());
    }
    List<Node> nodesToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).subList(3, 3 + numNodesToRetire);
    String currentyRetiringHostname;
    {
        nodesToRetire.forEach(nodeToRetire -> tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true))));
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // The retiring node should be one of the nodes we marked for retirement
        currentyRetiringHostname = nodesCurrentlyRetiring.get(0).hostname();
        assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count() == 1);
    }
    {
        // Redeploying while the node is still retiring has no effect
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // The node that started retiring is still the only one retiring
        assertEquals(currentyRetiringHostname, nodesCurrentlyRetiring.get(0).hostname());
    }
    {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(10, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(10, nodes.size());
        // Verify the node we previously set to retire has finished retiring
        assertEquals(Node.State.inactive, tester.nodeRepository().getNode(currentyRetiringHostname).orElseThrow(RuntimeException::new).state());
        // Verify that a node is currently retiring
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // This node is different from the one that was retiring previously
        String newRetiringHostname = nodesCurrentlyRetiring.get(0).hostname();
        assertNotEquals(currentyRetiringHostname, newRetiringHostname);
        // ... but is one of the nodes that were put to wantToRetire earlier
        assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(newRetiringHostname)).count() == 1);
    }
    for (int i = 0; i < 10; i++) {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        tester.activate(application, new HashSet<>(hosts));
    }
    // After a long time, all currently active proxy nodes are not marked with wantToRetire or as retired
    long numRetiredActiveProxyNodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).stream().filter(node -> !node.status().wantToRetire()).filter(node -> !node.allocation().get().membership().retired()).count();
    assertEquals(11 - numNodesToRetire, numRetiredActiveProxyNodes);
    // All the nodes that were marked with wantToRetire earlier are now inactive
    assertEquals(nodesToRetire.stream().map(Node::hostname).collect(Collectors.toSet()), tester.nodeRepository().getNodes(Node.State.inactive).stream().map(Node::hostname).collect(Collectors.toSet()));
}