Search in sources :

Example 1 with RetiredExpirer

use of com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer in project vespa by vespa-engine.

the class MultigroupProvisioningTest method test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration.

@Test
public void test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration() {
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
    ApplicationId application1 = tester.makeApplicationId();
    tester.makeReadyNodes(10, "small");
    tester.makeReadyNodes(10, "large");
    deploy(application1, 8, 1, "small", tester);
    deploy(application1, 8, 1, "large", tester);
    // Expire small nodes
    tester.advanceTime(Duration.ofDays(7));
    MockDeployer deployer = new MockDeployer(tester.provisioner(), Collections.singletonMap(application1, new MockDeployer.ApplicationContext(application1, cluster(), Capacity.fromNodeCount(8, Optional.of("large"), false), 1)));
    new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofHours(12), new JobControl(tester.nodeRepository().database())).run();
    assertEquals(8, tester.getNodes(application1, Node.State.inactive).flavor("small").size());
    deploy(application1, 8, 8, "large", tester);
}
Also used : Zone(com.yahoo.config.provision.Zone) JobControl(com.yahoo.vespa.hosted.provision.maintenance.JobControl) RetiredExpirer(com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer) ApplicationId(com.yahoo.config.provision.ApplicationId) MockDeployer(com.yahoo.vespa.hosted.provision.testutils.MockDeployer) Test(org.junit.Test)

Example 2 with RetiredExpirer

use of com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer in project vespa by vespa-engine.

the class NodeTypeProvisioningTest method retire_proxy.

@Test
public void retire_proxy() {
    MockDeployer deployer = new MockDeployer(tester.provisioner(), Collections.singletonMap(application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
    RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database()));
    {
        // Deploy
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals("Reserved all proxies", 11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals("Activated all proxies", 11, nodes.size());
    }
    Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5);
    {
        // Pick out a node and retire it
        tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)));
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()).flatMap(Node::allocation).map(allocation -> allocation.membership().retired()).orElseThrow(RuntimeException::new));
    }
    {
        // Redeploying while the node is still retiring has no effect
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that the node is still marked as retired
        assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()).flatMap(Node::allocation).map(allocation -> allocation.membership().retired()).orElseThrow(RuntimeException::new));
    }
    {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(10, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(10, nodes.size());
        // Verify that the node is now inactive
        assertEquals(Node.State.inactive, tester.nodeRepository().getNode(nodeToRetire.hostname()).orElseThrow(RuntimeException::new).state());
    }
}
Also used : Capacity(com.yahoo.config.provision.Capacity) Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) Environment(com.yahoo.config.provision.Environment) NodeType(com.yahoo.config.provision.NodeType) JobControl(com.yahoo.vespa.hosted.provision.maintenance.JobControl) ClusterSpec(com.yahoo.config.provision.ClusterSpec) HostSpec(com.yahoo.config.provision.HostSpec) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Node(com.yahoo.vespa.hosted.provision.Node) Collectors(java.util.stream.Collectors) RegionName(com.yahoo.config.provision.RegionName) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) MockDeployer(com.yahoo.vespa.hosted.provision.testutils.MockDeployer) HashSet(java.util.HashSet) RetiredExpirer(com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer) List(java.util.List) Agent(com.yahoo.vespa.hosted.provision.node.Agent) Duration(java.time.Duration) Zone(com.yahoo.config.provision.Zone) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Before(org.junit.Before) Node(com.yahoo.vespa.hosted.provision.Node) JobControl(com.yahoo.vespa.hosted.provision.maintenance.JobControl) List(java.util.List) RetiredExpirer(com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer) MockDeployer(com.yahoo.vespa.hosted.provision.testutils.MockDeployer) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 3 with RetiredExpirer

use of com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer in project vespa by vespa-engine.

the class NodeTypeProvisioningTest method retire_multiple_proxy_simultaneously.

@Test
public void retire_multiple_proxy_simultaneously() {
    MockDeployer deployer = new MockDeployer(tester.provisioner(), Collections.singletonMap(application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
    RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database()));
    final int numNodesToRetire = 5;
    {
        // Deploy
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals("Reserved all proxies", 11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals("Activated all proxies", 11, nodes.size());
    }
    List<Node> nodesToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).subList(3, 3 + numNodesToRetire);
    String currentyRetiringHostname;
    {
        nodesToRetire.forEach(nodeToRetire -> tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true))));
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // The retiring node should be one of the nodes we marked for retirement
        currentyRetiringHostname = nodesCurrentlyRetiring.get(0).hostname();
        assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count() == 1);
    }
    {
        // Redeploying while the node is still retiring has no effect
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // The node that started retiring is still the only one retiring
        assertEquals(currentyRetiringHostname, nodesCurrentlyRetiring.get(0).hostname());
    }
    {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(10, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(10, nodes.size());
        // Verify the node we previously set to retire has finished retiring
        assertEquals(Node.State.inactive, tester.nodeRepository().getNode(currentyRetiringHostname).orElseThrow(RuntimeException::new).state());
        // Verify that a node is currently retiring
        List<Node> nodesCurrentlyRetiring = nodes.stream().filter(node -> node.allocation().get().membership().retired()).collect(Collectors.toList());
        assertEquals(1, nodesCurrentlyRetiring.size());
        // This node is different from the one that was retiring previously
        String newRetiringHostname = nodesCurrentlyRetiring.get(0).hostname();
        assertNotEquals(currentyRetiringHostname, newRetiringHostname);
        // ... but is one of the nodes that were put to wantToRetire earlier
        assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(newRetiringHostname)).count() == 1);
    }
    for (int i = 0; i < 10; i++) {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        tester.activate(application, new HashSet<>(hosts));
    }
    // After a long time, all currently active proxy nodes are not marked with wantToRetire or as retired
    long numRetiredActiveProxyNodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).stream().filter(node -> !node.status().wantToRetire()).filter(node -> !node.allocation().get().membership().retired()).count();
    assertEquals(11 - numNodesToRetire, numRetiredActiveProxyNodes);
    // All the nodes that were marked with wantToRetire earlier are now inactive
    assertEquals(nodesToRetire.stream().map(Node::hostname).collect(Collectors.toSet()), tester.nodeRepository().getNodes(Node.State.inactive).stream().map(Node::hostname).collect(Collectors.toSet()));
}
Also used : Capacity(com.yahoo.config.provision.Capacity) Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) Environment(com.yahoo.config.provision.Environment) NodeType(com.yahoo.config.provision.NodeType) JobControl(com.yahoo.vespa.hosted.provision.maintenance.JobControl) ClusterSpec(com.yahoo.config.provision.ClusterSpec) HostSpec(com.yahoo.config.provision.HostSpec) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Node(com.yahoo.vespa.hosted.provision.Node) Collectors(java.util.stream.Collectors) RegionName(com.yahoo.config.provision.RegionName) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) MockDeployer(com.yahoo.vespa.hosted.provision.testutils.MockDeployer) HashSet(java.util.HashSet) RetiredExpirer(com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer) List(java.util.List) Agent(com.yahoo.vespa.hosted.provision.node.Agent) Duration(java.time.Duration) Zone(com.yahoo.config.provision.Zone) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Before(org.junit.Before) Node(com.yahoo.vespa.hosted.provision.Node) JobControl(com.yahoo.vespa.hosted.provision.maintenance.JobControl) RetiredExpirer(com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer) HostSpec(com.yahoo.config.provision.HostSpec) List(java.util.List) MockDeployer(com.yahoo.vespa.hosted.provision.testutils.MockDeployer) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

ApplicationId (com.yahoo.config.provision.ApplicationId)3 Zone (com.yahoo.config.provision.Zone)3 JobControl (com.yahoo.vespa.hosted.provision.maintenance.JobControl)3 RetiredExpirer (com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer)3 MockDeployer (com.yahoo.vespa.hosted.provision.testutils.MockDeployer)3 Test (org.junit.Test)3 Version (com.yahoo.component.Version)2 Capacity (com.yahoo.config.provision.Capacity)2 ClusterSpec (com.yahoo.config.provision.ClusterSpec)2 Environment (com.yahoo.config.provision.Environment)2 HostSpec (com.yahoo.config.provision.HostSpec)2 NodeType (com.yahoo.config.provision.NodeType)2 RegionName (com.yahoo.config.provision.RegionName)2 Node (com.yahoo.vespa.hosted.provision.Node)2 Agent (com.yahoo.vespa.hosted.provision.node.Agent)2 Duration (java.time.Duration)2 Collections (java.util.Collections)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2