Examples with Capacity com.yahoo.config.provision.Capacity used on opensource projects

Example 1 with Capacity

use of com.yahoo.config.provision.Capacity in project vespa by vespa-engine.

the class NodeFailTester method withTwoApplicationsOnDocker.

public static NodeFailTester withTwoApplicationsOnDocker(int numberOfHosts) {
    NodeFailTester tester = new NodeFailTester();
    int nodesPerHost = 3;
    List<Node> hosts = tester.createHostNodes(numberOfHosts);
    for (int i = 0; i < hosts.size(); i++) {
        tester.createReadyNodes(nodesPerHost, i * nodesPerHost, Optional.of("parent" + i), nodeFlavors.getFlavorOrThrow("docker"), NodeType.tenant);
    }
    // Create applications
    ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false);
    ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false);
    ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false);
    Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host);
    Capacity capacity1 = Capacity.fromNodeCount(3, Optional.of("docker"), false);
    Capacity capacity2 = Capacity.fromNodeCount(5, Optional.of("docker"), false);
    tester.activate(nodeAdminApp, clusterNodeAdminApp, allHosts);
    tester.activate(app1, clusterApp1, capacity1);
    tester.activate(app2, clusterApp2, capacity2);
    assertEquals(new HashSet<>(tester.nodeRepository.getNodes(NodeType.host)), new HashSet<>(tester.nodeRepository.getNodes(nodeAdminApp, Node.State.active)));
    assertEquals(capacity1.nodeCount(), tester.nodeRepository.getNodes(app1, Node.State.active).size());
    assertEquals(capacity2.nodeCount(), tester.nodeRepository.getNodes(app2, Node.State.active).size());
    Map<ApplicationId, MockDeployer.ApplicationContext> apps = new HashMap<>();
    apps.put(nodeAdminApp, new MockDeployer.ApplicationContext(nodeAdminApp, clusterNodeAdminApp, allHosts, 1));
    apps.put(app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1));
    apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1));
    tester.deployer = new MockDeployer(tester.provisioner, apps);
    tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
    tester.metric = new MetricsReporterTest.TestMetric();
    tester.failer = tester.createFailer();
    return tester;
}

Example 2 with Capacity

use of com.yahoo.config.provision.Capacity in project vespa by vespa-engine.

the class NodeTypeProvisioningTest method retire_proxy.

@Test
public void retire_proxy() {
    MockDeployer deployer = new MockDeployer(tester.provisioner(), Collections.singletonMap(application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
    RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database()));
    {
        // Deploy
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals("Reserved all proxies", 11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals("Activated all proxies", 11, nodes.size());
    }
    Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5);
    {
        // Pick out a node and retire it
        tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)));
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that wantToRetire has been propagated
        assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()).flatMap(Node::allocation).map(allocation -> allocation.membership().retired()).orElseThrow(RuntimeException::new));
    }
    {
        // Redeploying while the node is still retiring has no effect
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(11, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(11, nodes.size());
        // Verify that the node is still marked as retired
        assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()).flatMap(Node::allocation).map(allocation -> allocation.membership().retired()).orElseThrow(RuntimeException::new));
    }
    {
        tester.advanceTime(Duration.ofMinutes(11));
        retiredExpirer.run();
        List<HostSpec> hosts = deployProxies(application, tester);
        assertEquals(10, hosts.size());
        tester.activate(application, new HashSet<>(hosts));
        List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active);
        assertEquals(10, nodes.size());
        // Verify that the node is now inactive
        assertEquals(Node.State.inactive, tester.nodeRepository().getNode(nodeToRetire.hostname()).orElseThrow(RuntimeException::new).state());
    }
}

Example 3 with Capacity

use of com.yahoo.config.provision.Capacity in project vespa by vespa-engine.

the class VespaModelFactoryTest method hostedVespaZoneApplicationAllocatesNodesFromNodeRepo.

@Test
public void hostedVespaZoneApplicationAllocatesNodesFromNodeRepo() {
    String hostName = "test-host-name";
    String routingClusterName = "routing-cluster";
    String hosts = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<hosts>\n" + "  <host name='" + hostName + "'>\n" + "    <alias>proxy1</alias>\n" + "  </host>\n" + "</hosts>";
    String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services version='1.0' xmlns:deploy='vespa'>\n" + "    <admin version='2.0'>\n" + "        <adminserver hostalias='proxy1' />\n" + "    </admin>" + "    <jdisc id='" + routingClusterName + "' version='1.0'>\n" + "        <nodes type='proxy'/>\n" + "    </jdisc>\n" + "</services>";
    HostProvisioner provisionerToOverride = new HostProvisioner() {

        @Override
        public HostSpec allocateHost(String alias) {
            return new HostSpec(hostName, Collections.emptyList(), ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName), ClusterSpec.Group.from(0), Version.fromString("6.42"), false), 0));
        }

        @Override
        public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
            return Collections.singletonList(new HostSpec(hostName, Collections.emptyList(), ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName), ClusterSpec.Group.from(0), Version.fromString("6.42"), false), 0)));
        }
    };
    ModelContext modelContext = createMockModelContext(hosts, services, provisionerToOverride);
    Model model = new VespaModelFactory(new NullConfigModelRegistry()).createModel(modelContext);
    List<HostInfo> allocatedHosts = new ArrayList<>(model.getHosts());
    assertThat(allocatedHosts.size(), is(1));
    HostInfo hostInfo = allocatedHosts.get(0);
    assertThat(hostInfo.getHostname(), is(hostName));
    assertTrue("Routing service should run on host " + hostName, hostInfo.getServices().stream().map(ServiceInfo::getConfigId).anyMatch(configId -> configId.contains(routingClusterName)));
}

Example 4 with Capacity

use of com.yahoo.config.provision.Capacity in project vespa by vespa-engine.

the class ProvisioningTest method prepare.

private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, boolean required, String flavor, Version wantedVersion, ProvisioningTester tester) {
    // "deploy prepare" with a two container clusters and a storage cluster having of two groups
    ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false);
    ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false);
    ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false);
    ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false);
    Set<HostSpec> container0 = prepare(application, containerCluster0, container0Size, 1, required, flavor, tester);
    Set<HostSpec> container1 = prepare(application, containerCluster1, container1Size, 1, required, flavor, tester);
    Set<HostSpec> content0 = prepare(application, contentCluster0, content0Size, 1, required, flavor, tester);
    Set<HostSpec> content1 = prepare(application, contentCluster1, content1Size, 1, required, flavor, tester);
    Set<HostSpec> allHosts = new HashSet<>();
    allHosts.addAll(container0);
    allHosts.addAll(container1);
    allHosts.addAll(content0);
    allHosts.addAll(content1);
    Function<Integer, Capacity> capacity = count -> Capacity.fromNodeCount(count, Optional.empty(), required);
    int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size));
    int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size));
    int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size));
    int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size));
    assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct", expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size, tester.nonRetired(allHosts).size());
    // Check cluster/group sizes
    assertEquals(expectedContainer0Size, tester.nonRetired(container0).size());
    assertEquals(expectedContainer1Size, tester.nonRetired(container1).size());
    assertEquals(expectedContent0Size, tester.nonRetired(content0).size());
    assertEquals(expectedContent1Size, tester.nonRetired(content1).size());
    // Check cluster membership
    tester.assertMembersOf(containerCluster0, container0);
    tester.assertMembersOf(containerCluster1, container1);
    tester.assertMembersOf(contentCluster0, content0);
    tester.assertMembersOf(contentCluster1, content1);
    return new SystemState(allHosts, container0, container1, content0, content1);
}

Example 5 with Capacity

use of com.yahoo.config.provision.Capacity in project vespa by vespa-engine.

the class ProvisioningTest method application_deployment_extends_existing_reservations_on_deploy.

@Test
public void application_deployment_extends_existing_reservations_on_deploy() {
    ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")));
    ApplicationId application = tester.makeApplicationId();
    tester.makeReadyNodes(2, "default");
    // Deploy fails with out of capacity
    try {
        prepare(application, 2, 0, 2, 0, "default", tester);
        fail("Expected exception");
    } catch (OutOfCapacityException ignored) {
    }
    assertEquals("Reserved a subset of required nodes", 2, tester.getNodes(application, Node.State.reserved).size());
    // Enough nodes become available
    tester.makeReadyNodes(2, "default");
    // Deploy is retried after a few minutes
    tester.clock().advance(Duration.ofMinutes(2));
    SystemState state = prepare(application, 2, 0, 2, 0, "default", tester);
    List<Node> reserved = tester.getNodes(application, Node.State.reserved).asList();
    assertEquals("Reserved required nodes", 4, reserved.size());
    assertTrue("Time of event is updated for all nodes", reserved.stream().allMatch(n -> n.history().event(History.Event.Type.reserved).get().at().equals(tester.clock().instant())));
    // Over 10 minutes pass since first reservation. First set of reserved nodes are not expired
    tester.clock().advance(Duration.ofMinutes(8).plus(Duration.ofSeconds(1)));
    ReservationExpirer expirer = new ReservationExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database()));
    expirer.run();
    assertEquals("Nodes remain reserved", 4, tester.getNodes(application, Node.State.reserved).size());
    tester.activate(application, state.allHosts);
    assertEquals(4, tester.getNodes(application, Node.State.active).size());
}