Examples with VespaModelTester com.yahoo.vespa.model.test.VespaModelTester used on opensource projects

Example 31 with VespaModelTester

use of com.yahoo.vespa.model.test.VespaModelTester in project vespa by vespa-engine.

the class ModelProvisioningTest method testUsingNodesAndGroupCountAttributesAndGettingTooFewNodes.

@Test
public void testUsingNodesAndGroupCountAttributesAndGettingTooFewNodes() {
    String services = "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + "  <admin version='3.0'>" + // Ignored
    "    <nodes count='3'/>" + "  </admin>" + "  <content version='1.0' id='bar'>" + "     <redundancy reply-after='3'>4</redundancy>" + "     <documents>" + "       <document type='type1' mode='index'/>" + "     </documents>" + "     <nodes count='24' groups='3'/>" + "     <engine><proton><searchable-copies>3</searchable-copies></proton></engine>" + "  </content>" + "</services>";
    // We only have 6 content nodes -> 3 groups with redundancy 2 in each
    int numberOfHosts = 6;
    VespaModelTester tester = new VespaModelTester();
    tester.addHosts(numberOfHosts);
    VespaModel model = tester.createModel(services, false);
    assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts));
    ContentCluster cluster = model.getContentClusters().get("bar");
    // Reduced from 3*3
    assertEquals(2 * 3, cluster.redundancy().effectiveInitialRedundancy());
    // Reduced from 3*4
    assertEquals(2 * 3, cluster.redundancy().effectiveFinalRedundancy());
    // Reduced from 3*3
    assertEquals(2 * 3, cluster.redundancy().effectiveReadyCopies());
    // Reduced from 4|4|*
    assertEquals("2|2|*", cluster.getRootGroup().getPartitions().get());
    assertEquals(0, cluster.getRootGroup().getNodes().size());
    assertEquals(3, cluster.getRootGroup().getSubgroups().size());
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(2));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0"));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(2));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(2));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/2"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(3));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/3"));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getIndex(), is("2"));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().size(), is(2));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getDistributionKey(), is(4));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getConfigId(), is("bar/storage/4"));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(1).getDistributionKey(), is(5));
    assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(1).getConfigId(), is("bar/storage/5"));
}

Example 32 with VespaModelTester

use of com.yahoo.vespa.model.test.VespaModelTester in project vespa by vespa-engine.

the class ModelProvisioningTest method testSeparateClusters.

@Test
public void testSeparateClusters() {
    String xmlWithNodes = "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + "  <container version='1.0' id='container1'>" + "     <search/>" + "     <nodes count='1'/>" + "  </container>" + "  <content version='1.0' id='content1'>" + "     <redundancy>2</redundancy>" + "     <documents>" + "       <document type='type1' mode='index'/>" + "     </documents>" + "     <nodes count='2'/>" + "   </content>" + "</services>";
    VespaModelTester tester = new VespaModelTester();
    tester.addHosts(3);
    VespaModel model = tester.createModel(xmlWithNodes, true);
    assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
    assertEquals("Nodes in container1", 1, model.getContainerClusters().get("container1").getContainers().size());
    assertEquals("Heap size for container", 60, physicalMemoryPercentage(model.getContainerClusters().get("container1")));
}

Example 33 with VespaModelTester

use of com.yahoo.vespa.model.test.VespaModelTester in project vespa by vespa-engine.

the class ModelProvisioningTest method testUsingNodesAndGroupCountAttributes.

@Test
public void testUsingNodesAndGroupCountAttributes() {
    String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + "  <admin version='4.0'/>" + "  <container version='1.0' id='foo'>" + "     <nodes count='10'/>" + "  </container>" + "  <content version='1.0' id='bar'>" + "     <redundancy>2</redundancy>" + "     <documents>" + "       <document type='type1' mode='index'/>" + "     </documents>" + "     <nodes count='27' groups='9'/>" + "  </content>" + "  <content version='1.0' id='baz'>" + "     <redundancy>1</redundancy>" + "     <documents>" + "       <document type='type1' mode='index'/>" + "     </documents>" + "     <nodes count='27' groups='27'/>" + "   </content>" + "</services>";
    int numberOfHosts = 64;
    VespaModelTester tester = new VespaModelTester();
    tester.addHosts(numberOfHosts);
    VespaModel model = tester.createModel(services, true);
    assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts));
    // Check container cluster
    assertEquals(1, model.getContainerClusters().size());
    Set<com.yahoo.vespa.model.Host> containerHosts = model.getContainerClusters().get("foo").getContainers().stream().map(Container::getHost).collect(Collectors.toSet());
    assertEquals(10, containerHosts.size());
    // Check admin clusters
    Admin admin = model.getAdmin();
    Set<com.yahoo.vespa.model.Host> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet());
    assertEquals(3, slobrokHosts.size());
    assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts));
    assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost()));
    assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size());
    assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers());
    // Check content clusters
    ContentCluster cluster = model.getContentClusters().get("bar");
    ContainerCluster clusterControllers = cluster.getClusterControllers();
    assertEquals(3, clusterControllers.getContainers().size());
    assertEquals("bar-controllers", clusterControllers.getName());
    assertEquals("default28", clusterControllers.getContainers().get(0).getHostName());
    assertEquals("default31", clusterControllers.getContainers().get(1).getHostName());
    assertEquals("default54", clusterControllers.getContainers().get(2).getHostName());
    assertEquals(0, cluster.getRootGroup().getNodes().size());
    assertEquals(9, cluster.getRootGroup().getSubgroups().size());
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0"));
    assertEquals("default54", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1"));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getConfigId(), is("bar/storage/2"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3"));
    assertEquals("default51", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5"));
    // ...
    assertEquals("default48", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
    // ...
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8"));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(0).getDistributionKey(), is(24));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(0).getConfigId(), is("bar/storage/24"));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(1).getDistributionKey(), is(25));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(1).getConfigId(), is("bar/storage/25"));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getDistributionKey(), is(26));
    assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26"));
    cluster = model.getContentClusters().get("baz");
    clusterControllers = cluster.getClusterControllers();
    assertEquals(3, clusterControllers.getContainers().size());
    assertEquals("baz-controllers", clusterControllers.getName());
    assertEquals("default01", clusterControllers.getContainers().get(0).getHostName());
    assertEquals("default02", clusterControllers.getContainers().get(1).getHostName());
    assertEquals("default27", clusterControllers.getContainers().get(2).getHostName());
    assertEquals(0, cluster.getRootGroup().getNodes().size());
    assertEquals(27, cluster.getRootGroup().getSubgroups().size());
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0"));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0));
    assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0"));
    assertEquals("default27", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName());
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1"));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1"));
    assertEquals("default26", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName());
    // ...
    assertEquals("default25", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName());
    // ...
    assertThat(cluster.getRootGroup().getSubgroups().get(26).getIndex(), is("26"));
    assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().size(), is(1));
    assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().get(0).getDistributionKey(), is(26));
    assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().get(0).getConfigId(), is("baz/storage/26"));
}

Example 34 with VespaModelTester

use of com.yahoo.vespa.model.test.VespaModelTester in project vespa by vespa-engine.

the class ModelProvisioningTest method require_that_config_override_and_explicit_proton_tuning_have_precedence_over_default_node_flavor_tuning.

@Test
public void require_that_config_override_and_explicit_proton_tuning_have_precedence_over_default_node_flavor_tuning() {
    String services = joinLines("<?xml version='1.0' encoding='utf-8' ?>", "<services>", "  <content version='1.0' id='test'>", "    <config name='vespa.config.search.core.proton'>", "      <flush><memory><maxtlssize>2000</maxtlssize></memory></flush>", "    </config>", "    <documents>", "      <document type='type1' mode='index'/>", "    </documents>", "    <nodes count='1' flavor='content-test-flavor'/>", "    <engine>", "      <proton>", "        <tuning>", "          <searchnode>", "            <flushstrategy>", "              <native>", "                <total>", "                  <maxmemorygain>1000</maxmemorygain>", "                </total>", "              </native>", "            </flushstrategy>", "          </searchnode>", "        </tuning>", "      </proton>", "    </engine>", "  </content>", "</services>");
    VespaModelTester tester = new VespaModelTester();
    tester.addHosts("default", 1);
    tester.addHosts(createFlavorFromMemoryAndDisk("content-test-flavor", 128, 100), 1);
    VespaModel model = tester.createModel(services, true, 0);
    ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch();
    ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId());
    // from config override
    assertEquals(2000, cfg.flush().memory().maxtlssize());
    // from explicit tuning
    assertEquals(1000, cfg.flush().memory().maxmemory());
    // from default node flavor tuning
    assertEquals((long) 16 * GB, cfg.flush().memory().each().maxmemory());
}

Example 35 with VespaModelTester

use of com.yahoo.vespa.model.test.VespaModelTester in project vespa by vespa-engine.

the class ModelProvisioningTest method testSlobroksClustersAreExpandedToIncludeRetiredNodes.

@Test
public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() {
    String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + "  <admin version='4.0'/>" + "  <container version='1.0' id='foo'>" + "     <nodes count='10'/>" + "  </container>" + "</services>";
    int numberOfHosts = 10;
    VespaModelTester tester = new VespaModelTester();
    tester.addHosts(numberOfHosts);
    VespaModel model = tester.createModel(services, true, "default09");
    assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts));
    // Check slobroks clusters
    assertEquals("Includes retired node", 1 + 3, model.getAdmin().getSlobroks().size());
    assertEquals("default01", model.getAdmin().getSlobroks().get(0).getHostName());
    assertEquals("default02", model.getAdmin().getSlobroks().get(1).getHostName());
    assertEquals("default10", model.getAdmin().getSlobroks().get(2).getHostName());
    assertEquals("Included in addition because it is retired", "default09", model.getAdmin().getSlobroks().get(3).getHostName());
}