Examples with ApiClusterTemplateService com.cloudera.api.swagger.model.ApiClusterTemplateService used on opensource projects

Example 36 with ApiClusterTemplateService

use of com.cloudera.api.swagger.model.ApiClusterTemplateService in project cloudbreak by hortonworks.

the class CmHostGroupRoleConfigProviderProcessor method updateConfigsInTemplate.

private void updateConfigsInTemplate(CmTemplateProcessor templateProcessor, Map<String, Map<String, List<ApiClusterTemplateConfig>>> newConfigsByRCG) {
    List<ApiClusterTemplateHostTemplate> hostTemplates = getHostTemplates(templateProcessor);
    Map<String, ApiClusterTemplateService> serviceByRCG = templateProcessor.getTemplate().getServices().stream().flatMap(service -> ofNullable(service.getRoleConfigGroups()).orElseGet(List::of).stream().map(rcg -> Pair.of(rcg, service))).collect(toMap(pair -> pair.getLeft().getRefName(), Pair::getRight));
    Map<String, ApiClusterTemplateRoleConfigGroup> roleConfigGroupByName = templateProcessor.getTemplate().getServices().stream().flatMap(service -> ofNullable(service.getRoleConfigGroups()).orElseGet(List::of).stream()).collect(toMap(ApiClusterTemplateRoleConfigGroup::getRefName, Function.identity()));
    ApiClusterTemplateInstantiator instantiator = templateProcessor.getTemplate().getInstantiator();
    List<ApiClusterTemplateRoleConfigGroupInfo> instantiatorRoleConfigGroups = instantiator.getRoleConfigGroups();
    for (Map.Entry<String, Map<String, List<ApiClusterTemplateConfig>>> entry : newConfigsByRCG.entrySet()) {
        String configGroupName = entry.getKey();
        ApiClusterTemplateRoleConfigGroup configGroup = roleConfigGroupByName.get(configGroupName);
        ApiClusterTemplateService templateService = serviceByRCG.get(configGroupName);
        Map<String, List<ApiClusterTemplateConfig>> configsByHostGroup = entry.getValue();
        int groupCount = configsByHostGroup.size();
        Optional<CmHostGroupRoleConfigProvider> provider = providers.stream().filter(it -> it.getServiceType().equals(templateService.getServiceType())).findFirst();
        boolean sharedRoleType = true;
        if (provider.isPresent()) {
            sharedRoleType = provider.get().sharedRoleType(configGroup.getRoleType());
        }
        if (groupCount == 1 || sharedRoleType) {
            templateProcessor.mergeRoleConfigs(configGroup, configsByHostGroup.values().iterator().next());
        } else if (groupCount > 1) {
            LOGGER.debug("Cloning config group {} into {} host groups: {}", configGroupName, groupCount, configsByHostGroup.keySet());
            // "clone" config group for each host group
            Map<String, ApiClusterTemplateRoleConfigGroup> clonesByHostGroup = configsByHostGroup.keySet().stream().map(hostGroupName -> Pair.of(hostGroupName, copyForHostGroup(configGroup, hostGroupName))).collect(toMap(Pair::getKey, Pair::getValue));
            // add host group-specific configs to clones
            configsByHostGroup.forEach((hostGroupName, newConfigs) -> templateProcessor.mergeRoleConfigs(clonesByHostGroup.get(hostGroupName), newConfigs));
            // remove original from service
            ApiClusterTemplateService service = serviceByRCG.get(configGroupName);
            service.getRoleConfigGroups().removeIf(group -> Objects.equals(group.getRefName(), configGroupName));
            // remove original from instantiator
            if (instantiatorRoleConfigGroups != null) {
                instantiatorRoleConfigGroups.removeIf(groupInfo -> Objects.equals(groupInfo.getRcgRefName(), configGroupName));
            }
            // add clones to service
            service.getRoleConfigGroups().addAll(clonesByHostGroup.values());
            // add clones to instantiator
            clonesByHostGroup.values().forEach(group -> instantiator.addRoleConfigGroupsItem(new ApiClusterTemplateRoleConfigGroupInfo().rcgRefName(group.getRefName())));
            // replace references in host groups
            for (ApiClusterTemplateHostTemplate hostTemplate : hostTemplates) {
                ApiClusterTemplateRoleConfigGroup rcgForHostGroup = clonesByHostGroup.get(hostTemplate.getRefName());
                if (rcgForHostGroup != null) {
                    hostTemplate.getRoleConfigGroupsRefNames().remove(configGroupName);
                    hostTemplate.getRoleConfigGroupsRefNames().add(rcgForHostGroup.getRefName());
                }
            }
        }
    }
}

Example 37 with ApiClusterTemplateService

use of com.cloudera.api.swagger.model.ApiClusterTemplateService in project cloudbreak by hortonworks.

the class CmTemplateProcessorTest method testIfCustomRoleConfigsAreMerged.

@Test
public void testIfCustomRoleConfigsAreMerged() {
    underTest = new CmTemplateProcessor(getBlueprintText("input/de-ha.bp"));
    // present in cluster template/blueprint
    List<ApiClusterTemplateConfig> hs2RoleConfigs = List.of(new ApiClusterTemplateConfig().name("hiveserver2_mv_files_thread").value("30"));
    List<ApiClusterTemplateRoleConfigGroup> hs2Rcg = List.of(new ApiClusterTemplateRoleConfigGroup().roleType("hiveserver2").configs(hs2RoleConfigs));
    // not present in cluster template/blueprint
    List<ApiClusterTemplateConfig> gatewayRoleConfigs = List.of(new ApiClusterTemplateConfig().name("hive_client_java_heapsize").value("6442450944"));
    List<ApiClusterTemplateRoleConfigGroup> gatewayRcg = List.of(new ApiClusterTemplateRoleConfigGroup().roleType("gateway").configs(gatewayRoleConfigs));
    ApiClusterTemplateService hive = underTest.getTemplate().getServices().stream().filter(service -> "HIVE_ON_TEZ".equals(service.getServiceType())).findFirst().get();
    List<ApiClusterTemplateConfig> existingGatewayConfigs = hive.getRoleConfigGroups().get(0).getConfigs();
    List<ApiClusterTemplateConfig> existingHs2Configs = hive.getRoleConfigGroups().get(1).getConfigs();
    underTest.mergeCustomRoleConfigs(hive, hs2Rcg);
    underTest.mergeCustomRoleConfigs(hive, gatewayRcg);
    assertEquals(existingHs2Configs.size(), hive.getRoleConfigGroups().get(1).getConfigs().size());
    assertEquals((existingGatewayConfigs == null ? 0 : existingGatewayConfigs.size()) + gatewayRoleConfigs.size(), hive.getRoleConfigGroups().get(0).getConfigs().size());
}

Example 38 with ApiClusterTemplateService

use of com.cloudera.api.swagger.model.ApiClusterTemplateService in project cloudbreak by hortonworks.

the class CmTemplateProcessorTest method testAddServiceConfigs.

@Test
public void testAddServiceConfigs() {
    underTest = new CmTemplateProcessor(getBlueprintText("input/clouderamanager.bp"));
    List<ApiClusterTemplateConfig> configs = new ArrayList<>();
    configs.add(new ApiClusterTemplateConfig().name("hive_metastore_database_type").variable("hive-hive_metastore_database_type"));
    underTest.addServiceConfigs("HIVE", List.of("HIVEMETASTORE"), configs);
    ApiClusterTemplateService service = underTest.getTemplate().getServices().stream().filter(srv -> "HIVE".equals(srv.getServiceType())).findAny().get();
    List<ApiClusterTemplateConfig> serviceConfigs = service.getServiceConfigs();
    assertEquals(1, serviceConfigs.size());
    assertEquals("hive_metastore_database_type", serviceConfigs.get(0).getName());
    assertEquals("hive-hive_metastore_database_type", serviceConfigs.get(0).getVariable());
}

Example 39 with ApiClusterTemplateService

use of com.cloudera.api.swagger.model.ApiClusterTemplateService in project cloudbreak by hortonworks.

the class CmTemplateProcessorTest method addExistingSafetyValveConfigs.

@Test
public void addExistingSafetyValveConfigs() {
    underTest = new CmTemplateProcessor(getBlueprintText("input/clouderamanager-existing-conf.bp"));
    List<ApiClusterTemplateConfig> configs = new ArrayList<>();
    configs.add(new ApiClusterTemplateConfig().name("hive_service_config_safety_valve").value("<property><name>testkey</name><value>testvalue</value></property>"));
    underTest.addServiceConfigs("HIVE", List.of("GATEWAY", "HIVEMETASTORE"), configs);
    ApiClusterTemplateService service = underTest.getTemplate().getServices().stream().filter(srv -> "HIVE".equals(srv.getServiceType())).findAny().get();
    List<ApiClusterTemplateConfig> serviceConfigs = service.getServiceConfigs();
    assertEquals(1, serviceConfigs.size());
    assertEquals("hive_service_config_safety_valve", serviceConfigs.get(0).getName());
    assertTrue(serviceConfigs.get(0).getValue().startsWith("<property><name>testkey</name><value>testvalue</value></property>"));
    assertTrue(serviceConfigs.get(0).getValue().endsWith("<property><name>hive.metastore.server.filter.enabled</name><value>true</value></property> " + "<property><name>hive.metastore.filter.hook</name>" + "<value>org.apache.hadoop.hive.ql.security.authorization.plugin.metastore.HiveMetaStoreAuthorizer</value></property>"));
    assertTrue(serviceConfigs.get(0).getValue().contains("\n"));
    Map<String, List<ApiClusterTemplateConfig>> roleConfigs = new HashMap<>();
    roleConfigs.put("spark_on_yarn-GATEWAY-BASE", List.of(new ApiClusterTemplateConfig().name("spark-conf/spark-defaults.conf_client_config_safety_valve").value("testkey=testvalue")));
    underTest.addRoleConfigs("SPARK_ON_YARN", roleConfigs);
    service = underTest.getTemplate().getServices().stream().filter(srv -> "SPARK_ON_YARN".equals(srv.getServiceType())).findAny().get();
    ApiClusterTemplateRoleConfigGroup gw = service.getRoleConfigGroups().stream().filter(rcg -> "GATEWAY".equals(rcg.getRoleType())).findAny().get();
    List<ApiClusterTemplateConfig> gwConfigs = gw.getConfigs();
    assertEquals(1, gwConfigs.size());
    assertEquals("spark-conf/spark-defaults.conf_client_config_safety_valve", gwConfigs.get(0).getName());
    assertTrue(gwConfigs.get(0).getValue().startsWith("testkey=testvalue"));
    assertTrue(gwConfigs.get(0).getValue().endsWith("spark.yarn.access.hadoopFileSystems=s3a://expn-cis-sandbox-prod-cdp-us-east-1"));
    assertTrue(gwConfigs.get(0).getValue().contains("\n"));
}

Example 40 with ApiClusterTemplateService

use of com.cloudera.api.swagger.model.ApiClusterTemplateService in project cloudbreak by hortonworks.

the class CmTemplateProcessorTest method addExistingRoleConfigs.

@Test
public void addExistingRoleConfigs() {
    underTest = new CmTemplateProcessor(getBlueprintText("input/clouderamanager-existing-conf.bp"));
    Map<String, List<ApiClusterTemplateConfig>> configs = new HashMap<>();
    configs.put("hdfs-NAMENODE-BASE", List.of(new ApiClusterTemplateConfig().name("dfs_name_dir_list").variable("master_NAMENODE")));
    configs.put("hdfs-DATANODE-BASE", List.of(new ApiClusterTemplateConfig().name("dfs_data_dir_list").variable("worker_DATANODE"), new ApiClusterTemplateConfig().name("dfs_data_dir_list_2").variable("worker_DATANODE_2")));
    underTest.addRoleConfigs("HDFS", configs);
    ApiClusterTemplateService service = underTest.getTemplate().getServices().stream().filter(srv -> "HDFS".equals(srv.getServiceType())).findAny().get();
    ApiClusterTemplateRoleConfigGroup dn = service.getRoleConfigGroups().stream().filter(rcg -> "DATANODE".equals(rcg.getRoleType())).findAny().get();
    List<ApiClusterTemplateConfig> dnConfigs = dn.getConfigs();
    ApiClusterTemplateRoleConfigGroup nn = service.getRoleConfigGroups().stream().filter(rcg -> "NAMENODE".equals(rcg.getRoleType())).findAny().get();
    List<ApiClusterTemplateConfig> nnConfigs = nn.getConfigs();
    assertEquals(1, nnConfigs.size());
    assertEquals(4, dnConfigs.size());
    assertEquals("dfs_name_dir_list", nnConfigs.get(0).getName());
    ApiClusterTemplateConfig dfs1Config = dnConfigs.stream().filter(c -> "dfs_data_dir_list".equals(c.getName())).findFirst().get();
    assertEquals("/dfs/dn", dfs1Config.getValue());
    assertNull(dfs1Config.getVariable());
    ApiClusterTemplateConfig dfs2Config = dnConfigs.stream().filter(c -> "dfs_data_dir_list_2".equals(c.getName())).findFirst().get();
    assertEquals("worker_DATANODE_2", dfs2Config.getVariable());
    assertNull(dfs2Config.getValue());
}