Examples with ClouderaManagerRepo com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo used on opensource projects

Example 56 with ClouderaManagerRepo

use of com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo in project cloudbreak by hortonworks.

the class ClouderaManagerModificationServiceTest method testUpgradeClusterWhenPatchUpgradeAndPostUpgradeCommandIsAvailable.

@Test
void testUpgradeClusterWhenPatchUpgradeAndPostUpgradeCommandIsAvailable() throws CloudbreakException, ApiException, ClouderaManagerClientInitException {
    TestUtil.clusterComponents(cluster);
    when(clouderaManagerApiFactory.getParcelResourceApi(any())).thenReturn(parcelResourceApi);
    when(clouderaManagerApiFactory.getClustersResourceApi(any())).thenReturn(clustersResourceApi);
    when(clouderaManagerApiFactory.getClouderaManagerResourceApi(any())).thenReturn(clouderaManagerResourceApi);
    when(clouderaManagerApiFactory.getMgmtServiceResourceApi(any())).thenReturn(mgmtServiceResourceApi);
    when(clouderaManagerApiFactory.getServicesResourceApi(any())).thenReturn(servicesResourceApi);
    BigDecimal apiCommandId = new BigDecimal(200);
    // Mgmt Service restart
    ApiCommandList apiCommandList = new ApiCommandList();
    apiCommandList.setItems(new ArrayList<>());
    when(mgmtServiceResourceApi.listActiveCommands("SUMMARY")).thenReturn(apiCommandList);
    when(mgmtServiceResourceApi.restartCommand()).thenReturn(new ApiCommand().id(apiCommandId));
    when(clouderaManagerPollingServiceProvider.startPollingCmServicesRestart(stack, apiClientMock, apiCommandId)).thenReturn(success);
    // Start services if they are not running
    ApiServiceList serviceList = new ApiServiceList();
    ApiService service = new ApiService();
    service.setServiceState(ApiServiceState.STOPPED);
    serviceList.addItemsItem(service);
    ApiCommand startCommand = mock(ApiCommand.class);
    when(startCommand.getId()).thenReturn(apiCommandId);
    when(servicesResourceApi.readServices(any(), any())).thenReturn(serviceList);
    when(clustersResourceApi.startCommand(STACK_NAME)).thenReturn(startCommand);
    // Post parcel activation
    ClouderaManagerRepo clouderaManagerRepo = mock(ClouderaManagerRepo.class);
    when(clusterComponentConfigProvider.getClouderaManagerRepoDetails(CLUSTER_ID)).thenReturn(clouderaManagerRepo);
    when(clouderaManagerRepo.getVersion()).thenReturn(CLOUDERAMANAGER_VERSION_7_5_1.getVersion());
    // Restart services
    when(clustersResourceApi.listActiveCommands(stack.getName(), "SUMMARY")).thenReturn(new ApiCommandList().items(List.of()));
    when(clustersResourceApi.restartCommand(eq(stack.getName()), any(ApiRestartClusterArgs.class))).thenReturn(new ApiCommand().id(apiCommandId));
    when(clouderaManagerPollingServiceProvider.startPollingCmServicesRestart(stack, apiClientMock, apiCommandId)).thenReturn(success);
    when(clouderaManagerPollingServiceProvider.startPollingCmHostStatus(stack, apiClientMock)).thenReturn(success);
    InstanceGroup instanceGroup = new InstanceGroup();
    InstanceMetaData instanceMetaData = new InstanceMetaData();
    instanceMetaData.setDiscoveryFQDN(HOSTNAME);
    instanceGroup.setGroupName(GROUP_NAME);
    instanceGroup.setInstanceMetaData(Set.of(instanceMetaData));
    stack.setInstanceGroups(Set.of(instanceGroup));
    when(clouderaManagerApiFactory.getClustersResourceApi(any())).thenReturn(clustersResourceApi);
    ApiHostList clusterHostsRef = new ApiHostList().items(List.of(new ApiHost().hostname(HOSTNAME)));
    when(clustersResourceApi.listHosts(eq(STACK_NAME), eq(null), eq(null), eq(null))).thenReturn(clusterHostsRef);
    when(clusterComponentProvider.getClouderaManagerRepoDetails(CLUSTER_ID)).thenReturn(clouderaManagerRepo);
    when(clouderaManagerRepo.getVersion()).thenReturn(CLOUDERAMANAGER_VERSION_7_6_0.getVersion());
    when(clouderaManagerApiFactory.getHostsResourceApi(any())).thenReturn(hostResourceApi);
    Call call = mock(Call.class);
    when(hostResourceApi.addTagsAsync(eq(HOSTNAME), any(), any())).thenReturn(call);
    when(clouderaManagerApiClientProvider.getV45Client(any(), any(), any(), any())).thenReturn(apiClientMock);
    underTest.upgradeClusterRuntime(cluster.getComponents(), true, Optional.empty());
    verify(clouderaManagerPollingServiceProvider, times(1)).startPollingCmStartup(stack, apiClientMock);
    verify(clouderaManagerPollingServiceProvider, times(1)).startPollingCmHostStatus(stack, apiClientMock);
    verify(clouderaManagerParcelManagementService, times(1)).checkParcelApiAvailability(stack, apiClientMock);
    verify(clouderaManagerParcelManagementService, times(1)).setParcelRepos(any(), eq(clouderaManagerResourceApi));
    verify(clouderaManagerParcelManagementService, times(1)).refreshParcelRepos(clouderaManagerResourceApi, stack, apiClientMock);
    verify(mgmtServiceResourceApi, times(1)).listActiveCommands("SUMMARY");
    verify(mgmtServiceResourceApi, times(1)).restartCommand();
    verify(clouderaManagerPollingServiceProvider, times(2)).startPollingCmServicesRestart(stack, apiClientMock, apiCommandId);
    verify(clouderaManagerParcelManagementService, times(1)).downloadParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    verify(clouderaManagerParcelManagementService, times(1)).distributeParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    verify(clouderaManagerParcelManagementService, times(1)).activateParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    verify(eventService, times(1)).fireCloudbreakEvent(stack.getId(), UPDATE_IN_PROGRESS.name(), ResourceEvent.CLUSTER_UPGRADE_START_POST_UPGRADE);
    verify(clustersResourceApi, times(1)).startCommand(STACK_NAME);
    verify(clouderaManagerUpgradeService, times(1)).callPostRuntimeUpgradeCommand(clustersResourceApi, stack, apiClientMock);
    verify(clustersResourceApi, times(1)).restartCommand(eq(stack.getName()), any(ApiRestartClusterArgs.class));
    verify(clustersResourceApi, times(2)).listActiveCommands(stack.getName(), "SUMMARY");
    verify(clouderaManagerApiClientProvider, times(1)).getV45Client(any(), any(), any(), any());
    ArgumentCaptor<List<ApiEntityTag>> entityTagListCaptor = ArgumentCaptor.forClass(List.class);
    verify(hostResourceApi, times(1)).addTagsAsync(eq(HOSTNAME), entityTagListCaptor.capture(), any());
    assertEquals("_cldr_cm_host_template_name", entityTagListCaptor.getValue().get(0).getName());
    assertEquals(GROUP_NAME, entityTagListCaptor.getValue().get(0).getValue());
    InOrder inOrder = Mockito.inOrder(clouderaManagerPollingServiceProvider, clouderaManagerParcelManagementService, clustersResourceApi, clouderaManagerUpgradeService, clouderaManagerApiClientProvider);
    inOrder.verify(clouderaManagerPollingServiceProvider).startPollingCmStartup(stack, apiClientMock);
    inOrder.verify(clouderaManagerPollingServiceProvider).startPollingCmHostStatus(stack, apiClientMock);
    inOrder.verify(clouderaManagerParcelManagementService).checkParcelApiAvailability(stack, apiClientMock);
    inOrder.verify(clouderaManagerParcelManagementService).setParcelRepos(any(), eq(clouderaManagerResourceApi));
    inOrder.verify(clouderaManagerParcelManagementService).refreshParcelRepos(clouderaManagerResourceApi, stack, apiClientMock);
    inOrder.verify(clouderaManagerParcelManagementService).downloadParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    inOrder.verify(clouderaManagerParcelManagementService).distributeParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    inOrder.verify(clouderaManagerParcelManagementService).activateParcels(any(), eq(parcelResourceApi), eq(stack), eq(apiClientMock));
    inOrder.verify(clustersResourceApi).startCommand(STACK_NAME);
    inOrder.verify(clustersResourceApi).listActiveCommands(eq(stack.getName()), any());
    inOrder.verify(clouderaManagerApiClientProvider).getV45Client(any(), any(), any(), any());
    inOrder.verify(clouderaManagerUpgradeService).callPostRuntimeUpgradeCommand(eq(clustersResourceApi), eq(stack), eq(apiClientMock));
    inOrder.verify(clustersResourceApi).restartCommand(eq(stack.getName()), any(ApiRestartClusterArgs.class));
}

Example 57 with ClouderaManagerRepo

use of com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo in project cloudbreak by hortonworks.

the class StackToTemplatePreparationObjectConverter method convert.

public TemplatePreparationObject convert(Stack source) {
    try {
        Map<String, Collection<ClusterExposedServiceView>> views = serviceEndpointCollector.prepareClusterExposedServicesViews(source.getCluster(), stackUtil.extractClusterManagerAddress(source));
        DetailedEnvironmentResponse environment = environmentClientService.getByCrn(source.getEnvironmentCrn());
        Credential credential = credentialConverter.convert(environment.getCredential());
        Cluster cluster = clusterService.getById(source.getCluster().getId());
        FileSystem fileSystem = cluster.getFileSystem();
        Optional<LdapView> ldapView = ldapConfigService.get(source.getEnvironmentCrn(), source.getName());
        ClouderaManagerRepo cm = clusterComponentConfigProvider.getClouderaManagerRepoDetails(cluster.getId());
        List<ClouderaManagerProduct> products = clusterComponentConfigProvider.getClouderaManagerProductDetails(cluster.getId());
        BaseFileSystemConfigurationsView fileSystemConfigurationView = getFileSystemConfigurationView(credential, source, fileSystem);
        updateFileSystemViewWithBackupLocation(environment, fileSystemConfigurationView);
        StackInputs stackInputs = getStackInputs(source);
        Map<String, Object> fixInputs = stackInputs.getFixInputs() == null ? new HashMap<>() : stackInputs.getFixInputs();
        fixInputs.putAll(stackInputs.getDatalakeInputs() == null ? new HashMap<>() : stackInputs.getDatalakeInputs());
        Gateway gateway = cluster.getGateway();
        String gatewaySignKey = null;
        if (gateway != null) {
            gatewaySignKey = gateway.getSignKey();
        }
        IdBroker idbroker = idBrokerService.getByCluster(cluster);
        if (idbroker == null) {
            idbroker = idBrokerConverterUtil.generateIdBrokerSignKeys(cluster);
            idBrokerService.save(idbroker);
        }
        String envCrnForVirtualGroups = getEnvironmentCrnForVirtualGroups(environment);
        VirtualGroupRequest virtualGroupRequest = new VirtualGroupRequest(envCrnForVirtualGroups, ldapView.map(LdapView::getAdminGroup).orElse(""));
        String accountId = Crn.safeFromString(source.getResourceCrn()).getAccountId();
        List<UserManagementProto.ServicePrincipalCloudIdentities> servicePrincipalCloudIdentities = grpcUmsClient.listServicePrincipalCloudIdentities(accountId, source.getEnvironmentCrn(), MDCUtils.getRequestId());
        BlueprintView blueprintView = blueprintViewProvider.getBlueprintView(cluster.getBlueprint());
        Optional<String> version = Optional.ofNullable(blueprintView.getVersion());
        Builder builder = Builder.builder().withCloudPlatform(CloudPlatform.valueOf(source.getCloudPlatform())).withRdsConfigs(postgresConfigService.createRdsConfigIfNeeded(source, cluster)).withRdsSslCertificateFilePath(dbCertificateProvider.getSslCertsFilePath()).withGateway(gateway, gatewaySignKey, exposedServiceCollector.getAllKnoxExposed(version)).withIdBroker(idbroker).withCustomConfigurationsView(getCustomConfigurationsView(source, cluster)).withCustomInputs(stackInputs.getCustomInputs() == null ? new HashMap<>() : stackInputs.getCustomInputs()).withFixInputs(fixInputs).withBlueprintView(blueprintView).withFileSystemConfigurationView(fileSystemConfigurationView).withGeneralClusterConfigs(calculateGeneralClusterConfigs(source, cluster)).withLdapConfig(ldapView.orElse(null)).withKerberosConfig(kerberosConfigService.get(source.getEnvironmentCrn(), source.getName()).orElse(null)).withProductDetails(cm, products).withExposedServices(views).withDefaultTags(getStackTags(source)).withSharedServiceConfigs(datalakeService.createSharedServiceConfigsView(source)).withStackType(source.getType()).withVirtualGroupView(virtualGroupRequest);
        transactionService.required(() -> {
            builder.withHostgroups(hostGroupService.getByCluster(cluster.getId()));
        });
        decorateBuilderWithPlacement(source, builder);
        decorateBuilderWithAccountMapping(source, environment, credential, builder, virtualGroupRequest);
        decorateBuilderWithServicePrincipals(source, builder, servicePrincipalCloudIdentities);
        decorateDatalakeView(source, builder);
        return builder.build();
    } catch (AccountTagValidationFailed aTVF) {
        throw new CloudbreakServiceException(aTVF);
    } catch (BlueprintProcessingException | IOException | TransactionService.TransactionExecutionException e) {
        throw new CloudbreakServiceException(e.getMessage(), e);
    }
}

Example 58 with ClouderaManagerRepo

use of com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo in project cloudbreak by hortonworks.

the class ClusterHostServiceRunnerTest method testDecoratePillarWithClouderaManagerRepo.

@Test
void testDecoratePillarWithClouderaManagerRepo() throws IOException, CloudbreakOrchestratorFailedException {
    String license = FileReaderUtils.readFileFromClasspath("cm-license.txt");
    when(cmLicenseParser.parseLicense(license)).thenCallRealMethod();
    ClouderaManagerRepo clouderaManagerRepo = new ClouderaManagerRepo();
    clouderaManagerRepo.setVersion("7.2.0");
    clouderaManagerRepo.setBaseUrl("https://archive.cloudera.com/cm/7.2.0/");
    Map<String, SaltPillarProperties> pillar = new HashMap<>();
    underTest.decoratePillarWithClouderaManagerRepo(clouderaManagerRepo, pillar, Optional.of(license));
    SaltPillarProperties resultPillar = pillar.get("cloudera-manager-repo");
    Map<String, Object> properties = resultPillar.getProperties();
    Map<String, Object> values = (Map<String, Object>) properties.get("cloudera-manager");
    assertEquals("7.2.0", ((ClouderaManagerRepo) values.get("repo")).getVersion());
    assertEquals("https://archive.cloudera.com/cm/7.2.0/", ((ClouderaManagerRepo) values.get("repo")).getBaseUrl());
    assertEquals("d2834876-30fe-4000-ba85-6e99e537897e", values.get("paywall_username"));
    assertEquals("db5d119ac130", values.get("paywall_password"));
}

Example 59 with ClouderaManagerRepo

use of com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo in project cloudbreak by hortonworks.

the class ClusterHostServiceRunnerTest method testDecoratePillarWithClouderaManagerRepoWithEmptyLicense.

@Test
void testDecoratePillarWithClouderaManagerRepoWithEmptyLicense() throws IOException, CloudbreakOrchestratorFailedException {
    String license = FileReaderUtils.readFileFromClasspath("cm-license-empty.txt");
    when(cmLicenseParser.parseLicense(license)).thenCallRealMethod();
    ClouderaManagerRepo clouderaManagerRepo = new ClouderaManagerRepo();
    clouderaManagerRepo.setVersion("7.2.0");
    clouderaManagerRepo.setBaseUrl("https://archive.cloudera.com/cm/7.2.0/");
    Map<String, SaltPillarProperties> pillar = new HashMap<>();
    underTest.decoratePillarWithClouderaManagerRepo(clouderaManagerRepo, pillar, Optional.of(license));
    SaltPillarProperties resultPillar = pillar.get("cloudera-manager-repo");
    Map<String, Object> properties = resultPillar.getProperties();
    Map<String, Object> values = (Map<String, Object>) properties.get("cloudera-manager");
    assertEquals("7.2.0", ((ClouderaManagerRepo) values.get("repo")).getVersion());
    assertEquals("https://archive.cloudera.com/cm/7.2.0/", ((ClouderaManagerRepo) values.get("repo")).getBaseUrl());
    assertNull(values.get("paywall_username"));
    assertNull(values.get("paywall_password"));
}

Example 60 with ClouderaManagerRepo

use of com.sequenceiq.cloudbreak.cloud.model.ClouderaManagerRepo in project cloudbreak by hortonworks.

the class ClouderaManagerClusterCreationSetupService method checkCmStackRepositories.

private void checkCmStackRepositories(ClusterComponent cmRepoConfig, Component imageComponent) throws IOException, CloudbreakImageCatalogException {
    if (Objects.nonNull(cmRepoConfig)) {
        ClouderaManagerRepo clouderaManagerRepo = cmRepoConfig.getAttributes().get(ClouderaManagerRepo.class);
        Image image = imageComponent.getAttributes().get(Image.class);
        StackMatrixV4Response stackMatrixV4Response = stackMatrixService.getStackMatrix(cmRepoConfig.getCluster().getWorkspace().getId(), platformStringTransformer.getPlatformStringForImageCatalog(cmRepoConfig.getCluster().getStack().getCloudPlatform(), cmRepoConfig.getCluster().getStack().getPlatformVariant()), image.getImageCatalogName());
        Map<String, ClouderaManagerStackDescriptorV4Response> stackDescriptorMap = stackMatrixV4Response.getCdh();
        ClouderaManagerStackDescriptorV4Response clouderaManagerStackDescriptor = stackDescriptorMap.get(clouderaManagerRepo.getVersion());
        if (clouderaManagerStackDescriptor != null) {
            boolean hasDefaultStackRepoUrlForOsType = clouderaManagerStackDescriptor.getRepository().getStack().containsKey(image.getOsType());
            boolean hasDefaultCmRepoUrlForOsType = clouderaManagerStackDescriptor.getClouderaManager().getRepository().containsKey(image.getOsType());
            if (!hasDefaultCmRepoUrlForOsType || !hasDefaultStackRepoUrlForOsType) {
                String message = String.format("The given repository information seems to be incompatible." + " CM version: %s, Image Id: %s, Os type: %s.", clouderaManagerRepo.getVersion(), image.getImageId(), image.getOsType());
                LOGGER.warn(message);
            }
        }
    }
}