Examples with CloudStorageRequest com.sequenceiq.common.api.cloudstorage.CloudStorageRequest used on opensource projects

Example 46 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageDecoratorTest method testUpdateCloudStorageLocationsWhenRequestContainsOneTemplatedLocationAndOneWithoutTemplatePlaceholderThenThatShouldBeReplaced.

@Test
void testUpdateCloudStorageLocationsWhenRequestContainsOneTemplatedLocationAndOneWithoutTemplatePlaceholderThenThatShouldBeReplaced() {
    String templatedStorageLocationBaseValue = "s3a://some-dir/some-other-dir/";
    CloudStorageCdpService templatedStorageLocationType = CloudStorageCdpService.DEFAULT_FS;
    String templatedStorageLocationValue = templatedStorageLocationBaseValue + "{{{clusterName}}}";
    StorageLocationBase templatedStorageLocationBase = new StorageLocationBase();
    templatedStorageLocationBase.setType(templatedStorageLocationType);
    templatedStorageLocationBase.setValue(templatedStorageLocationValue);
    CloudStorageCdpService storageLocationType = CloudStorageCdpService.FLINK_JOBMANAGER_ARCHIVE;
    String eStorageLocationValue = "s3a://some-awesome-dir/some-other-awesome-dir/" + CLUSTER_NAME;
    StorageLocationBase storageLocationBase = new StorageLocationBase();
    storageLocationBase.setType(storageLocationType);
    storageLocationBase.setValue(eStorageLocationValue);
    List<StorageLocationBase> storageLocations = new ArrayList<>(2);
    storageLocations.add(storageLocationBase);
    storageLocations.add(templatedStorageLocationBase);
    ConfigQueryEntry cqe = new ConfigQueryEntry();
    cqe.setType(CloudStorageCdpService.DEFAULT_FS);
    cqe.setDefaultPath(templatedStorageLocationBaseValue + CLUSTER_NAME);
    CloudStorageRequest request = new CloudStorageRequest();
    request.setLocations(storageLocations);
    SdxClusterResponse sdxReponse = new SdxClusterResponse();
    String storageLocationValue = eStorageLocationValue;
    sdxReponse.setCloudStorageBaseLocation(storageLocationValue);
    sdxReponse.setCloudStorageFileSystemType(FileSystemType.S3);
    ConfigQueryEntry cqeFlink = new ConfigQueryEntry();
    cqeFlink.setType(CloudStorageCdpService.FLINK_JOBMANAGER_ARCHIVE);
    cqeFlink.setDefaultPath(storageLocationValue);
    Set<ConfigQueryEntry> cqes = new LinkedHashSet<>(1);
    cqes.add(cqe);
    cqes.add(cqeFlink);
    when(blueprintService.queryFileSystemParameters(BLUEPRINT_NAME, CLUSTER_NAME, storageLocationValue, FileSystemType.S3.name(), "", true, false, 0L)).thenReturn(cqes);
    Pair<Blueprint, String> mockBt = mock(Pair.class);
    when(blueprintService.getBlueprintAndText(BLUEPRINT_NAME, 0L)).thenReturn(mockBt);
    FileSystemConfigQueryObject mockfscqo = mock(FileSystemConfigQueryObject.class);
    when(blueprintService.createFileSystemConfigQueryObject(mockBt, CLUSTER_NAME, sdxReponse.getCloudStorageBaseLocation(), sdxReponse.getCloudStorageFileSystemType().name(), "", true, false)).thenReturn(mockfscqo);
    when(cmCloudStorageConfigProvider.queryParameters(any(), eq(mockfscqo))).thenReturn(cqes);
    CloudStorageRequest result = underTest.updateCloudStorageLocations(BLUEPRINT_NAME, CLUSTER_NAME, request, List.of(sdxReponse));
    assertNotNull(result);
    assertEquals(0, result.getLocations().stream().filter(slb -> slb.getValue().contains("{{{") && slb.getValue().contains("}}}")).count());
}

Example 47 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageConverter method fileSystemToRequest.

public CloudStorageRequest fileSystemToRequest(FileSystem fileSystem) {
    CloudStorageRequest result = new CloudStorageRequest();
    CloudStorage cloudStorage = fileSystem.getCloudStorage();
    if (cloudStorage != null) {
        List<StorageIdentityBase> storageIdentityRequests = cloudStorage.getCloudIdentities().stream().map(this::cloudIdentityToRequest).collect(Collectors.toList());
        result.setIdentities(storageIdentityRequests);
        List<StorageLocationBase> storageLocationRequests = cloudStorage.getLocations().stream().map(this::storageLocationToRequest).collect(Collectors.toList());
        result.setLocations(storageLocationRequests);
        NullUtil.doIfNotNull(cloudStorage.getS3GuardDynamoTableName(), tableName -> {
            AwsStorageParameters awsStorageParameters = new AwsStorageParameters();
            S3Guard s3Guard = new S3Guard();
            s3Guard.setDynamoTableName(tableName);
            awsStorageParameters.setS3Guard(s3Guard);
            result.setAws(awsStorageParameters);
        });
        result.setAccountMapping(accountMappingToAccountMappingRequest(cloudStorage.getAccountMapping()));
    }
    return result;
}

Example 48 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageConverter method requestToAdditionalFileSystem.

public FileSystem requestToAdditionalFileSystem(CloudStorageBase cloudStorageRequest) {
    FileSystem fileSystem = new FileSystem();
    AwsEfsParameters efsParameters = cloudStorageRequest.getAws() == null ? null : cloudStorageRequest.getAws().getEfsParameters();
    if (efsParameters == null || StringUtils.isEmpty(efsParameters.getName())) {
        return null;
    }
    fileSystem.setName(efsParameters.getName());
    FileSystemType fileSystemType = FileSystemType.EFS;
    fileSystem.setType(fileSystemType);
    Map<String, Object> configurations = new HashMap<>();
    configurations.put(CloudEfsConfiguration.KEY_BACKUP_POLICY_STATUS, efsParameters.getBackupPolicyStatus());
    configurations.put(CloudEfsConfiguration.KEY_ENCRYPTED, efsParameters.getEncrypted());
    configurations.put(CloudEfsConfiguration.KEY_FILESYSTEM_POLICY, efsParameters.getFileSystemPolicy());
    configurations.put(CloudEfsConfiguration.KEY_FILESYSTEM_TAGS, efsParameters.getFileSystemTags());
    configurations.put(CloudEfsConfiguration.KEY_KMSKEYID, efsParameters.getKmsKeyId());
    configurations.put(CloudEfsConfiguration.KEY_LIFECYCLE_POLICIES, efsParameters.getLifeCyclePolicies());
    configurations.put(CloudEfsConfiguration.KEY_PERFORMANCE_MODE, efsParameters.getPerformanceMode());
    configurations.put(CloudEfsConfiguration.KEY_PROVISIONED_THROUGHPUT_INMIBPS, efsParameters.getProvisionedThroughputInMibps());
    configurations.put(CloudEfsConfiguration.KEY_THROUGHPUT_MODE, efsParameters.getThroughputMode());
    String configString;
    try {
        configString = JsonUtil.writeValueAsString(configurations);
    } catch (JsonProcessingException ignored) {
        configString = configurations.toString();
    }
    fileSystem.setConfigurations(new Json(configString));
    CloudStorage cloudStorage = new CloudStorage();
    if (cloudStorageRequest.getIdentities() != null) {
        Optional<CloudIdentity> cloudIdentity = cloudStorageRequest.getIdentities().stream().map(this::identityRequestToCloudIdentity).filter(currCloudIdentity -> currCloudIdentity.getEfsIdentity() != null).findFirst();
        if (cloudIdentity != null && cloudIdentity.get() != null) {
            cloudStorage.setCloudIdentities(List.of(cloudIdentity.get()));
        }
    }
    cloudStorage.setAccountMapping(accountMappingRequestToAccountMapping(cloudStorageRequest.getAccountMapping()));
    fileSystem.setCloudStorage(cloudStorage);
    return fileSystem;
}

Example 49 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifester method initCloudStorageRequest.

public CloudStorageRequest initCloudStorageRequest(DetailedEnvironmentResponse environment, ClusterV4Request clusterRequest, SdxCluster sdxCluster, SdxClusterRequest sdxClusterRequest) {
    CloudStorageRequest cloudStorageRequest = null;
    boolean anyCloudStorageIsConfigured = isCloudStorageConfigured(sdxClusterRequest) || isInternalCloudStorageConfigured(clusterRequest);
    boolean loggingConfigured = isLoggingConfigured(environment);
    if (isCloudStorageConfigured(sdxClusterRequest)) {
        LOGGER.debug("Cloud storage configurations found in SDX cluster request.");
        cloudStorageRequest = initSdxCloudStorageRequest(environment.getCloudPlatform(), clusterRequest.getBlueprintName(), sdxCluster.getClusterName(), sdxClusterRequest.getCloudStorage());
    } else if (isInternalCloudStorageConfigured(clusterRequest)) {
        LOGGER.debug("Cloud storage configurations found in internal SDX stack request.");
        cloudStorageRequest = clusterRequest.getCloudStorage();
    }
    if (loggingConfigured) {
        LOGGER.debug("Cloud storage logging is enabled.");
        if (!anyCloudStorageIsConfigured) {
            LOGGER.debug("Creating cloud storage request only for logging identity.");
            cloudStorageRequest = new CloudStorageRequest();
        }
        addLogIdentity(cloudStorageRequest, environment);
    }
    if (loggingConfigured || anyCloudStorageIsConfigured) {
        addS3Guard(cloudStorageRequest, environment);
    }
    return cloudStorageRequest;
}

Example 50 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifester method initSdxCloudStorageRequest.

public CloudStorageRequest initSdxCloudStorageRequest(String cloudPlatform, String blueprint, String clusterName, SdxCloudStorageRequest cloudStorage) {
    CloudStorageRequest cloudStorageRequest = new CloudStorageRequest();
    normalizeCloudStorageRequest(cloudStorage);
    storageValidationService.validateCloudStorage(cloudPlatform, cloudStorage);
    FileSystemParameterV4Responses fileSystemRecommendations = getFileSystemRecommendations(blueprint, clusterName, cloudStorage);
    LOGGER.info("File recommendations {}", fileSystemRecommendations);
    setStorageLocations(fileSystemRecommendations, cloudStorageRequest);
    addIdBrokerIdentity(cloudStorage, cloudStorageRequest);
    return cloudStorageRequest;
}