Examples with StoragePolicy com.epam.pipeline.entity.datastorage.StoragePolicy used on opensource projects

Example 1 with StoragePolicy

use of com.epam.pipeline.entity.datastorage.StoragePolicy in project cloud-pipeline by epam.

the class DataStorageManager method updateStoragePolicy.

private AbstractDataStorage updateStoragePolicy(AbstractDataStorage dataStorage, DataStorageVO dataStorageVO) {
    verifyStoragePolicy(dataStorageVO, dataStorageVO.getStoragePolicy(), false);
    StoragePolicy policy = dataStorageVO.getStoragePolicy() == null ? new StoragePolicy() : dataStorageVO.getStoragePolicy();
    dataStorage.setStoragePolicy(policy);
    return dataStorage;
}

Example 2 with StoragePolicy

use of com.epam.pipeline.entity.datastorage.StoragePolicy in project cloud-pipeline by epam.

the class DataStorageLoaderTest method shouldLoadDataStorageTest.

@Test
void shouldLoadDataStorageTest() throws EntityNotFoundException {
    StoragePolicy policy = new StoragePolicy();
    policy.setBackupDuration(DURATION);
    policy.setLongTermStorageDuration(DURATION);
    policy.setShortTermStorageDuration(DURATION);
    S3bucketDataStorage expectedDataStorage = new S3bucketDataStorage();
    expectedDataStorage.setId(1L);
    expectedDataStorage.setParentFolderId(1L);
    expectedDataStorage.setName(TEST_NAME);
    expectedDataStorage.setPath(TEST_PATH);
    expectedDataStorage.setOwner(TEST_NAME);
    expectedDataStorage.setStoragePolicy(policy);
    DataStorageLoader dataStorageLoader = new DataStorageLoader(apiClient);
    when(apiClient.loadDataStorage(anyLong())).thenReturn(expectedDataStorage);
    Optional<EntityContainer<DataStorageDoc>> container = dataStorageLoader.loadEntity(1L);
    EntityContainer<DataStorageDoc> storageDocEntityContainer = container.orElseThrow(AssertionError::new);
    DataStorageDoc actualDataStorageDoc = storageDocEntityContainer.getEntity();
    assertNotNull(actualDataStorageDoc);
    AbstractDataStorage actualDataStorage = actualDataStorageDoc.getStorage();
    verifyDataStorage(expectedDataStorage, actualDataStorage);
    verifyPipelineUser(storageDocEntityContainer.getOwner());
    verifyPermissions(PERMISSIONS_CONTAINER_WITH_OWNER, storageDocEntityContainer.getPermissions());
    verifyMetadata(EXPECTED_METADATA, new ArrayList<>(storageDocEntityContainer.getMetadata().values()));
}

Example 3 with StoragePolicy

use of com.epam.pipeline.entity.datastorage.StoragePolicy in project cloud-pipeline by epam.

the class DataStorageMapperTest method shouldMapS3DataStorage.

@Test
void shouldMapS3DataStorage() throws IOException {
    DataStorageMapper mapper = new DataStorageMapper(SearchDocumentType.S3_STORAGE);
    StoragePolicy policy = new StoragePolicy();
    policy.setBackupDuration(DURATION);
    policy.setLongTermStorageDuration(DURATION);
    policy.setShortTermStorageDuration(DURATION);
    S3bucketDataStorage dataStorage = new S3bucketDataStorage();
    fillStorage(dataStorage);
    dataStorage.setStoragePolicy(policy);
    DataStorageDoc doc = DataStorageDoc.builder().regionName(TEST_REGION).storage(dataStorage).build();
    XContentBuilder contentBuilder = mapper.map(buildContainer(doc));
    verifyS3Storage(dataStorage, TEST_REGION, contentBuilder);
    verifyPermissions(PERMISSIONS_CONTAINER, contentBuilder);
    verifyMetadata(EXPECTED_METADATA, contentBuilder);
    verifyPipelineUser(USER, contentBuilder);
}

Example 4 with StoragePolicy

use of com.epam.pipeline.entity.datastorage.StoragePolicy in project cloud-pipeline by epam.

the class DataStorageManager method verifyStoragePolicy.

private void verifyStoragePolicy(DataStorageVO dataStorageVO, StoragePolicy policy, boolean useDefault) {
    Integer defaultBackupDuration = preferenceManager.getPreference(SystemPreferences.DATA_STORAGE_POLICY_BACKUP_DURATION);
    if (policy == null) {
        if (useDefault) {
            StoragePolicy storagePolicy = new StoragePolicy();
            Boolean versioningEnabled = preferenceManager.getPreference(SystemPreferences.DATA_STORAGE_POLICY_BACKUP_ENABLED);
            if (versioningEnabled != null) {
                storagePolicy.setVersioningEnabled(versioningEnabled);
            }
            if (storagePolicy.isVersioningEnabled()) {
                storagePolicy.setBackupDuration(defaultBackupDuration);
            }
            dataStorageVO.setStoragePolicy(storagePolicy);
        }
        return;
    }
    if (policy.isVersioningEnabled() && policy.getBackupDuration() == null) {
        policy.setBackupDuration(defaultBackupDuration);
    }
    Integer stsDuration = policy.getShortTermStorageDuration();
    Integer ltsDuration = policy.getLongTermStorageDuration();
    Assert.isTrue(!(stsDuration != null && ltsDuration == null), messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_RULE_STS_OR_LTS_REQUIRED));
    Assert.isTrue(stsDuration == null || stsDuration > 0, messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_ILLEGAL_DURATION, stsDuration));
    Assert.isTrue(ltsDuration == null || ltsDuration > 0, messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_ILLEGAL_DURATION, ltsDuration));
    Assert.isTrue(!(stsDuration != null && ltsDuration < stsDuration), messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_ILLEGAL_DURATION_COMBINATION, stsDuration, ltsDuration));
    Assert.isTrue(policy.getBackupDuration() == null || policy.getBackupDuration() > 0, messageHelper.getMessage(MessageConstants.ERROR_DATASTORAGE_ILLEGAL_DURATION, policy.getBackupDuration()));
}

Example 5 with StoragePolicy

use of com.epam.pipeline.entity.datastorage.StoragePolicy in project cloud-pipeline by epam.

the class PipelineDocumentTemplateManager method applyFilesGeneratedByPipeline.

private void applyFilesGeneratedByPipeline(PipelineDocumentTemplate template) {
    Optional<AbstractDataStorage> dataStorageOptional = dataStorageManager.getDataStorages().stream().filter(ds -> ds.getName().toLowerCase().contains("analysis")).findAny();
    if (dataStorageOptional.isPresent()) {
        AbstractDataStorage abstractDataStorage = dataStorageOptional.get();
        StoragePolicy policy = abstractDataStorage.getStoragePolicy();
        Integer totalDays = policy == null ? 0 : policy.getLongTermStorageDuration();
        final Integer daysInYear = 365;
        final Integer daysInMonth = 30;
        Integer years = Math.floorDiv(totalDays, daysInYear);
        Integer months = Math.floorDiv(totalDays - years * daysInYear, daysInMonth);
        Integer days = totalDays - years * daysInYear - months * daysInMonth;
        List<String> fates = new ArrayList<>();
        if (years > 1) {
            fates.add(String.format("%d years", years));
        } else if (years == 1) {
            fates.add(String.format("%d year", years));
        }
        if (months > 1) {
            fates.add(String.format("%d months", months));
        } else if (months == 1) {
            fates.add(String.format("%d month", months));
        }
        if (days > 1 || (days == 0 && fates.size() == 0)) {
            fates.add(String.format("%d days", days));
        } else if (days == 1) {
            fates.add(String.format("%d day", days));
        }
        String fateDescription = String.join(", ", fates);
        List<DataStorageRule> rules = dataStorageRuleManager.loadAllRulesForPipeline(template.getPipeline().getId());
        for (DataStorageRule rule : rules) {
            String name = rule.getFileMask();
            String fate = rule.getMoveToSts() ? fateDescription : "Removed at completion";
            template.getFilesGeneratedDuringPipelineProcessing().add(new ImmutablePair<>(name, fate));
        }
    }
}