Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 46 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method createStorageUnitEntity.

/**
 * Creates a storage unit entity per specified parameters.
 *
 * @param businessObjectDataEntity the business object data entity
 * @param storageEntity the storage entity
 * @param storageDirectory the storage directory
 * @param storageFiles the list of storage files
 * @param isDiscoverStorageFiles specifies if
 *
 * @return the newly created storage unit entity
 */
public StorageUnitEntity createStorageUnitEntity(BusinessObjectDataEntity businessObjectDataEntity, StorageEntity storageEntity, StorageDirectory storageDirectory, List<StorageFile> storageFiles, Boolean isDiscoverStorageFiles) {
    // Get the storage unit status entity for the ENABLED status.
    StorageUnitStatusEntity storageUnitStatusEntity = storageUnitStatusDaoHelper.getStorageUnitStatusEntity(StorageUnitStatusEntity.ENABLED);
    // Set up flags which are used to make flow logic easier.
    boolean isS3StoragePlatform = storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3);
    boolean isStorageDirectorySpecified = (storageDirectory != null);
    boolean validatePathPrefix = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), storageEntity, false, true);
    boolean validateFileExistence = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), storageEntity, false, true);
    boolean validateFileSize = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_SIZE), storageEntity, false, true);
    // Ensure that file size validation is not enabled without file existence validation.
    if (validateFileSize) {
        Assert.isTrue(validateFileExistence, String.format("Storage \"%s\" has file size validation enabled without file existence validation.", storageEntity.getName()));
    }
    String expectedS3KeyPrefix = null;
    // Retrieve S3 key prefix velocity template storage attribute value and store it in memory.
    // Please note that it is not required, so we pass in a "false" flag.
    String s3KeyPrefixVelocityTemplate = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), storageEntity, false);
    if (StringUtils.isNotBlank(s3KeyPrefixVelocityTemplate)) {
        // If the storage has any validation configured, get the expected S3 key prefix.
        expectedS3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(s3KeyPrefixVelocityTemplate, businessObjectDataEntity.getBusinessObjectFormat(), businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
    }
    if ((validatePathPrefix || validateFileExistence) && isS3StoragePlatform) {
        // If path prefix validation is enabled, validate that S3 key prefix velocity template is configured.
        Assert.isTrue(!validatePathPrefix || StringUtils.isNotBlank(s3KeyPrefixVelocityTemplate), String.format("Storage \"%s\" has enabled path validation without S3 key prefix velocity template configured.", storageEntity.getName()));
    }
    // Process storage directory path if it is specified.
    String directoryPath = null;
    if (isStorageDirectorySpecified) {
        // Get the specified directory path.
        directoryPath = storageDirectory.getDirectoryPath();
        // If the validate path prefix flag is configured for this storage, validate the directory path value.
        if (validatePathPrefix && isS3StoragePlatform) {
            // Ensure the directory path adheres to the S3 naming convention.
            Assert.isTrue(directoryPath.equals(expectedS3KeyPrefix), String.format("Specified directory path \"%s\" does not match the expected S3 key prefix \"%s\".", directoryPath, expectedS3KeyPrefix));
            // Ensure that the directory path is not already registered with another business object data instance.
            StorageUnitEntity alreadyRegisteredStorageUnitEntity = storageUnitDao.getStorageUnitByStorageNameAndDirectoryPath(storageEntity.getName(), directoryPath);
            if (alreadyRegisteredStorageUnitEntity != null) {
                throw new AlreadyExistsException(String.format("Storage directory \"%s\" in \"%s\" storage is already registered by the business object data {%s}.", directoryPath, storageEntity.getName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(alreadyRegisteredStorageUnitEntity.getBusinessObjectData())));
            }
        }
    } else if (Boolean.TRUE.equals(businessObjectDataEntity.getStatus().getPreRegistrationStatus())) {
        directoryPath = expectedS3KeyPrefix;
    }
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitDaoHelper.setStorageUnitStatus(storageUnitEntity, storageUnitStatusEntity);
    storageUnitEntity.setDirectoryPath(directoryPath);
    // Discover storage files if storage file discovery is enabled. Otherwise, get the storage files specified in the request, if any.
    List<StorageFile> resultStorageFiles = BooleanUtils.isTrue(isDiscoverStorageFiles) ? discoverStorageFiles(storageEntity, directoryPath) : storageFiles;
    // Create the storage file entities.
    createStorageFileEntitiesFromStorageFiles(resultStorageFiles, storageEntity, BooleanUtils.isTrue(isDiscoverStorageFiles), expectedS3KeyPrefix, storageUnitEntity, directoryPath, validatePathPrefix, validateFileExistence, validateFileSize, isS3StoragePlatform);
    return storageUnitEntity;
}

Example 47 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataInitiateDestroyHelperServiceImpl method prepareToInitiateDestroyImpl.

/**
 * Prepares to initiate a business object data destroy process by validating specified business object data along with other related database entities. The
 * method also initializes business object data destroy DTO passed as a parameter.
 *
 * @param businessObjectDataDestroyDto the DTO that holds various parameters needed to initiate a business object data destroy
 * @param businessObjectDataKey the business object data key
 */
protected void prepareToInitiateDestroyImpl(BusinessObjectDataDestroyDto businessObjectDataDestroyDto, BusinessObjectDataKey businessObjectDataKey) {
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Get the S3 object tag key to be used to tag the objects for archiving.
    String s3ObjectTagKey = configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_KEY);
    // Get the S3 object tag value to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTagValue = configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_VALUE);
    // Get the ARN of the role to assume to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleArn = configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_ARN);
    // Get the session identifier for the assumed role to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleSessionName = configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_SESSION_NAME);
    // Get the configured delay (in days) for business object data finalize destroy.
    int finalDestroyInDays = getAndValidateFinalDestroyInDays();
    // Retrieve business object data entity and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Validate business object data including the retention information.
    validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
    // Retrieve and validate a storage unit entity for this business object data.
    StorageUnitEntity storageUnitEntity = getAndValidateStorageUnit(businessObjectDataEntity, businessObjectDataKey);
    // Get the storage entity.
    StorageEntity storageEntity = storageUnitEntity.getStorage();
    // Validate the storage.
    validateStorage(storageUnitEntity.getStorage());
    // Validate that S3 storage has S3 bucket name configured.
    // Please note that since S3 bucket name attribute value is required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageEntity, true);
    // Get storage specific S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectDataEntity.getBusinessObjectFormat(), businessObjectDataKey);
    // Get the storage name.
    String storageName = storageEntity.getName();
    // Retrieve and validate storage files registered with the storage unit, if they exist.
    List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFilesIfPresent(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
    // Validate that this storage does not have any other registered storage files that
    // start with the S3 key prefix, but belong to other business object data instances.
    storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
    // Change the storage unit status to DISABLING and update the DTO.
    String reason = StorageUnitStatusEntity.DISABLING;
    businessObjectDataDestroyDto.setOldStorageUnitStatus(storageUnitEntity.getStatus().getCode());
    storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.DISABLING, reason);
    businessObjectDataDestroyDto.setNewStorageUnitStatus(storageUnitEntity.getStatus().getCode());
    // Change the business object data status to DELETED and update the DTO.
    businessObjectDataDestroyDto.setOldBusinessObjectDataStatus(businessObjectDataEntity.getStatus().getCode());
    businessObjectDataDaoHelper.updateBusinessObjectDataStatus(businessObjectDataEntity, BusinessObjectDataStatusEntity.DELETED);
    businessObjectDataDestroyDto.setNewBusinessObjectDataStatus(businessObjectDataEntity.getStatus().getCode());
    // Initialize other parameters in the business object data destroy parameters DTO.
    businessObjectDataDestroyDto.setBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity));
    businessObjectDataDestroyDto.setStorageName(storageName);
    businessObjectDataDestroyDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
    businessObjectDataDestroyDto.setS3BucketName(s3BucketName);
    businessObjectDataDestroyDto.setS3KeyPrefix(s3KeyPrefix);
    businessObjectDataDestroyDto.setS3ObjectTagKey(s3ObjectTagKey);
    businessObjectDataDestroyDto.setS3ObjectTagValue(s3ObjectTagValue);
    businessObjectDataDestroyDto.setS3ObjectTaggerRoleArn(s3ObjectTaggerRoleArn);
    businessObjectDataDestroyDto.setS3ObjectTaggerRoleSessionName(s3ObjectTaggerRoleSessionName);
    businessObjectDataDestroyDto.setFinalDestroyInDays(finalDestroyInDays);
}

Example 48 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelper method createStorageFileFromEntity.

/**
 * Creates a storage file from the storage file entity.
 *
 * @param storageFileEntity the storage file entity
 *
 * @return the storage file
 */
public StorageFile createStorageFileFromEntity(StorageFileEntity storageFileEntity) {
    StorageFile storageFile = new StorageFile();
    storageFile.setFilePath(storageFileEntity.getPath());
    storageFile.setFileSizeBytes(storageFileEntity.getFileSizeBytes());
    storageFile.setRowCount(storageFileEntity.getRowCount());
    return storageFile;
}

Example 49 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelper method validateDownloadedS3Files.

/**
 * Validate downloaded S3 files per specified list of storage files.
 *
 * @param baseDirectory the local parent directory path, relative to which the files are expected to be located
 * @param s3KeyPrefix the S3 key prefix that was prepended to the S3 file paths, when they were uploaded to S3
 * @param storageFiles the list of storage files
 *
 * @throws IllegalStateException if files are not valid
 */
public void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, List<StorageFile> storageFiles) throws IllegalStateException {
    // Build a target local directory path, which is the parent directory plus the S3 key prefix.
    File targetLocalDirectory = Paths.get(baseDirectory, s3KeyPrefix).toFile();
    // Get a list of all files within the target local directory and its subdirectories.
    Collection<File> actualLocalFiles = FileUtils.listFiles(targetLocalDirectory, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
    // Validate the total file count.
    int storageFilesCount = CollectionUtils.isEmpty(storageFiles) ? 0 : storageFiles.size();
    if (storageFilesCount != actualLocalFiles.size()) {
        throw new IllegalStateException(String.format("Number of downloaded files does not match the storage unit information (expected %d files, actual %d files).", storageFiles.size(), actualLocalFiles.size()));
    }
    // Validate each downloaded file.
    if (storageFilesCount > 0) {
        for (StorageFile storageFile : storageFiles) {
            // Create a "real file" that points to the actual file on the file system.
            File localFile = Paths.get(baseDirectory, storageFile.getFilePath()).toFile();
            // Verify that the file exists.
            if (!localFile.isFile()) {
                throw new IllegalStateException(String.format("Downloaded \"%s\" file doesn't exist.", localFile));
            }
            // Validate the file size.
            if (localFile.length() != storageFile.getFileSizeBytes()) {
                throw new IllegalStateException(String.format("Size of the downloaded \"%s\" S3 file does not match the expected value (expected %d bytes, actual %d bytes).", localFile.getPath(), storageFile.getFileSizeBytes(), localFile.length()));
            }
        }
    }
}

Example 50 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageUnitHelper method createStorageUnitsFromEntities.

/**
 * Creates a list of storage units from the list of storage unit entities.
 *
 * @param storageUnitEntities the storage unit entities.
 * @param includeStorageUnitStatusHistory specifies to include storage unit status history for each storage unit in the response
 *
 * @return the list of storage units.
 */
public List<StorageUnit> createStorageUnitsFromEntities(Collection<StorageUnitEntity> storageUnitEntities, Boolean includeStorageUnitStatusHistory) {
    List<StorageUnit> storageUnits = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        StorageUnit storageUnit = new StorageUnit();
        storageUnits.add(storageUnit);
        Storage storage = new Storage();
        storageUnit.setStorage(storage);
        StorageEntity storageEntity = storageUnitEntity.getStorage();
        storage.setName(storageEntity.getName());
        storage.setStoragePlatformName(storageEntity.getStoragePlatform().getName());
        // Add the storage attributes.
        if (!CollectionUtils.isEmpty(storageEntity.getAttributes())) {
            List<Attribute> storageAttributes = new ArrayList<>();
            storage.setAttributes(storageAttributes);
            for (StorageAttributeEntity storageAttributeEntity : storageEntity.getAttributes()) {
                Attribute attribute = new Attribute();
                storageAttributes.add(attribute);
                attribute.setName(storageAttributeEntity.getName());
                attribute.setValue(storageAttributeEntity.getValue());
            }
        }
        // Add the storage directory.
        if (storageUnitEntity.getDirectoryPath() != null) {
            StorageDirectory storageDirectory = new StorageDirectory();
            storageUnit.setStorageDirectory(storageDirectory);
            storageDirectory.setDirectoryPath(storageUnitEntity.getDirectoryPath());
        }
        // Add the storage files.
        if (!storageUnitEntity.getStorageFiles().isEmpty()) {
            List<StorageFile> storageFiles = new ArrayList<>();
            storageUnit.setStorageFiles(storageFiles);
            for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
                storageFiles.add(storageFileHelper.createStorageFileFromEntity(storageFileEntity));
            }
        }
        // Set the storage unit status.
        storageUnit.setStorageUnitStatus(storageUnitEntity.getStatus().getCode());
        // If specified, add storage unit status history.
        if (BooleanUtils.isTrue(includeStorageUnitStatusHistory)) {
            List<StorageUnitStatusChangeEvent> storageUnitStatusChangeEvents = new ArrayList<>();
            storageUnit.setStorageUnitStatusHistory(storageUnitStatusChangeEvents);
            for (StorageUnitStatusHistoryEntity storageUnitStatusHistoryEntity : storageUnitEntity.getHistoricalStatuses()) {
                storageUnitStatusChangeEvents.add(new StorageUnitStatusChangeEvent(storageUnitStatusHistoryEntity.getStatus().getCode(), HerdDateUtils.getXMLGregorianCalendarValue(storageUnitStatusHistoryEntity.getCreatedOn()), storageUnitStatusHistoryEntity.getCreatedBy()));
            }
        }
        // Set the number of failed attempts to execute a storage policy transition.
        storageUnit.setStoragePolicyTransitionFailedAttempts(storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
        if (storageUnitEntity.getRestoreExpirationOn() != null) {
            storageUnit.setRestoreExpirationOn(HerdDateUtils.getXMLGregorianCalendarValue(storageUnitEntity.getRestoreExpirationOn()));
        }
    }
    return storageUnits;
}