Examples with StorageFileEntity org.finra.herd.model.jpa.StorageFileEntity used on opensource projects

Example 26 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class GetBusinessObjectDataTest method setupDatabase.

/**
 * Inserts a business object data and their FK relationships into the database.
 *
 * @param dataProviderName the data provider name.
 * @param businessObjectDefinitionName the business object definition name.
 * @param fileTypeCode the file type code.
 * @param businessObjectFormatUsage the business object format usage.
 * @param businessObjectFormatVersion the business object format version.
 * @param businessObjectDataVersion the business object data version.
 * @param partitionKey the partition key.
 * @param partitionValues the partition values.
 */
private void setupDatabase(String namespace, String dataProviderName, String businessObjectDefinitionName, String fileTypeCode, String businessObjectFormatUsage, Integer businessObjectFormatVersion, Integer businessObjectDataVersion, String partitionKey, String[] partitionValues) {
    DataProviderEntity dataProviderEntity = new DataProviderEntity();
    dataProviderEntity.setName(dataProviderName);
    herdDao.saveAndRefresh(dataProviderEntity);
    NamespaceEntity namespaceEntity = new NamespaceEntity();
    namespaceEntity.setCode(namespace);
    herdDao.saveAndRefresh(namespaceEntity);
    BusinessObjectDefinitionEntity businessObjectDefinition = new BusinessObjectDefinitionEntity();
    businessObjectDefinition.setDataProvider(dataProviderEntity);
    businessObjectDefinition.setName(businessObjectDefinitionName);
    businessObjectDefinition.setNamespace(namespaceEntity);
    herdDao.saveAndRefresh(businessObjectDefinition);
    FileTypeEntity fileTypeEntity = new FileTypeEntity();
    fileTypeEntity.setCode(fileTypeCode);
    herdDao.saveAndRefresh(fileTypeEntity);
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    businessObjectFormatEntity.setBusinessObjectDefinition(businessObjectDefinition);
    businessObjectFormatEntity.setBusinessObjectFormatVersion(businessObjectFormatVersion);
    businessObjectFormatEntity.setFileType(fileTypeEntity);
    businessObjectFormatEntity.setLatestVersion(true);
    businessObjectFormatEntity.setNullValue("#");
    businessObjectFormatEntity.setPartitionKey(partitionKey);
    businessObjectFormatEntity.setUsage(businessObjectFormatUsage);
    herdDao.saveAndRefresh(businessObjectFormatEntity);
    StoragePlatformEntity storagePlatformEntity = new StoragePlatformEntity();
    storagePlatformEntity.setName(randomString());
    herdDao.saveAndRefresh(storagePlatformEntity);
    StorageEntity storageEntity = new StorageEntity();
    storageEntity.setName(randomString());
    storageEntity.setStoragePlatform(storagePlatformEntity);
    herdDao.saveAndRefresh(storageEntity);
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    businessObjectDataEntity.setBusinessObjectFormat(businessObjectFormatEntity);
    businessObjectDataEntity.setLatestVersion(true);
    businessObjectDataEntity.setPartitionValue(partitionValues[0]);
    businessObjectDataEntity.setPartitionValue2(partitionValues[1]);
    businessObjectDataEntity.setPartitionValue3(partitionValues[2]);
    businessObjectDataEntity.setPartitionValue4(partitionValues[3]);
    businessObjectDataEntity.setPartitionValue5(partitionValues[4]);
    businessObjectDataEntity.setStatus(businessObjectDataStatusDao.getBusinessObjectDataStatusByCode(BusinessObjectDataStatusEntity.VALID));
    Collection<StorageUnitEntity> storageUnits = new ArrayList<>();
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    Collection<StorageFileEntity> storageFiles = new ArrayList<>();
    StorageFileEntity storageFileEntity = new StorageFileEntity();
    storageFileEntity.setPath(randomString());
    storageFileEntity.setFileSizeBytes(1000l);
    storageFileEntity.setStorageUnit(storageUnitEntity);
    storageFiles.add(storageFileEntity);
    storageUnitEntity.setStorageFiles(storageFiles);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitEntity.setStatus(storageUnitStatusDao.getStorageUnitStatusByCode(StorageUnitStatusEntity.ENABLED));
    storageUnits.add(storageUnitEntity);
    businessObjectDataEntity.setStorageUnits(storageUnits);
    businessObjectDataEntity.setVersion(businessObjectDataVersion);
    herdDao.saveAndRefresh(businessObjectDataEntity);
}

Example 27 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceImpl method deleteBusinessObjectData.

@NamespacePermission(fields = "#businessObjectDataKey.namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public BusinessObjectData deleteBusinessObjectData(BusinessObjectDataKey businessObjectDataKey, Boolean deleteFiles) {
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Validate the mandatory deleteFiles flag.
    Assert.notNull(deleteFiles, "A delete files flag must be specified.");
    // Retrieve the business object data and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // If the business object data has children, remove the parent children relationship.
    if (!businessObjectDataEntity.getBusinessObjectDataChildren().isEmpty()) {
        for (BusinessObjectDataEntity childBusinessObjectEntity : businessObjectDataEntity.getBusinessObjectDataChildren()) {
            childBusinessObjectEntity.getBusinessObjectDataParents().remove(businessObjectDataEntity);
        }
        String businessObjectDataChildren = businessObjectDataEntity.getBusinessObjectDataChildren().stream().map(bData -> String.format("{%s}", businessObjectDataHelper.businessObjectDataEntityAltKeyToString(bData))).collect(Collectors.joining(", "));
        businessObjectDataEntity.setBusinessObjectDataChildren(new ArrayList<BusinessObjectDataEntity>());
        businessObjectDataDao.save(businessObjectDataEntity);
        LOGGER.warn(String.format("Deleting business object data {%s} that has children associated with it. The parent relationship has been removed from: %s.", businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), businessObjectDataChildren));
    }
    // If the flag is set, clean up the data files from all storages of S3 storage platform type.
    LOGGER.info("deleteFiles={}", deleteFiles);
    if (deleteFiles) {
        // Loop over all storage units for this business object data.
        for (StorageUnitEntity storageUnitEntity : businessObjectDataEntity.getStorageUnits()) {
            StorageEntity storageEntity = storageUnitEntity.getStorage();
            // Currently, we only support data file deletion from S3 platform type.
            if (storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3)) {
                LOGGER.info("Deleting business object data files from the storage... storageName=\"{}\" businessObjectDataKey={}", storageEntity.getName(), jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
                // Get the S3 validation flags.
                boolean validatePathPrefix = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), storageEntity, false, true);
                // If this storage conforms to the path prefix validation, then delete all keys found under the S3 key prefix.
                if (validatePathPrefix) {
                    // Retrieve S3 key prefix velocity template storage attribute value and store it in memory.
                    // Please note that it is not required, so we pass in a "false" flag.
                    String s3KeyPrefixVelocityTemplate = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), storageEntity, false);
                    // Validate that S3 key prefix velocity template is configured.
                    Assert.isTrue(StringUtils.isNotBlank(s3KeyPrefixVelocityTemplate), String.format("Storage \"%s\" has enabled path validation without S3 key prefix velocity template configured.", storageEntity.getName()));
                    // Build the S3 key prefix as per S3 Naming Convention Wiki page.
                    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(s3KeyPrefixVelocityTemplate, businessObjectDataEntity.getBusinessObjectFormat(), businessObjectDataKey, storageEntity.getName());
                    // Get S3 bucket access parameters, such as bucket name, AWS access key ID, AWS secret access key, etc...
                    S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity);
                    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
                    params.setS3KeyPrefix(s3KeyPrefix + "/");
                    // Delete a list of all keys/objects from S3 managed bucket matching the expected S3 key prefix.
                    // Please note that when deleting S3 files, we also delete all 0 byte objects that represent S3 directories.
                    s3Service.deleteDirectory(params);
                } else // For a non S3 prefixed paths, delete the files explicitly or if only directory is registered, delete all files/subfolders found under it.
                {
                    // Get S3 bucket access parameters, such as bucket name, AWS access key ID, AWS secret access key, etc...
                    S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity);
                    // If only directory is registered delete all files/sub-folders found under it.
                    if (StringUtils.isNotBlank(storageUnitEntity.getDirectoryPath()) && storageUnitEntity.getStorageFiles().isEmpty()) {
                        // Since the directory path represents a directory, we add a trailing '/' character to it.
                        params.setS3KeyPrefix(storageUnitEntity.getDirectoryPath() + "/");
                        // Delete a list of all keys/objects from S3 bucket matching the directory path.
                        // Please note that when deleting S3 files, we also delete all 0 byte objects that represent S3 directories.
                        s3Service.deleteDirectory(params);
                    } else // Delete the files explicitly.
                    {
                        // Create a list of files to delete.
                        List<File> files = new ArrayList<>();
                        for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
                            files.add(new File(storageFileEntity.getPath()));
                        }
                        params.setFiles(files);
                        s3Service.deleteFileList(params);
                    }
                }
            } else {
                LOGGER.info("Skipping business object data file removal for a storage unit from the storage since it is not an S3 storage platform. " + " storageName=\"{}\" businessObjectDataKey={}", storageEntity.getName(), jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
            }
        }
    }
    // Create the business object data object from the entity.
    BusinessObjectData deletedBusinessObjectData = businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity);
    // Delete this business object data.
    businessObjectDataDao.delete(businessObjectDataEntity);
    // If this business object data version is the latest, set the latest flag on the previous version of this object data, if it exists.
    if (businessObjectDataEntity.getLatestVersion()) {
        // Get the maximum version for this business object data, if it exists.
        Integer maxBusinessObjectDataVersion = businessObjectDataDao.getBusinessObjectDataMaxVersion(businessObjectDataKey);
        if (maxBusinessObjectDataVersion != null) {
            // Retrieve the previous version business object data entity. Since we successfully got the maximum
            // version for this business object data, the retrieved entity is not expected to be null.
            BusinessObjectDataEntity previousVersionBusinessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(new BusinessObjectDataKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion(), businessObjectDataKey.getPartitionValue(), businessObjectDataKey.getSubPartitionValues(), maxBusinessObjectDataVersion));
            // Update the previous version business object data entity.
            previousVersionBusinessObjectDataEntity.setLatestVersion(true);
            businessObjectDataDao.saveAndRefresh(previousVersionBusinessObjectDataEntity);
        }
    }
    // Return the deleted business object data.
    return deletedBusinessObjectData;
}

Example 28 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionOtherBusinessObjectDataHasStorageFilesMatchingS3KeyPrefix.

@Test
public void testInitiateStoragePolicyTransitionOtherBusinessObjectDataHasStorageFilesMatchingS3KeyPrefix() throws Exception {
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create two business object data keys.
    List<BusinessObjectDataKey> businessObjectDataKeys = Arrays.asList(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION), new BusinessObjectDataKey(BDEF_NAMESPACE_2, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION));
    // For the business object data keys, create and persist two storage units.
    List<StorageUnitEntity> storageUnitEntities = Arrays.asList(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKeys.get(0), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH), storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKeys.get(1), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH));
    // Get the expected S3 key prefix for the business object data key.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // To both storage unit add storage files having the same S3 key prefix.
    List<StorageFileEntity> storageFileEntities = Arrays.asList(storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntities.get(0), expectedS3KeyPrefix + "/" + LOCAL_FILES.get(0), FILE_SIZE_1_KB, ROW_COUNT_1000), storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntities.get(1), expectedS3KeyPrefix + "/" + LOCAL_FILES.get(1), FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    // business object data storage files matching the expected S3 key prefix.
    try {
        storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKeys.get(0), storagePolicyKey, INITIAL_VERSION));
        fail();
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found %d registered storage file(s) matching business object data S3 key prefix in the storage that is not equal to the number " + "of storage files (%d) registered with the business object data in that storage. " + "Storage: {%s}, s3KeyPrefix {%s}, business object data: {%s}", storageFileEntities.size(), 1, STORAGE_NAME, expectedS3KeyPrefix + "/", businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKeys.get(0))), e.getMessage());
    } finally {
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}

Example 29 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionStorageFileDoesNotMatchS3KeyPrefix.

@Test
public void testInitiateStoragePolicyTransitionStorageFileDoesNotMatchS3KeyPrefix() throws Exception {
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist a storage unit.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Get the expected S3 key prefix for the business object data key.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Add to the storage unit a storage file that is not matching the expected S3 key prefix.
    StorageFileEntity storageFileEntity = storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, STORAGE_DIRECTORY_PATH + "/" + LOCAL_FILE, FILE_SIZE_1_KB, ROW_COUNT_1000);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    // Try to initiate a storage policy transition when storage unit has a storage file that is not matching the expected S3 key prefix.
    try {
        storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage file \"%s\" registered with business object data {%s} in \"%s\" storage " + "does not match the expected S3 key prefix \"%s\".", storageFileEntity.getPath(), businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey), STORAGE_NAME, expectedS3KeyPrefix), e.getMessage());
    } finally {
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}

Example 30 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceImpl method discoverStorageFiles.

/**
 * Discovers new storage files in S3 for the specified storage unit.
 *
 * @param storageUnitEntity the storage unit entity
 *
 * @return the list of discovered storage files
 */
private List<StorageFile> discoverStorageFiles(StorageUnitEntity storageUnitEntity) {
    // Retrieve all storage files already registered for this storage unit loaded in a map for easy access.
    Map<String, StorageFileEntity> storageFileEntities = storageFileHelper.getStorageFileEntitiesMap(storageUnitEntity.getStorageFiles());
    // Validate and get storage directory path from the storage unit.
    Assert.hasText(storageUnitEntity.getDirectoryPath(), "Business object data has no storage directory path which is required for auto-discovery of storage files.");
    String directoryPath = storageUnitEntity.getDirectoryPath();
    // Add a trailing slash to the storage directory path if it doesn't already have it.
    String directoryPathWithTrailingSlash = StringUtils.appendIfMissing(directoryPath, "/");
    // Retrieve all already registered storage files from the storage that start with the directory path.
    List<String> registeredStorageFilePaths = storageFileDao.getStorageFilesByStorageAndFilePathPrefix(storageUnitEntity.getStorage().getName(), directoryPathWithTrailingSlash);
    // Sanity check already registered storage files.
    if (storageFileEntities.size() != registeredStorageFilePaths.size()) {
        throw new IllegalArgumentException(String.format("Number of storage files (%d) already registered for the business object data in \"%s\" storage is not equal to " + "the number of registered storage files (%d) matching \"%s\" S3 key prefix in the same storage.", storageFileEntities.size(), storageUnitEntity.getStorage().getName(), registeredStorageFilePaths.size(), directoryPathWithTrailingSlash));
    }
    // Get S3 bucket access parameters and set the key prefix to the directory path with a trailing slash.
    // Please note that since we got here, the directory path can not be empty.
    S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
    params.setS3KeyPrefix(directoryPathWithTrailingSlash);
    // List S3 files ignoring 0 byte objects that represent S3 directories.
    // Please note that the map implementation returned by the helper method below
    // preserves the original order of files as returned by the S3 list command.
    Map<String, StorageFile> actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
    // For the already registered storage files, validate file existence and file size against S3 keys and metadata reported by S3.
    for (Map.Entry<String, StorageFileEntity> entry : storageFileEntities.entrySet()) {
        storageFileHelper.validateStorageFileEntity(entry.getValue(), params.getS3BucketName(), actualS3Keys, true);
    }
    // Remove all already registered storage files from the map of actual S3 keys.
    actualS3Keys.keySet().removeAll(storageFileEntities.keySet());
    // Validate that we have at least one unregistered storage file discovered in S3.
    Assert.notEmpty(actualS3Keys.keySet(), String.format("No unregistered storage files were discovered at s3://%s/%s location.", params.getS3BucketName(), directoryPathWithTrailingSlash));
    // Build and return a list of storage files.
    return new ArrayList<>(actualS3Keys.values());
}