Examples with StorageFileEntity org.finra.herd.model.jpa.StorageFileEntity used on opensource projects

Example 21 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesAutoDiscoveryPreviouslyRegisteredS3FileSizeMismatch.

@Test
public void testCreateBusinessObjectDataStorageFilesAutoDiscoveryPreviouslyRegisteredS3FileSizeMismatch() throws Exception {
    // Create and persist a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(StorageEntity.MANAGED_STORAGE, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.UPLOADING, StorageUnitStatusEntity.ENABLED, testS3KeyPrefix);
    // Create and persist a storage file with file size that would not match S3 reported size.
    StorageFileEntity storageFileEntity = storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, testS3KeyPrefix + "/" + FILE_PATH_1, FILE_SIZE_2_KB, NO_ROW_COUNT);
    storageUnitEntity.getStorageFiles().add(storageFileEntity);
    // Create and upload to S3 managed storage two test files with 1 KB file size.
    businessObjectDataServiceTestHelper.prepareTestS3Files(testS3KeyPrefix, localTempPath, Arrays.asList(FILE_PATH_1, FILE_PATH_2));
    // Try to discover storage files when file size reported by S3 does not match for an already registered storage file.
    try {
        businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, StorageEntity.MANAGED_STORAGE, NO_STORAGE_FILES, DISCOVER_STORAGE_FILES));
        fail("Should throw an IllegalArgumentException when an already registered storage file size does not match file size reported by S3.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Previously registered storage file \"%s/%s\" has file size of %d bytes that does not match file size of %d bytes reported by S3.", testS3KeyPrefix, FILE_PATH_1, FILE_SIZE_2_KB, FILE_SIZE_1_KB), e.getMessage());
    }
}

Example 22 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method createStorageFileEntitiesFromStorageFiles.

/**
 * Creates a list of storage file entities from a list of storage files.
 *
 * @param storageFiles the list of storage files
 * @param storageEntity the storage entity
 * @param storageFilesDiscovered specifies whether the storage files were actually discovered in the storage
 * @param expectedS3KeyPrefix the expected S3 key prefix
 * @param storageUnitEntity the storage unit entity that storage file entities will belong to
 * @param directoryPath the storage directory path
 * @param validatePathPrefix specifies whether the storage has S3 key prefix validation enabled
 * @param validateFileExistence specifies whether the storage has file existence enabled
 * @param validateFileSize specifies whether the storage has file validation enabled
 * @param isS3StoragePlatform specifies whether the storage platform type is S3
 *
 * @return the list of storage file entities
 */
private List<StorageFileEntity> createStorageFileEntitiesFromStorageFiles(List<StorageFile> storageFiles, StorageEntity storageEntity, boolean storageFilesDiscovered, String expectedS3KeyPrefix, StorageUnitEntity storageUnitEntity, String directoryPath, boolean validatePathPrefix, boolean validateFileExistence, boolean validateFileSize, boolean isS3StoragePlatform) {
    List<StorageFileEntity> storageFileEntities = null;
    // Process storage files if they are specified.
    if (CollectionUtils.isNotEmpty(storageFiles)) {
        storageFileEntities = new ArrayList<>();
        storageUnitEntity.setStorageFiles(storageFileEntities);
        // If the validate file existence flag is configured for this storage and storage files were not discovered, prepare for S3 file validation.
        S3FileTransferRequestParamsDto params = null;
        Map<String, StorageFile> actualS3Keys = null;
        if (validateFileExistence && isS3StoragePlatform && !storageFilesDiscovered) {
            // Get the validate file parameters.
            params = getFileValidationParams(storageEntity, expectedS3KeyPrefix, storageUnitEntity, validatePathPrefix);
            // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
            actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
        }
        // storage by some other business object data that start with the expected S3 key prefix.
        if (validatePathPrefix && isS3StoragePlatform) {
            // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
            String expectedS3KeyPrefixWithTrailingSlash = expectedS3KeyPrefix + "/";
            Long registeredStorageFileCount = storageFileDao.getStorageFileCount(storageEntity.getName(), expectedS3KeyPrefixWithTrailingSlash);
            if (registeredStorageFileCount > 0) {
                throw new AlreadyExistsException(String.format("Found %d storage file(s) matching \"%s\" S3 key prefix in \"%s\" " + "storage that is registered with another business object data.", registeredStorageFileCount, expectedS3KeyPrefix, storageEntity.getName()));
            }
        }
        for (StorageFile storageFile : storageFiles) {
            StorageFileEntity storageFileEntity = new StorageFileEntity();
            storageFileEntities.add(storageFileEntity);
            storageFileEntity.setStorageUnit(storageUnitEntity);
            storageFileEntity.setPath(storageFile.getFilePath());
            storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes());
            storageFileEntity.setRowCount(storageFile.getRowCount());
            // Skip storage file validation if storage files were discovered.
            if (!storageFilesDiscovered) {
                // Otherwise, if a directory path is specified, ensure it is consistent with the file path.
                if (validatePathPrefix && isS3StoragePlatform) {
                    // Ensure the S3 file key prefix adheres to the S3 naming convention.
                    Assert.isTrue(storageFileEntity.getPath().startsWith(expectedS3KeyPrefix), String.format("Specified storage file path \"%s\" does not match the expected S3 key prefix \"%s\".", storageFileEntity.getPath(), expectedS3KeyPrefix));
                } else if (directoryPath != null) {
                    // When storage directory path is specified, ensure that storage file path starts with it.
                    Assert.isTrue(storageFileEntity.getPath().startsWith(directoryPath), String.format("Storage file path \"%s\" does not match the storage directory path \"%s\".", storageFileEntity.getPath(), directoryPath));
                }
                // Ensure the file exists in S3 if the validate file existence flag is configured for this storage.
                if (validateFileExistence && isS3StoragePlatform) {
                    // Validate storage file.
                    storageFileHelper.validateStorageFile(storageFile, params.getS3BucketName(), actualS3Keys, validateFileSize);
                }
            }
        }
    }
    return storageFileEntities;
}

Example 23 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class FileUploadCleanupServiceImpl method deleteBusinessObjectData.

@Override
public List<BusinessObjectDataKey> deleteBusinessObjectData(String storageName, int thresholdMinutes) {
    LOGGER.info("Deleting dangling business object data from the storage that is older than the threshold... storageName=\"{}\" thresholdMinutes={}", storageName, thresholdMinutes);
    // Get the storage entity and make sure it exists.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageName);
    // Returns a new instance of S3FileTransferRequestParamsDto populated with all parameters, required to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity);
    // Get dangling business object data records having storage files associated with the specified storage.
    List<BusinessObjectDataEntity> businessObjectDataEntities = businessObjectDataDao.getBusinessObjectDataFromStorageOlderThan(storageName, thresholdMinutes, Arrays.asList(BusinessObjectDataStatusEntity.DELETED));
    // Build a list of keys for business object data that got marked as DELETED.
    List<BusinessObjectDataKey> resultBusinessObjectDataKeys = new ArrayList<>();
    for (BusinessObjectDataEntity businessObjectDataEntity : businessObjectDataEntities) {
        try {
            // Get the storage unit. Please note that the storage unit entity must exist,
            // since we selected business object data entities associated with the storage.
            StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(storageName, businessObjectDataEntity);
            // Validate that none of the storage files (if any) exist in the relative S3 bucket.
            boolean foundExistingS3File = false;
            for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
                // Check if the actual S3 file does not exist.
                s3FileTransferRequestParamsDto.setS3KeyPrefix(storageFileEntity.getPath());
                if (s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto) != null) {
                    foundExistingS3File = true;
                    break;
                }
            }
            // If not S3 files exist, mark the business object data as DELETED.
            if (!foundExistingS3File) {
                // Update the business object data status.
                BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity);
                String originalBusinessObjectStatus = businessObjectDataEntity.getStatus().getCode();
                uploadDownloadHelperService.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.DELETED);
                // Create business object data notification.
                notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, businessObjectDataKey, BusinessObjectDataStatusEntity.DELETED, originalBusinessObjectStatus);
                // Add the business object data key to the result list.
                resultBusinessObjectDataKeys.add(businessObjectDataKey);
                // Log the business object data status change.
                LOGGER.info("Changed business object data status. " + "oldBusinessObjectDataStatus=\"{}\" newBusinessObjectDataStatus=\"{}\" businessObjectDataKey={}", originalBusinessObjectStatus, BusinessObjectDataStatusEntity.DELETED, jsonHelper.objectToJson(businessObjectDataKey));
            }
        } catch (RuntimeException e) {
            // Log the exception and continue the processing.
            LOGGER.error("Failed to delete business object data. businessObjectDataKey={}", jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)), e);
        }
    }
    return resultBusinessObjectDataKeys;
}

Example 24 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StorageUnitHelper method createStorageUnitsFromEntities.

/**
 * Creates a list of storage units from the list of storage unit entities.
 *
 * @param storageUnitEntities the storage unit entities.
 * @param includeStorageUnitStatusHistory specifies to include storage unit status history for each storage unit in the response
 *
 * @return the list of storage units.
 */
public List<StorageUnit> createStorageUnitsFromEntities(Collection<StorageUnitEntity> storageUnitEntities, Boolean includeStorageUnitStatusHistory) {
    List<StorageUnit> storageUnits = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        StorageUnit storageUnit = new StorageUnit();
        storageUnits.add(storageUnit);
        Storage storage = new Storage();
        storageUnit.setStorage(storage);
        StorageEntity storageEntity = storageUnitEntity.getStorage();
        storage.setName(storageEntity.getName());
        storage.setStoragePlatformName(storageEntity.getStoragePlatform().getName());
        // Add the storage attributes.
        if (!CollectionUtils.isEmpty(storageEntity.getAttributes())) {
            List<Attribute> storageAttributes = new ArrayList<>();
            storage.setAttributes(storageAttributes);
            for (StorageAttributeEntity storageAttributeEntity : storageEntity.getAttributes()) {
                Attribute attribute = new Attribute();
                storageAttributes.add(attribute);
                attribute.setName(storageAttributeEntity.getName());
                attribute.setValue(storageAttributeEntity.getValue());
            }
        }
        // Add the storage directory.
        if (storageUnitEntity.getDirectoryPath() != null) {
            StorageDirectory storageDirectory = new StorageDirectory();
            storageUnit.setStorageDirectory(storageDirectory);
            storageDirectory.setDirectoryPath(storageUnitEntity.getDirectoryPath());
        }
        // Add the storage files.
        if (!storageUnitEntity.getStorageFiles().isEmpty()) {
            List<StorageFile> storageFiles = new ArrayList<>();
            storageUnit.setStorageFiles(storageFiles);
            for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
                storageFiles.add(storageFileHelper.createStorageFileFromEntity(storageFileEntity));
            }
        }
        // Set the storage unit status.
        storageUnit.setStorageUnitStatus(storageUnitEntity.getStatus().getCode());
        // If specified, add storage unit status history.
        if (BooleanUtils.isTrue(includeStorageUnitStatusHistory)) {
            List<StorageUnitStatusChangeEvent> storageUnitStatusChangeEvents = new ArrayList<>();
            storageUnit.setStorageUnitStatusHistory(storageUnitStatusChangeEvents);
            for (StorageUnitStatusHistoryEntity storageUnitStatusHistoryEntity : storageUnitEntity.getHistoricalStatuses()) {
                storageUnitStatusChangeEvents.add(new StorageUnitStatusChangeEvent(storageUnitStatusHistoryEntity.getStatus().getCode(), HerdDateUtils.getXMLGregorianCalendarValue(storageUnitStatusHistoryEntity.getCreatedOn()), storageUnitStatusHistoryEntity.getCreatedBy()));
            }
        }
        // Set the number of failed attempts to execute a storage policy transition.
        storageUnit.setStoragePolicyTransitionFailedAttempts(storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
        if (storageUnitEntity.getRestoreExpirationOn() != null) {
            storageUnit.setRestoreExpirationOn(HerdDateUtils.getXMLGregorianCalendarValue(storageUnitEntity.getRestoreExpirationOn()));
        }
    }
    return storageUnits;
}

Example 25 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StorageFileDaoHelper method createStorageFileEntitiesFromStorageFiles.

/**
 * Creates storage file entities from the list of storage files.
 *
 * @param storageUnitEntity the storage unit entity
 * @param storageFiles the list of storage files
 *
 * @return the list storage file entities
 */
public List<StorageFileEntity> createStorageFileEntitiesFromStorageFiles(StorageUnitEntity storageUnitEntity, List<StorageFile> storageFiles) {
    List<StorageFileEntity> storageFileEntities = new ArrayList<>();
    for (StorageFile storageFile : storageFiles) {
        StorageFileEntity storageFileEntity = new StorageFileEntity();
        storageFileEntities.add(storageFileEntity);
        storageFileEntity.setStorageUnit(storageUnitEntity);
        storageFileEntity.setPath(storageFile.getFilePath());
        storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes());
        storageFileEntity.setRowCount(storageFile.getRowCount());
        storageFileDao.saveAndRefresh(storageFileEntity);
    }
    return storageFileEntities;
}