Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 16 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class MockHttpClientOperationsImpl method buildGetBusinessObjectDataResponse.

/**
 * Builds a business object data response.
 *
 * @param response the response.
 * @param uri the URI of the incoming request.
 *
 * @throws JAXBException if a JAXB error occurred.
 */
private void buildGetBusinessObjectDataResponse(MockCloseableHttpResponse response, URI uri) throws JAXBException {
    Pattern pattern = Pattern.compile("/herd-app/rest/businessObjectData/namespaces/(.*)/businessObjectDefinitionNames/(.*)/businessObjectFormatUsages/(.*)" + "/businessObjectFormatFileTypes/(.*).*");
    Matcher matcher = pattern.matcher(uri.getPath());
    if (matcher.find()) {
        BusinessObjectData businessObjectData = new BusinessObjectData();
        businessObjectData.setNamespace(matcher.group(1));
        businessObjectData.setBusinessObjectDefinitionName(matcher.group(2));
        businessObjectData.setBusinessObjectFormatUsage(matcher.group(3));
        businessObjectData.setBusinessObjectFormatFileType(matcher.group(4));
        businessObjectData.setPartitionValue("2014-01-31");
        businessObjectData.setPartitionKey("PROCESS_DATE");
        businessObjectData.setAttributes(new ArrayList<Attribute>());
        businessObjectData.setBusinessObjectFormatVersion(0);
        businessObjectData.setLatestVersion(true);
        businessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
        List<StorageUnit> storageUnits = new ArrayList<>();
        businessObjectData.setStorageUnits(storageUnits);
        StorageUnit storageUnit = new StorageUnit();
        storageUnits.add(storageUnit);
        storageUnit.setStorage(getNewStorage(StorageEntity.MANAGED_STORAGE));
        List<StorageFile> storageFiles = new ArrayList<>();
        storageUnit.setStorageFiles(storageFiles);
        storageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
        List<String> localFiles = Arrays.asList("foo1.dat", "Foo2.dat", "FOO3.DAT", "folder/foo3.dat", "folder/foo2.dat", "folder/foo1.dat");
        for (String filename : localFiles) {
            StorageFile storageFile = new StorageFile();
            storageFiles.add(storageFile);
            storageFile.setFilePath(businessObjectData.getNamespace().toLowerCase().replace('_', '-') + "/exchange-a/" + businessObjectData.getBusinessObjectFormatUsage().toLowerCase().replace('_', '-') + "/" + businessObjectData.getBusinessObjectFormatFileType().toLowerCase().replace('_', '-') + "/" + businessObjectData.getBusinessObjectDefinitionName().toLowerCase().replace('_', '-') + "/frmt-v" + businessObjectData.getBusinessObjectFormatVersion() + "/data-v" + businessObjectData.getVersion() + "/" + businessObjectData.getPartitionKey().toLowerCase().replace('_', '-') + "=" + businessObjectData.getPartitionValue() + "/" + filename);
            storageFile.setFileSizeBytes(1024L);
            storageFile.setRowCount(10L);
        }
        businessObjectData.setSubPartitionValues(new ArrayList<String>());
        businessObjectData.setId(1234);
        businessObjectData.setVersion(0);
        response.setEntity(getHttpEntity(businessObjectData));
    }
}

Example 17 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceImpl method validateStorageFiles.

/**
 * Validates a list of storage files to be added to the specified storage unit.
 *
 * @param storageFiles the list of storage files
 * @param storageUnitEntity the storage unit entity
 * @param validatePathPrefix the validate path prefix flag
 * @param validateFileExistence the validate file existence flag
 * @param validateFileSize the validate file size flag
 */
private void validateStorageFiles(List<StorageFile> storageFiles, StorageUnitEntity storageUnitEntity, boolean validatePathPrefix, boolean validateFileExistence, boolean validateFileSize) {
    // Retrieve all storage files already registered for this storage unit loaded in a map for easy access.
    Map<String, StorageFileEntity> storageFileEntities = storageFileHelper.getStorageFileEntitiesMap(storageUnitEntity.getStorageFiles());
    // Perform validation of storage files listed in the request per storage directory path and/or validation flags.
    String directoryPath = null;
    String directoryPathWithTrailingSlash = null;
    if (StringUtils.isNotBlank(storageUnitEntity.getDirectoryPath())) {
        // Use the storage directory path from the storage unit.
        directoryPath = storageUnitEntity.getDirectoryPath();
        // Add a trailing slash to the storage directory path if it doesn't already have it.
        directoryPathWithTrailingSlash = StringUtils.appendIfMissing(directoryPath, "/");
        // If a storage directory path exists, then validate that all files being added are contained within that directory.
        for (StorageFile storageFile : storageFiles) {
            Assert.isTrue(storageFile.getFilePath().startsWith(directoryPathWithTrailingSlash), String.format("Storage file path \"%s\" does not match the storage directory path \"%s\".", storageFile.getFilePath(), directoryPathWithTrailingSlash));
        }
    } else if (validatePathPrefix || validateFileExistence) {
        // Use the expected S3 key prefix value as the storage directory path.
        directoryPath = s3KeyPrefixHelper.buildS3KeyPrefix(storageUnitEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData()));
        // Add a trailing slash to the expected S3 key prefix if it doesn't already have it.
        directoryPathWithTrailingSlash = StringUtils.appendIfMissing(directoryPath, "/");
        // Validate that all files are contained within the expected S3 key prefix.
        for (StorageFile storageFile : storageFiles) {
            Assert.isTrue(storageFile.getFilePath().startsWith(directoryPathWithTrailingSlash), String.format("Specified storage file path \"%s\" does not match the expected S3 key prefix \"%s\".", storageFile.getFilePath(), directoryPathWithTrailingSlash));
        }
    }
    // Validate that files in the request does not already exist in the database.
    if (StringUtils.isNotBlank(directoryPath)) {
        // Get a list of request storage file paths.
        List<String> requestStorageFilePaths = storageFileHelper.getFilePathsFromStorageFiles(storageFiles);
        // Retrieve all already registered storage files from the storage that start with the directory path.
        List<String> registeredStorageFilePaths = storageFileDao.getStorageFilesByStorageAndFilePathPrefix(storageUnitEntity.getStorage().getName(), directoryPathWithTrailingSlash);
        // Check if request contains any of the already registered files.
        registeredStorageFilePaths.retainAll(requestStorageFilePaths);
        if (!CollectionUtils.isEmpty(registeredStorageFilePaths)) {
            // Retrieve the storage file entity for the first "already registered" storage file.
            // Since the discovered storage file path exists in the database, we should not get a null back.
            StorageFileEntity storageFileEntity = storageFileDao.getStorageFileByStorageNameAndFilePath(storageUnitEntity.getStorage().getName(), registeredStorageFilePaths.get(0));
            // Throw an exception reporting the information on the "already registered" storage file.
            throw new AlreadyExistsException(String.format("S3 file \"%s\" in \"%s\" storage is already registered by the business object data {%s}.", registeredStorageFilePaths.get(0), storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(storageFileEntity.getStorageUnit().getBusinessObjectData())));
        }
    } else {
        // Since directory path is not available, we need to validate each storage file specified in the request individually.
        for (StorageFile storageFile : storageFiles) {
            // Ensure that the file is not already registered in this storage by some other business object data.
            StorageFileEntity storageFileEntity = storageFileDao.getStorageFileByStorageNameAndFilePath(storageUnitEntity.getStorage().getName(), storageFile.getFilePath());
            if (storageFileEntity != null) {
                throw new AlreadyExistsException(String.format("S3 file \"%s\" in \"%s\" storage is already registered by the business object data {%s}.", storageFile.getFilePath(), storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(storageFileEntity.getStorageUnit().getBusinessObjectData())));
            }
        }
    }
    // Validate file existence.
    if (validateFileExistence) {
        // Get S3 bucket access parameters and set the key prefix to the directory path with a trailing slash.
        // Please note that since we got here, the directory path can not be empty.
        S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
        params.setS3KeyPrefix(directoryPathWithTrailingSlash);
        // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
        Map<String, StorageFile> actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
        // For the already registered storage files, validate each storage file against S3 keys and metadata reported by S3.
        for (Map.Entry<String, StorageFileEntity> entry : storageFileEntities.entrySet()) {
            storageFileHelper.validateStorageFileEntity(entry.getValue(), params.getS3BucketName(), actualS3Keys, validateFileSize);
        }
        // Validate each storage file listed in the request.
        for (StorageFile storageFile : storageFiles) {
            storageFileHelper.validateStorageFile(storageFile, params.getS3BucketName(), actualS3Keys, validateFileSize);
        }
    }
}

Example 18 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceImpl method createBusinessObjectDataStorageFilesImpl.

/**
 * Adds files to Business object data storage.
 *
 * @param businessObjectDataStorageFilesCreateRequest the business object data storage files create request
 *
 * @return BusinessObjectDataStorageFilesCreateResponse
 */
protected BusinessObjectDataStorageFilesCreateResponse createBusinessObjectDataStorageFilesImpl(BusinessObjectDataStorageFilesCreateRequest businessObjectDataStorageFilesCreateRequest) {
    // validate request
    validateBusinessObjectDataStorageFilesCreateRequest(businessObjectDataStorageFilesCreateRequest);
    // retrieve and validate that the business object data exists
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(getBusinessObjectDataKey(businessObjectDataStorageFilesCreateRequest));
    // Validate that business object data is in one of the pre-registered states.
    Assert.isTrue(BooleanUtils.isTrue(businessObjectDataEntity.getStatus().getPreRegistrationStatus()), String.format("Business object data status must be one of the pre-registration statuses. Business object data status {%s}, business object data {%s}", businessObjectDataEntity.getStatus().getCode(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
    // retrieve and validate that the storage unit exists
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(businessObjectDataStorageFilesCreateRequest.getStorageName(), businessObjectDataEntity);
    // Validate the storage unit has an acceptable status for adding new files.
    Assert.isTrue(StorageUnitStatusEntity.ENABLED.equals(storageUnitEntity.getStatus().getCode()), String.format("Storage unit must be in the ENABLED status. Storage unit status {%s}, business object data {%s}", storageUnitEntity.getStatus().getCode(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
    StorageEntity storageEntity = storageUnitEntity.getStorage();
    // Get the S3 validation flags.
    boolean validatePathPrefix = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), storageEntity, false, true);
    boolean validateFileExistence = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), storageEntity, false, true);
    boolean validateFileSize = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_SIZE), storageEntity, false, true);
    // Ensure that file size validation is not enabled without file existence validation.
    if (validateFileSize) {
        Assert.isTrue(validateFileExistence, String.format("Storage \"%s\" has file size validation enabled without file existence validation.", storageEntity.getName()));
    }
    // Process the add storage files request based on the auto-discovery of storage files being enabled or not.
    List<StorageFile> storageFiles;
    if (BooleanUtils.isTrue(businessObjectDataStorageFilesCreateRequest.isDiscoverStorageFiles())) {
        // Discover new storage files for this storage unit.
        storageFiles = discoverStorageFiles(storageUnitEntity);
    } else {
        // Get the list of storage files from the request.
        storageFiles = businessObjectDataStorageFilesCreateRequest.getStorageFiles();
        // Validate storage files.
        validateStorageFiles(storageFiles, storageUnitEntity, validatePathPrefix, validateFileExistence, validateFileSize);
    }
    // Add new storage files to the storage unit.
    storageFileDaoHelper.createStorageFileEntitiesFromStorageFiles(storageUnitEntity, storageFiles);
    // Construct and return the response.
    return createBusinessObjectDataStorageFilesCreateResponse(storageEntity, businessObjectDataEntity, storageFiles);
}

Example 19 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceTest method testExecuteS3SpecificStepsGlacierS3FileStillRestoring.

@Test
public void testExecuteS3SpecificStepsGlacierS3FileStillRestoring() throws Exception {
    // Create S3FileTransferRequestParamsDto to access the S3 bucket.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a business object data restore DTO.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, NO_S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, Arrays.asList(new StorageFile(TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE, FILE_SIZE_1_KB, NO_ROW_COUNT)), NO_EXCEPTION);
    try {
        // Put a "still restoring" Glacier storage class S3 file in the Glacier S3 bucket.
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
        metadata.setOngoingRestore(true);
        s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, String.format("%s/%s", TEST_S3_KEY_PREFIX, LOCAL_FILE), new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), metadata), NO_S3_CLIENT);
        // Try to execute S3 specific steps to finalize a restore for the storage unit when Glacier S3 file is still restoring.
        try {
            businessObjectDataFinalizeRestoreHelperService.executeS3SpecificSteps(businessObjectDataRestoreDto);
            fail();
        } catch (IllegalArgumentException e) {
            assertEquals(String.format("Archived Glacier S3 file \"%s/%s\" is not restored. " + "StorageClass {GLACIER}, OngoingRestore flag {true}, S3 bucket name {%s}", TEST_S3_KEY_PREFIX, LOCAL_FILE, S3_BUCKET_NAME), e.getMessage());
        }
        // Validate that we have a Glacier S3 file at the expected S3 location.
        assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
    }
}

Example 20 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() throws Exception {
    // Create S3FileTransferRequestParamsDto to access the S3 bucket.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files.
    List<StorageFile> storageFiles = new ArrayList<>();
    for (String filePath : LOCAL_FILES) {
        storageFiles.add(new StorageFile(TEST_S3_KEY_PREFIX + "/" + filePath, FILE_SIZE_1_KB, NO_ROW_COUNT));
    }
    // Create a business object data restore DTO.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, NO_S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    try {
        // Put relative Glacier storage class S3 files in the S3 bucket.
        for (StorageFile storageFile : storageFiles) {
            ObjectMetadata metadata = new ObjectMetadata();
            metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
            metadata.setOngoingRestore(false);
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[storageFile.getFileSizeBytes().intValue()]), metadata), NO_S3_CLIENT);
        }
        // Execute S3 specific steps to finalize a restore for the Glacier storage unit.
        businessObjectDataFinalizeRestoreHelperService.executeS3SpecificSteps(businessObjectDataRestoreDto);
        // Validate that we have the restored S3 files at the expected S3 location.
        assertEquals(storageFiles.size(), s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
    }
}