Examples with BusinessObjectDataStorageUnitKey org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey used on opensource projects

Example 46 with BusinessObjectDataStorageUnitKey

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceTest method testBusinessObjectDataFinalizeRestoreHelperServiceMethodsNewTransactionPropagation.

/**
 * This method is to get coverage for the business object data finalize restore helper service methods that have an explicit annotation for transaction
 * propagation.
 */
@Test
public void testBusinessObjectDataFinalizeRestoreHelperServiceMethodsNewTransactionPropagation() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = storageUnitHelper.createStorageUnitKey(businessObjectDataKey, STORAGE_NAME);
    try {
        businessObjectDataFinalizeRestoreHelperServiceImpl.prepareToFinalizeRestore(storageUnitKey);
        fail();
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(businessObjectDataKey, NO_BDATA_STATUS), e.getMessage());
    }
    // Create a storage file.
    StorageFile storageFile = new StorageFile(TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE, FILE_SIZE_1_KB, NO_ROW_COUNT);
    // Create a business object data restore DTO.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, NO_S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, Arrays.asList(storageFile), NO_EXCEPTION);
    try {
        businessObjectDataFinalizeRestoreHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
        fail();
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", storageFile.getFilePath(), STORAGE_NAME), e.getMessage());
    }
    try {
        businessObjectDataFinalizeRestoreHelperServiceImpl.completeFinalizeRestore(businessObjectDataRestoreDto);
        fail();
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(businessObjectDataKey, NO_BDATA_STATUS), e.getMessage());
    }
}

Example 47 with BusinessObjectDataStorageUnitKey

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreServiceTest method testFinalizeRestoreAmazonServiceException.

@Test
public void testFinalizeRestoreAmazonServiceException() throws Exception {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Get the expected S3 key prefix for the business object data key.
    String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(businessObjectDataKey, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.PARTITION_KEY, AbstractServiceTest.NO_SUB_PARTITION_KEYS);
    // Create S3FileTransferRequestParamsDto to access the S3 bucket.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create database entities required for testing.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataServiceTestHelper.createDatabaseEntitiesForFinalizeRestoreTesting(businessObjectDataKey);
    // Get the storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    // Add a mocked S3 file name to the storage unit that would trigger an Amazon service exception when we try to get metadata for the object.
    storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/%s", s3KeyPrefix, MockS3OperationsImpl.MOCK_S3_FILE_NAME_SERVICE_EXCEPTION), FILE_SIZE_1_KB, ROW_COUNT);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = storageUnitHelper.createStorageUnitKey(businessObjectDataKey, STORAGE_NAME);
    try {
        // Put relative "already restored" Glacier storage class S3 files in the S3 bucket.
        for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
            ObjectMetadata metadata = new ObjectMetadata();
            metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
            metadata.setOngoingRestore(false);
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFileEntity.getPath(), new ByteArrayInputStream(new byte[storageFileEntity.getFileSizeBytes().intValue()]), metadata), null);
        }
        // Try to finalize a restore for the storage unit when get S3 object metadata operation fails with an Amazon service exception.
        try {
            businessObjectDataFinalizeRestoreService.finalizeRestore(storageUnitKey);
            fail("Should throw an IllegalStateException when a get S3 object metadata operation fails.");
        } catch (IllegalStateException e) {
            assertEquals(String.format("Fail to check restore status for \"%s/%s\" key in \"%s\" bucket. " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", s3KeyPrefix, MockS3OperationsImpl.MOCK_S3_FILE_NAME_SERVICE_EXCEPTION, S3_BUCKET_NAME), e.getMessage());
        }
        // Validate that the storage unit status is still in RESTORING state.
        assertEquals(StorageUnitStatusEntity.RESTORING, storageUnitEntity.getStatus().getCode());
        // Validate that we have the S3 files at the expected S3 location.
        assertEquals(storageUnitEntity.getStorageFiles().size(), s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
    }
}

Example 48 with BusinessObjectDataStorageUnitKey

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreServiceTest method testFinalizeRestore.

@Test
public void testFinalizeRestore() throws Exception {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Get the expected S3 key prefix for the business object data key.
    String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(businessObjectDataKey, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.PARTITION_KEY, AbstractServiceTest.NO_SUB_PARTITION_KEYS);
    // Create S3FileTransferRequestParamsDto to access the S3 bucket.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create database entities required for testing.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataServiceTestHelper.createDatabaseEntitiesForFinalizeRestoreTesting(businessObjectDataKey);
    // Get the storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = storageUnitHelper.createStorageUnitKey(businessObjectDataKey, STORAGE_NAME);
    try {
        // Put relative "already restored" Glacier storage class S3 files in the S3 bucket.
        for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
            ObjectMetadata metadata = new ObjectMetadata();
            metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
            metadata.setOngoingRestore(false);
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFileEntity.getPath(), new ByteArrayInputStream(new byte[storageFileEntity.getFileSizeBytes().intValue()]), metadata), NO_S3_CLIENT);
        }
        // Finalize a restore of the storage unit.
        businessObjectDataFinalizeRestoreService.finalizeRestore(storageUnitKey);
        // Validate that the storage unit status is now RESTORED.
        assertEquals(StorageUnitStatusEntity.RESTORED, storageUnitEntity.getStatus().getCode());
        // Validate that we have the S3 files at the expected S3 location.
        assertEquals(storageUnitEntity.getStorageFiles().size(), s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
    }
}

Example 49 with BusinessObjectDataStorageUnitKey

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreServiceTest method testBusinessObjectDataFinalizeRestoreServiceMethodsNewTransactionPropagation.

/**
 * This method is to get coverage for the business object data finalize restore service methods that have an explicit annotation for transaction
 * propagation.
 */
@Test
public void testBusinessObjectDataFinalizeRestoreServiceMethodsNewTransactionPropagation() {
    assertEquals(0, businessObjectDataFinalizeRestoreServiceImpl.getS3StorageUnitsToRestore(MAX_RESULT).size());
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = storageUnitHelper.createStorageUnitKey(businessObjectDataKey, STORAGE_NAME);
    try {
        businessObjectDataFinalizeRestoreServiceImpl.finalizeRestore(storageUnitKey);
        fail();
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(businessObjectDataKey, NO_BDATA_STATUS), e.getMessage());
    }
}

Example 50 with BusinessObjectDataStorageUnitKey

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey in project herd by FINRAOS.

the class CleanupDestroyedBusinessObjectDataJob method executeInternal.

@Override
protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
    // Log that the system job is started.
    LOGGER.info("Started system job. systemJobName=\"{}\"", JOB_NAME);
    // Get the parameter values.
    int maxBusinessObjectDataInstancesToProcess = parameterHelper.getParameterValueAsInteger(parameters, ConfigurationValue.CLEANUP_DESTROYED_BDATA_JOB_MAX_BDATA_INSTANCES);
    // Log the parameter values.
    LOGGER.info("systemJobName=\"{}\" {}={}", JOB_NAME, ConfigurationValue.CLEANUP_DESTROYED_BDATA_JOB_MAX_BDATA_INSTANCES, maxBusinessObjectDataInstancesToProcess);
    // Continue the processing only if the maximum number of business object data instances
    // that is allowed to be processed in a single run of this system job is greater than zero.
    int processedBusinessObjectDataInstances = 0;
    if (maxBusinessObjectDataInstancesToProcess > 0) {
        // Select restored business object data that is already expired.
        List<BusinessObjectDataStorageUnitKey> businessObjectDataStorageUnitKeys = cleanupDeletedBusinessObjectDataService.getS3StorageUnitsToCleanup(maxBusinessObjectDataInstancesToProcess);
        // Log the number of storage units selected for processing.
        LOGGER.info("Selected for processing S3 storage units. systemJobName=\"{}\" storageUnitCount={}", JOB_NAME, businessObjectDataStorageUnitKeys.size());
        // Try to expire each of the selected storage units.
        for (BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey : businessObjectDataStorageUnitKeys) {
            try {
                cleanupDeletedBusinessObjectDataService.cleanupS3StorageUnit(businessObjectDataStorageUnitKey);
                processedBusinessObjectDataInstances += 1;
            } catch (RuntimeException runtimeException) {
                // Log the exception.
                LOGGER.error("Failed to cleanup a destroyed business object data. systemJobName=\"{}\" storageName=\"{}\" businessObjectDataKey={}", JOB_NAME, businessObjectDataStorageUnitKey.getStorageName(), jsonHelper.objectToJson(businessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(businessObjectDataStorageUnitKey)), runtimeException);
            }
        }
    }
    // Log the number of cleanup destroyed business object data instances.
    LOGGER.info("Cleanup destroyed business object data instances. systemJobName=\"{}\" businessObjectDataCount={}", JOB_NAME, processedBusinessObjectDataInstances);
    // Log that the system job is ended.
    LOGGER.info("Completed system job. systemJobName=\"{}\"", JOB_NAME);
}