Examples with StorageFileEntity org.finra.herd.model.jpa.StorageFileEntity used on opensource projects

Example 11 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method createData.

private void createData(String storageUnitDirectory, boolean s3Managed, Collection<String> files) {
    // Create and persist a "pre-registration" business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
    // Create and persist a business object data entity.
    BusinessObjectDataEntity bod = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, businessObjectDataStatusEntity.getCode());
    StorageEntity s;
    if (s3Managed) {
        s = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    } else {
        s = super.storageDaoTestHelper.createStorageEntity(STORAGE_NAME, STORAGE_PLATFORM_CODE);
    }
    StorageUnitEntity su = super.storageUnitDaoTestHelper.createStorageUnitEntity(s, bod, StorageUnitStatusEntity.ENABLED, storageUnitDirectory);
    for (String file : files) {
        StorageFileEntity f = super.storageFileDaoTestHelper.createStorageFileEntity(su, file, FILE_SIZE_1_KB, ROW_COUNT_1000);
        su.getStorageFiles().add(f);
    }
}

Example 12 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesPreviouslyRegisteredS3FileSizeMismatchIgnoredDueToDisabledFileSizeValidation.

@Test
public void testCreateBusinessObjectDataStorageFilesPreviouslyRegisteredS3FileSizeMismatchIgnoredDueToDisabledFileSizeValidation() throws Exception {
    // Create an S3 storage with file existence validation enabled, but without path prefix validation and file size validation.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageDaoTestHelper.getS3ManagedBucketName()), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), Boolean.toString(false)), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), Boolean.toString(true)), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_SIZE), Boolean.toString(false))));
    // Create and persist a storage unit entity without a storage directory path.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.UPLOADING, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Create and persist a storage file with file size that would not match S3 reported size.
    StorageFileEntity storageFileEntity = storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, testS3KeyPrefix + "/" + FILE_PATH_1, FILE_SIZE_2_KB, ROW_COUNT_1000);
    storageUnitEntity.getStorageFiles().add(storageFileEntity);
    // Create and upload to S3 managed storage two test files with 1 KB file size.
    businessObjectDataServiceTestHelper.prepareTestS3Files(testS3KeyPrefix, localTempPath, Arrays.asList(FILE_PATH_1, FILE_PATH_2));
    // Add a second storage file to this business object data.
    BusinessObjectDataStorageFilesCreateRequest request = new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, null, DATA_VERSION, STORAGE_NAME, Arrays.asList(createFile(testS3KeyPrefix + "/" + FILE_PATH_2, FILE_SIZE_1_KB, ROW_COUNT_1000)), NO_DISCOVER_STORAGE_FILES);
    BusinessObjectDataStorageFilesCreateResponse response = businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(request);
    // Validate the returned object.
    businessObjectDataServiceTestHelper.validateBusinessObjectDataStorageFilesCreateResponse(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME, request.getStorageFiles(), response);
}

Example 13 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesAutoDiscoveryStorageDirectoryPathMatchesAnotherBdataStorageFiles.

@Test
public void testCreateBusinessObjectDataStorageFilesAutoDiscoveryStorageDirectoryPathMatchesAnotherBdataStorageFiles() throws Exception {
    // Create test data.
    createData(testS3KeyPrefix, true, Arrays.asList(testS3KeyPrefix + "/" + FILE_PATH_1));
    // Create and persist a storage unit entity for another business object data that would also have a storage file starting with the test S3 key prefix.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(StorageEntity.MANAGED_STORAGE, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_2, NO_SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.UPLOADING, StorageUnitStatusEntity.ENABLED, testS3KeyPrefix);
    // Create and persist a storage file for the second business object data that is also starting with the test S3 key prefix.
    StorageFileEntity storageFileEntity = storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, testS3KeyPrefix + "/" + FILE_PATH_2, FILE_SIZE_1_KB, NO_ROW_COUNT);
    storageUnitEntity.getStorageFiles().add(storageFileEntity);
    // Try to discover storage files when another business object data have a storage file starting with the test S3 key prefix.
    try {
        businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, StorageEntity.MANAGED_STORAGE, NO_STORAGE_FILES, DISCOVER_STORAGE_FILES));
        fail("Should throw an IllegalArgumentException when another business object data have a storage file starting with the test S3 key prefix.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Number of storage files (1) already registered for the business object data in \"%s\" storage is not equal " + "to the number of registered storage files (2) matching \"%s/\" S3 key prefix in the same storage.", StorageEntity.MANAGED_STORAGE, testS3KeyPrefix), e.getMessage());
    }
}

Example 14 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StorageFileDaoImpl method getStorageFilePathsByStorageUnitIds.

@Override
public MultiValuedMap<Integer, String> getStorageFilePathsByStorageUnitIds(List<Integer> storageUnitIds) {
    // Create a map that can hold a collection of values against each key.
    MultiValuedMap<Integer, String> result = new ArrayListValuedHashMap<>();
    // Retrieve the pagination size for the storage file paths query configured in the system.
    Integer paginationSize = configurationHelper.getProperty(ConfigurationValue.STORAGE_FILE_PATHS_QUERY_PAGINATION_SIZE, Integer.class);
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<Tuple> criteria = builder.createTupleQuery();
    // The criteria root is the storage file.
    Root<StorageFileEntity> storageFileEntity = criteria.from(StorageFileEntity.class);
    // Get the columns.
    Path<Integer> storageUnitIdColumn = storageFileEntity.get(StorageFileEntity_.storageUnitId);
    Path<String> storageFilePathColumn = storageFileEntity.get(StorageFileEntity_.path);
    // Create the standard restrictions (i.e. the standard where clauses).
    Predicate queryRestriction = getPredicateForInClause(builder, storageUnitIdColumn, storageUnitIds);
    // Add the select clause.
    criteria.multiselect(storageUnitIdColumn, storageFilePathColumn);
    // Add the where clause.
    criteria.where(queryRestriction);
    // Execute the query using pagination and populate the result map.
    int startPosition = 0;
    while (true) {
        // Run the query to get a list of tuples back.
        List<Tuple> tuples = entityManager.createQuery(criteria).setFirstResult(startPosition).setMaxResults(paginationSize).getResultList();
        // Populate the result map from the returned tuples (i.e. 1 tuple for each row).
        for (Tuple tuple : tuples) {
            // Extract the tuple values.
            Integer storageUnitId = tuple.get(storageUnitIdColumn);
            String storageFilePath = tuple.get(storageFilePathColumn);
            // Update the result map.
            result.put(storageUnitId, storageFilePath);
        }
        // Break out of the while loop if we got less results than the pagination size.
        if (tuples.size() < paginationSize) {
            break;
        }
        // Increment the start position.
        startPosition += paginationSize;
    }
    return result;
}

Example 15 with StorageFileEntity

use of org.finra.herd.model.jpa.StorageFileEntity in project herd by FINRAOS.

the class StorageFileDaoImpl method getStorageFileCount.

@Override
public Long getStorageFileCount(String storageName, String filePathPrefix) {
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<Long> criteria = builder.createQuery(Long.class);
    // The criteria root is the storage files.
    Root<StorageFileEntity> storageFileEntity = criteria.from(StorageFileEntity.class);
    // Join to the other tables we can filter on.
    Join<StorageFileEntity, StorageUnitEntity> storageUnitEntity = storageFileEntity.join(StorageFileEntity_.storageUnit);
    Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
    // Create path.
    Expression<Long> storageFileCount = builder.count(storageFileEntity.get(StorageFileEntity_.id));
    // Create the standard restrictions (i.e. the standard where clauses).
    Predicate storageNameRestriction = builder.equal(builder.upper(storageEntity.get(StorageEntity_.name)), storageName.toUpperCase());
    Predicate filePathRestriction = builder.like(storageFileEntity.get(StorageFileEntity_.path), String.format("%s%%", filePathPrefix));
    // Add the clauses for the query.
    criteria.select(storageFileCount).where(builder.and(storageNameRestriction, filePathRestriction));
    return entityManager.createQuery(criteria).getSingleResult();
}