Examples with ObjectNotFoundException org.finra.herd.model.ObjectNotFoundException used on opensource projects

Example 1 with ObjectNotFoundException

use of org.finra.herd.model.ObjectNotFoundException in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method discoverStorageFiles.

/**
 * Discovers storage files in the specified S3 storage that match the S3 key prefix.
 *
 * @param storageEntity the storage entity
 * @param s3KeyPrefix the S3 key prefix
 *
 * @return the list of discovered storage files
 */
private List<StorageFile> discoverStorageFiles(StorageEntity storageEntity, String s3KeyPrefix) {
    // Only S3 storage platform is currently supported for storage file discovery.
    Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot discover storage files at \"%s\" storage platform.", storageEntity.getStoragePlatform().getName()));
    // Get S3 bucket access parameters.
    S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity);
    // Retrieve a list of all keys/objects from the S3 bucket matching the specified S3 key prefix.
    // Since S3 key prefix represents the directory, we add a trailing '/' character to it, unless it is already present.
    params.setS3KeyPrefix(StringUtils.appendIfMissing(s3KeyPrefix, "/"));
    // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params, true);
    // Fail registration if no storage files were discovered.
    if (CollectionUtils.isEmpty(s3ObjectSummaries)) {
        throw new ObjectNotFoundException(String.format("Found no files at \"s3://%s/%s\" location.", params.getS3BucketName(), params.getS3KeyPrefix()));
    }
    return storageFileHelper.createStorageFilesFromS3ObjectSummaries(s3ObjectSummaries);
}

Example 2 with ObjectNotFoundException

use of org.finra.herd.model.ObjectNotFoundException in project herd by FINRAOS.

the class BusinessObjectDataAttributeDaoHelper method getBusinessObjectDataAttributeEntity.

/**
 * Gets a business object data attribute entity on the key and makes sure that it exists.
 *
 * @param businessObjectDataAttributeKey the business object data attribute key
 *
 * @return the business object data attribute entity
 * @throws ObjectNotFoundException if the business object data or the business object data attribute don't exist
 */
public BusinessObjectDataAttributeEntity getBusinessObjectDataAttributeEntity(BusinessObjectDataAttributeKey businessObjectDataAttributeKey) throws ObjectNotFoundException {
    // Get the business object data and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(new BusinessObjectDataKey(businessObjectDataAttributeKey.getNamespace(), businessObjectDataAttributeKey.getBusinessObjectDefinitionName(), businessObjectDataAttributeKey.getBusinessObjectFormatUsage(), businessObjectDataAttributeKey.getBusinessObjectFormatFileType(), businessObjectDataAttributeKey.getBusinessObjectFormatVersion(), businessObjectDataAttributeKey.getPartitionValue(), businessObjectDataAttributeKey.getSubPartitionValues(), businessObjectDataAttributeKey.getBusinessObjectDataVersion()));
    // Load all existing business object data attribute entities into a map for quick access using lowercase attribute names.
    Map<String, BusinessObjectDataAttributeEntity> businessObjectDataAttributeEntityMap = getBusinessObjectDataAttributeEntityMap(businessObjectDataEntity.getAttributes());
    // Get the relative entity using the attribute name in lowercase.
    BusinessObjectDataAttributeEntity businessObjectDataAttributeEntity = businessObjectDataAttributeEntityMap.get(businessObjectDataAttributeKey.getBusinessObjectDataAttributeName().toLowerCase());
    if (businessObjectDataAttributeEntity == null) {
        throw new ObjectNotFoundException(String.format("Attribute with name \"%s\" does not exist for business object data {%s}.", businessObjectDataAttributeKey.getBusinessObjectDataAttributeName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
    }
    return businessObjectDataAttributeEntity;
}

Example 3 with ObjectNotFoundException

use of org.finra.herd.model.ObjectNotFoundException in project herd by FINRAOS.

the class Hive13DdlGenerator method processPartitionFiltersForGenerateDdl.

/**
 * Processes partition filters for DDL generation as per generate DDL request.
 *
 * @param generateDdlRequest the generate DDL request
 * @param sb the string builder to be updated with the "alter table add partition" statements
 * @param replacements the hash map of string values to be used to substitute the custom DDL tokens with their actual values
 * @param businessObjectFormatEntity the business object format entity
 * @param businessObjectFormat the business object format
 * @param ifNotExistsOption specifies if generated DDL contains "if not exists" option
 */
private void processPartitionFiltersForGenerateDdl(GenerateDdlRequest generateDdlRequest, StringBuilder sb, HashMap<String, String> replacements, BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectFormat businessObjectFormat, String ifNotExistsOption) {
    // Get the business object format key from the entity.
    BusinessObjectFormatKey businessObjectFormatKey = businessObjectFormatHelper.getBusinessObjectFormatKey(generateDdlRequest.businessObjectFormatEntity);
    // Override the business object format version with the original (optional) value from the request.
    businessObjectFormatKey.setBusinessObjectFormatVersion(generateDdlRequest.businessObjectFormatVersion);
    // Retrieve a list of storage unit entities for the specified list of partition filters. The entities will be sorted by partition values and storages.
    // For a non-partitioned table, there should only exist a single business object data entity (with partitionValue equals to "none"). We do validate that
    // all specified storages are of "S3" storage platform type, so we specify S3 storage platform type in the herdDao call below, so we select storage
    // units only from all S3 storages, when the specified list of storages is empty. We also specify to select only "available" storage units.
    List<StorageUnitEntity> storageUnitEntities = storageUnitDao.getStorageUnitsByPartitionFiltersAndStorages(businessObjectFormatKey, generateDdlRequest.partitionFilters, generateDdlRequest.businessObjectDataVersion, BusinessObjectDataStatusEntity.VALID, generateDdlRequest.storageNames, StoragePlatformEntity.S3, null, true);
    // Exclude duplicate business object data per specified list of storage names.
    // If storage names are not specified, the method fails on business object data instances registered with multiple storages.
    storageUnitEntities = excludeDuplicateBusinessObjectData(storageUnitEntities, generateDdlRequest.storageNames, generateDdlRequest.storageEntities);
    // Build a list of matched partition filters. Please note that each request partition
    // filter might result in multiple available business object data entities.
    List<List<String>> matchedAvailablePartitionFilters = new ArrayList<>();
    List<List<String>> availablePartitions = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData());
        matchedAvailablePartitionFilters.add(businessObjectDataHelper.getPartitionFilter(businessObjectDataKey, generateDdlRequest.partitionFilters.get(0)));
        availablePartitions.add(businessObjectDataHelper.getPrimaryAndSubPartitionValues(businessObjectDataKey));
    }
    // If request specifies to include all registered sub-partitions, fail if any of "non-available" registered sub-partitions are found.
    if (generateDdlRequest.businessObjectDataVersion == null && BooleanUtils.isTrue(generateDdlRequest.includeAllRegisteredSubPartitions) && !CollectionUtils.isEmpty(matchedAvailablePartitionFilters)) {
        notAllowNonAvailableRegisteredSubPartitions(businessObjectFormatKey, matchedAvailablePartitionFilters, availablePartitions, generateDdlRequest.storageNames);
    }
    // Fail on any missing business object data unless the flag is set to allow missing business object data.
    if (!BooleanUtils.isTrue(generateDdlRequest.allowMissingData)) {
        // Get a list of unmatched partition filters.
        List<List<String>> unmatchedPartitionFilters = new ArrayList<>(generateDdlRequest.partitionFilters);
        unmatchedPartitionFilters.removeAll(matchedAvailablePartitionFilters);
        // Throw an exception if we have any unmatched partition filters.
        if (!unmatchedPartitionFilters.isEmpty()) {
            // Get the first unmatched partition filter and throw exception.
            List<String> unmatchedPartitionFilter = getFirstUnmatchedPartitionFilter(unmatchedPartitionFilters);
            throw new ObjectNotFoundException(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d, partitionValue: \"%s\", " + "subpartitionValues: \"%s\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", businessObjectFormatKey.getNamespace(), businessObjectFormatKey.getBusinessObjectDefinitionName(), businessObjectFormatKey.getBusinessObjectFormatUsage(), businessObjectFormatKey.getBusinessObjectFormatFileType(), businessObjectFormatKey.getBusinessObjectFormatVersion(), unmatchedPartitionFilter.get(0), StringUtils.join(unmatchedPartitionFilter.subList(1, unmatchedPartitionFilter.size()), ","), generateDdlRequest.businessObjectDataVersion, StringUtils.join(generateDdlRequest.storageNames, ",")));
        }
    }
    // the table is non-partitioned, and there is no business object data found.
    if (generateDdlRequest.customDdlEntity == null && !generateDdlRequest.isPartitioned && CollectionUtils.isEmpty(storageUnitEntities)) {
        // Add a LOCATION clause with a token.
        sb.append(String.format("LOCATION '%s';", NON_PARTITIONED_TABLE_LOCATION_CUSTOM_DDL_TOKEN));
    } else // The table is partitioned, custom DDL is specified, or there is at least one business object data instance found.
    {
        // If drop partitions flag is set and the table is partitioned, drop partitions specified by the partition filters.
        if (generateDdlRequest.isPartitioned && BooleanUtils.isTrue(generateDdlRequest.includeDropPartitions)) {
            // Add a drop partition statement for each partition filter entry.
            for (List<String> partitionFilter : generateDdlRequest.partitionFilters) {
                sb.append(String.format("ALTER TABLE `%s` DROP IF EXISTS PARTITION (", generateDdlRequest.tableName));
                // Specify all partition column values as per this partition filter.
                List<String> partitionKeyValuePairs = new ArrayList<>();
                for (int i = 0; i < partitionFilter.size(); i++) {
                    if (StringUtils.isNotBlank(partitionFilter.get(i))) {
                        // We cannot hit ArrayIndexOutOfBoundsException on getPartitions() since partitionFilter would
                        // not have a value set at an index that is greater or equal than the number of partitions in the schema.
                        String partitionColumnName = businessObjectFormat.getSchema().getPartitions().get(i).getName();
                        partitionKeyValuePairs.add(String.format("`%s`='%s'", partitionColumnName, partitionFilter.get(i)));
                    }
                }
                sb.append(StringUtils.join(partitionKeyValuePairs, ", "));
                sb.append(");\n");
            }
            sb.append('\n');
        }
        // Process storage unit entities.
        if (!CollectionUtils.isEmpty(storageUnitEntities)) {
            processStorageUnitsForGenerateDdl(generateDdlRequest, sb, replacements, businessObjectFormatEntity, businessObjectFormat, ifNotExistsOption, storageUnitEntities);
        }
    }
}

Example 4 with ObjectNotFoundException

use of org.finra.herd.model.ObjectNotFoundException in project herd by FINRAOS.

the class Hive13DdlGenerator method notAllowNonAvailableRegisteredSubPartitions.

/**
 * Searches for and fails on any of "non-available" registered sub-partitions as per list of "matched" partition filters.
 *
 * @param businessObjectFormatKey the business object format key
 * @param matchedAvailablePartitionFilters the list of "matched" partition filters
 * @param availablePartitions the list of already discovered "available" partitions, where each partition consists of primary and optional sub-partition
 * values
 * @param storageNames the list of storage names
 */
protected void notAllowNonAvailableRegisteredSubPartitions(BusinessObjectFormatKey businessObjectFormatKey, List<List<String>> matchedAvailablePartitionFilters, List<List<String>> availablePartitions, List<String> storageNames) {
    // Query all matched partition filters to discover any non-available registered sub-partitions. Retrieve latest business object data per list of
    // matched filters regardless of business object data and/or storage unit statuses. This is done to discover all registered sub-partitions regardless
    // of business object data or storage unit statuses. We do validate that all specified storages are of "S3" storage platform type, so we specify S3
    // storage platform type in the herdDao call below, so we select storage units only from all S3 storages, when the specified list of storages is empty.
    // We want to select any existing storage units regardless of their status, so we pass "false" for selectOnlyAvailableStorageUnits parameter.
    List<StorageUnitEntity> matchedNotAvailableStorageUnitEntities = storageUnitDao.getStorageUnitsByPartitionFiltersAndStorages(businessObjectFormatKey, matchedAvailablePartitionFilters, null, null, storageNames, StoragePlatformEntity.S3, null, false);
    // Exclude all storage units with business object data having "DELETED" status.
    matchedNotAvailableStorageUnitEntities = storageUnitHelper.excludeBusinessObjectDataStatus(matchedNotAvailableStorageUnitEntities, BusinessObjectDataStatusEntity.DELETED);
    // Exclude all already discovered "available" partitions. Please note that, since we got here, the list of matched partitions can not be empty.
    matchedNotAvailableStorageUnitEntities = storageUnitHelper.excludePartitions(matchedNotAvailableStorageUnitEntities, availablePartitions);
    // Fail on any "non-available" registered sub-partitions.
    if (!CollectionUtils.isEmpty(matchedNotAvailableStorageUnitEntities)) {
        // Get the business object data key for the first "non-available" registered sub-partition.
        BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(matchedNotAvailableStorageUnitEntities.get(0).getBusinessObjectData());
        throw new ObjectNotFoundException(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d, partitionValue: \"%s\", " + "subpartitionValues: \"%s\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", businessObjectFormatKey.getNamespace(), businessObjectFormatKey.getBusinessObjectDefinitionName(), businessObjectFormatKey.getBusinessObjectFormatUsage(), businessObjectFormatKey.getBusinessObjectFormatFileType(), businessObjectFormatKey.getBusinessObjectFormatVersion(), businessObjectDataKey.getPartitionValue(), StringUtils.join(businessObjectDataKey.getSubPartitionValues(), ","), businessObjectDataKey.getBusinessObjectDataVersion(), StringUtils.join(storageNames, ",")));
    }
}

Example 5 with ObjectNotFoundException

use of org.finra.herd.model.ObjectNotFoundException in project herd by FINRAOS.

the class StorageUnitDaoHelperTest method testGetStorageUnitEntityByBusinessObjectDataAndStorageNameStorageUnitNoExists.

@Test
public void testGetStorageUnitEntityByBusinessObjectDataAndStorageNameStorageUnitNoExists() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist test database entities.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS, STORAGE_UNIT_STATUS, NO_STORAGE_DIRECTORY_PATH);
    // Get the business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    // Mock the external calls.
    when(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME)).thenReturn(null);
    when(businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING);
    // Try to call the method under test.
    try {
        storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
        fail();
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Could not find storage unit in \"%s\" storage for the business object data {%s}.", STORAGE_NAME, BUSINESS_OBJECT_DATA_KEY_AS_STRING), e.getMessage());
    }
    // Verify the external calls.
    verify(storageUnitDao).getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME);
    verify(businessObjectDataHelper).businessObjectDataEntityAltKeyToString(businessObjectDataEntity);
    verifyNoMoreInteractionsHelper();
}