Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 26 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class DataBridgeWebClient method addStorageFiles.

/**
 * Calls the registration server to add storage files to the business object data.
 *
 * @param businessObjectDataKey the business object data key
 * @param manifest the uploader input manifest file
 * @param s3FileTransferRequestParamsDto the S3 file transfer request parameters to be used to retrieve local path and S3 key prefix values
 * @param storageName the storage name
 *
 * @return the business object data create storage files response turned by the registration server.
 * @throws IOException if an I/O error was encountered.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = "VA_FORMAT_STRING_USES_NEWLINE", justification = "We will use the standard carriage return character.")
public BusinessObjectDataStorageFilesCreateResponse addStorageFiles(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, String storageName) throws IOException, JAXBException, URISyntaxException {
    LOGGER.info("Adding storage files to the business object data ...");
    BusinessObjectDataStorageFilesCreateRequest request = new BusinessObjectDataStorageFilesCreateRequest();
    request.setNamespace(businessObjectDataKey.getNamespace());
    request.setBusinessObjectDefinitionName(businessObjectDataKey.getBusinessObjectDefinitionName());
    request.setBusinessObjectFormatUsage(businessObjectDataKey.getBusinessObjectFormatUsage());
    request.setBusinessObjectFormatFileType(businessObjectDataKey.getBusinessObjectFormatFileType());
    request.setBusinessObjectFormatVersion(businessObjectDataKey.getBusinessObjectFormatVersion());
    request.setPartitionValue(businessObjectDataKey.getPartitionValue());
    request.setSubPartitionValues(businessObjectDataKey.getSubPartitionValues());
    request.setBusinessObjectDataVersion(businessObjectDataKey.getBusinessObjectDataVersion());
    request.setStorageName(storageName);
    List<StorageFile> storageFiles = new ArrayList<>();
    request.setStorageFiles(storageFiles);
    String localPath = s3FileTransferRequestParamsDto.getLocalPath();
    String s3KeyPrefix = s3FileTransferRequestParamsDto.getS3KeyPrefix();
    List<ManifestFile> localFiles = manifest.getManifestFiles();
    for (ManifestFile manifestFile : localFiles) {
        StorageFile storageFile = new StorageFile();
        storageFiles.add(storageFile);
        // Since the S3 key prefix represents a directory it is expected to contain a trailing '/' character.
        storageFile.setFilePath((s3KeyPrefix + manifestFile.getFileName()).replaceAll("\\\\", "/"));
        storageFile.setFileSizeBytes(Paths.get(localPath, manifestFile.getFileName()).toFile().length());
        storageFile.setRowCount(manifestFile.getRowCount());
    }
    // Create a JAXB context and marshaller
    JAXBContext requestContext = JAXBContext.newInstance(BusinessObjectDataStorageFilesCreateRequest.class);
    Marshaller requestMarshaller = requestContext.createMarshaller();
    requestMarshaller.setProperty(Marshaller.JAXB_ENCODING, StandardCharsets.UTF_8.name());
    requestMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
    StringWriter sw = new StringWriter();
    requestMarshaller.marshal(request, sw);
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse;
    try (CloseableHttpClient client = httpClientOperations.createHttpClient()) {
        URI uri = new URIBuilder().setScheme(getUriScheme()).setHost(regServerAccessParamsDto.getRegServerHost()).setPort(regServerAccessParamsDto.getRegServerPort()).setPath(HERD_APP_REST_URI_PREFIX + "/businessObjectDataStorageFiles").build();
        HttpPost post = new HttpPost(uri);
        post.addHeader("Content-Type", DEFAULT_CONTENT_TYPE);
        post.addHeader("Accepts", DEFAULT_ACCEPT);
        // If SSL is enabled, set the client authentication header.
        if (regServerAccessParamsDto.isUseSsl()) {
            post.addHeader(getAuthorizationHeader());
        }
        post.setEntity(new StringEntity(sw.toString()));
        LOGGER.info(String.format("    HTTP POST URI: %s", post.getURI().toString()));
        LOGGER.info(String.format("    HTTP POST Headers: %s", Arrays.toString(post.getAllHeaders())));
        LOGGER.info(String.format("    HTTP POST Entity Content:%n%s", sw.toString()));
        // getBusinessObjectDataStorageFilesCreateResponse() might return a null. That happens when the web client gets status code 200 back from
        // the service (add storage files is a success), but it fails to retrieve or deserialize the actual HTTP response.
        // Please note that processXmlHttpResponse() is responsible for logging the exception info as a warning.
        businessObjectDataStorageFilesCreateResponse = getBusinessObjectDataStorageFilesCreateResponse(httpClientOperations.execute(client, post));
    }
    LOGGER.info("Successfully added storage files to the registered business object data.");
    return businessObjectDataStorageFilesCreateResponse;
}

Example 27 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceCreateBusinessObjectDataTest method testCreateBusinessObjectDataDiscoverStorageFilesStorageFilesSpecified.

@Test
public void testCreateBusinessObjectDataDiscoverStorageFilesStorageFilesSpecified() {
    // Try to create an initial version of the business object data when discovery of storage files is enabled and storage files are specified.
    try {
        businessObjectDataService.createBusinessObjectData(new BusinessObjectDataCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, NO_SUBPARTITION_VALUES, BusinessObjectDataStatusEntity.VALID, Arrays.asList(new StorageUnitCreateRequest(STORAGE_NAME, new StorageDirectory(STORAGE_DIRECTORY_PATH), Arrays.asList(new StorageFile(LOCAL_FILE, FILE_SIZE_1_KB, ROW_COUNT_1000)), DISCOVER_STORAGE_FILES)), NO_ATTRIBUTES, NO_BUSINESS_OBJECT_DATA_PARENTS, NO_CREATE_NEW_VERSION));
        fail("Should throw an IllegalArgumentException when discovery of storage files is enabled and storage files are specified.");
    } catch (IllegalArgumentException e) {
        assertEquals("Storage files cannot be specified when discovery of storage files is enabled.", e.getMessage());
    }
}

Example 28 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceCreateBusinessObjectDataTest method testCreateBusinessObjectDataInitialDataVersionExists.

@Test
public void testCreateBusinessObjectDataInitialDataVersionExists() {
    // Create relative database entities including the initial version of the business object data.
    businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME);
    // Build a list of storage files.
    List<StorageFile> storageFiles = businessObjectDataServiceTestHelper.getTestStorageFiles(testS3KeyPrefix, SORTED_LOCAL_FILES);
    // Build a new business object data create request.
    BusinessObjectDataCreateRequest request = businessObjectDataServiceTestHelper.createBusinessObjectDataCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, BDATA_STATUS, STORAGE_NAME, testS3KeyPrefix, storageFiles);
    // Try to create the second version of the business object data (version 1) with createNewVersion flag set to null and to false.
    for (Boolean createNewVersionFlag : new Boolean[] { null, Boolean.FALSE }) {
        request.setCreateNewVersion(createNewVersionFlag);
        try {
            businessObjectDataService.createBusinessObjectData(request);
            fail(String.format("Should throw an IllegalArgumentException when the initial data version exists and createNewVersion flag is set to \"%s\".", createNewVersionFlag));
        } catch (AlreadyExistsException e) {
            assertEquals("Unable to create business object data because it already exists.", e.getMessage());
        }
    }
    // Try to create the second version of the business object data with createNewVersion flag set to true.
    request.setCreateNewVersion(true);
    BusinessObjectData resultBusinessObjectData = businessObjectDataService.createBusinessObjectData(request);
    // Validate the results for the second version if the business object data.
    businessObjectDataServiceTestHelper.validateBusinessObjectData(request, SECOND_DATA_VERSION, true, resultBusinessObjectData);
    // Confirm that the initial version of the business object data now does not have the latestFlag set.
    BusinessObjectDataEntity initialVersionBusinessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION));
    assertEquals(false, initialVersionBusinessObjectDataEntity.getLatestVersion());
}

Example 29 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceCreateBusinessObjectDataTest method testCreateBusinessObjectDataValidateFileExistenceNoValidatePrefix.

@Test
public void testCreateBusinessObjectDataValidateFileExistenceNoValidatePrefix() throws Exception {
    // Create an S3 storage entity.
    List<Attribute> attributes = new ArrayList<>();
    attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME));
    attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), Boolean.TRUE.toString()));
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, attributes);
    // Create relative database entities.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, true, PARTITION_KEY);
    businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS);
    // Build a list of storage files.
    List<StorageFile> storageFiles = businessObjectDataServiceTestHelper.getTestStorageFiles(testS3KeyPrefix, LOCAL_FILES);
    // Create and upload to S3 managed storage a set of test files.
    businessObjectDataServiceTestHelper.prepareTestS3Files(S3_BUCKET_NAME, testS3KeyPrefix, localTempPath, LOCAL_FILES, new ArrayList<String>());
    // Build a new business object data create request.
    BusinessObjectDataCreateRequest request = businessObjectDataServiceTestHelper.createBusinessObjectDataCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, BDATA_STATUS, STORAGE_NAME, testS3KeyPrefix, storageFiles);
    // Try to create a business object data instance. It should go through without any errors.
    businessObjectDataService.createBusinessObjectData(request);
}

Example 30 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceCreateBusinessObjectDataTest method testCreateBusinessObjectDataValidateFlagIgnoredForNonS3Storage.

@Test
public void testCreateBusinessObjectDataValidateFlagIgnoredForNonS3Storage() throws Exception {
    // Create an S3 storage entity.
    List<Attribute> attributes = new ArrayList<>();
    attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME));
    attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), Boolean.TRUE.toString()));
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME, "NON_S3_STORAGE", attributes);
    // Create relative database entities.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, true, PARTITION_KEY);
    businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS);
    // Build a list of storage files.
    List<StorageFile> storageFiles = businessObjectDataServiceTestHelper.getTestStorageFiles(testS3KeyPrefix, LOCAL_FILES);
    // Build a new business object data create request.
    BusinessObjectDataCreateRequest request = businessObjectDataServiceTestHelper.createBusinessObjectDataCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, BDATA_STATUS, STORAGE_NAME, testS3KeyPrefix, storageFiles);
    // Try to create a business object data instance. This should succeed since the bucket is not of type "S3", even though there are no files in S3.
    businessObjectDataService.createBusinessObjectData(request);
}