Examples with BusinessObjectDataStorageFilesCreateRequest org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest used on opensource projects

Example 1 with BusinessObjectDataStorageFilesCreateRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest in project herd by FINRAOS.

the class AddBusinessObjectDataStorageFiles method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    String contentTypeString = activitiHelper.getRequiredExpressionVariableAsString(contentType, execution, "ContentType").trim();
    String requestString = activitiHelper.getRequiredExpressionVariableAsString(businessObjectDataStorageFilesCreateRequest, execution, "BusinessObjectDataCreateRequest").trim();
    BusinessObjectDataStorageFilesCreateRequest request = getRequestObject(contentTypeString, requestString, BusinessObjectDataStorageFilesCreateRequest.class);
    // Call the BusinessObjectDataStorageFiles service.
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse = businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(request);
    // Set the JSON response as a workflow variable.
    setJsonResponseAsWorkflowVariable(businessObjectDataStorageFilesCreateResponse, execution);
}

Example 2 with BusinessObjectDataStorageFilesCreateRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest in project herd by FINRAOS.

the class DataBridgeWebClient method addStorageFiles.

/**
 * Calls the registration server to add storage files to the business object data.
 *
 * @param businessObjectDataKey the business object data key
 * @param manifest the uploader input manifest file
 * @param s3FileTransferRequestParamsDto the S3 file transfer request parameters to be used to retrieve local path and S3 key prefix values
 * @param storageName the storage name
 *
 * @return the business object data create storage files response turned by the registration server.
 * @throws IOException if an I/O error was encountered.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = "VA_FORMAT_STRING_USES_NEWLINE", justification = "We will use the standard carriage return character.")
public BusinessObjectDataStorageFilesCreateResponse addStorageFiles(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, String storageName) throws IOException, JAXBException, URISyntaxException {
    LOGGER.info("Adding storage files to the business object data ...");
    BusinessObjectDataStorageFilesCreateRequest request = new BusinessObjectDataStorageFilesCreateRequest();
    request.setNamespace(businessObjectDataKey.getNamespace());
    request.setBusinessObjectDefinitionName(businessObjectDataKey.getBusinessObjectDefinitionName());
    request.setBusinessObjectFormatUsage(businessObjectDataKey.getBusinessObjectFormatUsage());
    request.setBusinessObjectFormatFileType(businessObjectDataKey.getBusinessObjectFormatFileType());
    request.setBusinessObjectFormatVersion(businessObjectDataKey.getBusinessObjectFormatVersion());
    request.setPartitionValue(businessObjectDataKey.getPartitionValue());
    request.setSubPartitionValues(businessObjectDataKey.getSubPartitionValues());
    request.setBusinessObjectDataVersion(businessObjectDataKey.getBusinessObjectDataVersion());
    request.setStorageName(storageName);
    List<StorageFile> storageFiles = new ArrayList<>();
    request.setStorageFiles(storageFiles);
    String localPath = s3FileTransferRequestParamsDto.getLocalPath();
    String s3KeyPrefix = s3FileTransferRequestParamsDto.getS3KeyPrefix();
    List<ManifestFile> localFiles = manifest.getManifestFiles();
    for (ManifestFile manifestFile : localFiles) {
        StorageFile storageFile = new StorageFile();
        storageFiles.add(storageFile);
        // Since the S3 key prefix represents a directory it is expected to contain a trailing '/' character.
        storageFile.setFilePath((s3KeyPrefix + manifestFile.getFileName()).replaceAll("\\\\", "/"));
        storageFile.setFileSizeBytes(Paths.get(localPath, manifestFile.getFileName()).toFile().length());
        storageFile.setRowCount(manifestFile.getRowCount());
    }
    // Create a JAXB context and marshaller
    JAXBContext requestContext = JAXBContext.newInstance(BusinessObjectDataStorageFilesCreateRequest.class);
    Marshaller requestMarshaller = requestContext.createMarshaller();
    requestMarshaller.setProperty(Marshaller.JAXB_ENCODING, StandardCharsets.UTF_8.name());
    requestMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
    StringWriter sw = new StringWriter();
    requestMarshaller.marshal(request, sw);
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse;
    try (CloseableHttpClient client = httpClientOperations.createHttpClient()) {
        URI uri = new URIBuilder().setScheme(getUriScheme()).setHost(regServerAccessParamsDto.getRegServerHost()).setPort(regServerAccessParamsDto.getRegServerPort()).setPath(HERD_APP_REST_URI_PREFIX + "/businessObjectDataStorageFiles").build();
        HttpPost post = new HttpPost(uri);
        post.addHeader("Content-Type", DEFAULT_CONTENT_TYPE);
        post.addHeader("Accepts", DEFAULT_ACCEPT);
        // If SSL is enabled, set the client authentication header.
        if (regServerAccessParamsDto.isUseSsl()) {
            post.addHeader(getAuthorizationHeader());
        }
        post.setEntity(new StringEntity(sw.toString()));
        LOGGER.info(String.format("    HTTP POST URI: %s", post.getURI().toString()));
        LOGGER.info(String.format("    HTTP POST Headers: %s", Arrays.toString(post.getAllHeaders())));
        LOGGER.info(String.format("    HTTP POST Entity Content:%n%s", sw.toString()));
        // getBusinessObjectDataStorageFilesCreateResponse() might return a null. That happens when the web client gets status code 200 back from
        // the service (add storage files is a success), but it fails to retrieve or deserialize the actual HTTP response.
        // Please note that processXmlHttpResponse() is responsible for logging the exception info as a warning.
        businessObjectDataStorageFilesCreateResponse = getBusinessObjectDataStorageFilesCreateResponse(httpClientOperations.execute(client, post));
    }
    LOGGER.info("Successfully added storage files to the registered business object data.");
    return businessObjectDataStorageFilesCreateResponse;
}

Example 3 with BusinessObjectDataStorageFilesCreateRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesStorageHasValidateFileSizeEnabledWithoutValidateFileExistence.

@Test
public void testCreateBusinessObjectDataStorageFilesStorageHasValidateFileSizeEnabledWithoutValidateFileExistence() {
    // Create an S3 storage with file size validation enabled without file existence validation.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME_2), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_SIZE), Boolean.toString(true))));
    // Create a storage unit.
    storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.UPLOADING, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Try to add storage files to an S3 storage with file existence validation enabled without file size validation.
    try {
        businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME, Arrays.asList(createFile(FILE_PATH_1, FILE_SIZE_1_KB, ROW_COUNT_1000)), NO_DISCOVER_STORAGE_FILES));
        fail("Should throw an IllegalArgumentException when adding files to storage with file existence validation enabled without file size validation.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage \"%s\" has file size validation enabled without file existence validation.", STORAGE_NAME), e.getMessage());
    }
}

Example 4 with BusinessObjectDataStorageFilesCreateRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesAutoDiscoveryStorageFilesSpecified.

@Test
public void testCreateBusinessObjectDataStorageFilesAutoDiscoveryStorageFilesSpecified() {
    // Try to create storage files when discovery of storage files is enabled and storage files are specified.
    try {
        businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, StorageEntity.MANAGED_STORAGE, Arrays.asList(createFile(FILE_PATH_1, FILE_SIZE_0_BYTE, ROW_COUNT_1000)), DISCOVER_STORAGE_FILES));
        fail("Should throw an IllegalArgumentException when discovery of storage files is enabled and storage files are specified.");
    } catch (IllegalArgumentException e) {
        assertEquals("Storage files cannot be specified when discovery of storage files is enabled.", e.getMessage());
    }
}

Example 5 with BusinessObjectDataStorageFilesCreateRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataStorageFilesCreateRequest in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesS3ManagedPreviouslyRegisteredS3FileSizeMismatch.

@Test
public void testCreateBusinessObjectDataStorageFilesS3ManagedPreviouslyRegisteredS3FileSizeMismatch() throws Exception {
    // Create and persist a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(StorageEntity.MANAGED_STORAGE, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.UPLOADING, StorageUnitStatusEntity.ENABLED, testS3KeyPrefix);
    // Create and persist a storage file with file size that would not match S3 reported size.
    StorageFileEntity storageFileEntity = storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, testS3KeyPrefix + "/" + FILE_PATH_1, FILE_SIZE_2_KB, ROW_COUNT_1000);
    storageUnitEntity.getStorageFiles().add(storageFileEntity);
    // Create and upload to S3 managed storage two test files with 1 KB file size.
    businessObjectDataServiceTestHelper.prepareTestS3Files(testS3KeyPrefix, localTempPath, Arrays.asList(FILE_PATH_1, FILE_PATH_2));
    // Try to add storage file to the business object data when file size reported by S3 does not match for an already registered storage file.
    try {
        businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, null, DATA_VERSION, StorageEntity.MANAGED_STORAGE, Arrays.asList(createFile(testS3KeyPrefix + "/" + FILE_PATH_2, FILE_SIZE_1_KB, ROW_COUNT_1000)), NO_DISCOVER_STORAGE_FILES));
        fail("Should throw an IllegalArgumentException when an already registered storage file size does not match file size reported by S3.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Previously registered storage file \"%s/%s\" has file size of %d bytes that does not match file size of %d bytes reported by S3.", testS3KeyPrefix, FILE_PATH_1, FILE_SIZE_2_KB, FILE_SIZE_1_KB), e.getMessage());
    }
}