Examples with S3KeyPrefixInformation org.finra.herd.model.api.xml.S3KeyPrefixInformation used on opensource projects

Example 6 with S3KeyPrefixInformation

use of org.finra.herd.model.api.xml.S3KeyPrefixInformation in project herd by FINRAOS.

the class DataBridgeWebClient method getS3KeyPrefix.

/**
 * Retrieves S3 key prefix from the registration server.
 *
 * @param manifest the manifest file information
 * @param businessObjectDataVersion the business object data version (optional)
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created.  This parameter is ignored, when the
 * business object data version is specified.
 *
 * @return the S3 key prefix
 * @throws IOException if an I/O error was encountered.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
protected S3KeyPrefixInformation getS3KeyPrefix(DataBridgeBaseManifestDto manifest, Integer businessObjectDataVersion, Boolean createNewVersion) throws IOException, JAXBException, URISyntaxException {
    LOGGER.info("Retrieving S3 key prefix from the registration server...");
    StringBuilder uriPathBuilder = new StringBuilder(151);
    uriPathBuilder.append(HERD_APP_REST_URI_PREFIX + "/businessObjectData");
    // The namespace is optional. If not specified, do not add to the REST URI.
    if (StringUtils.isNotBlank(manifest.getNamespace())) {
        uriPathBuilder.append("/namespaces/").append(manifest.getNamespace());
    }
    uriPathBuilder.append("/businessObjectDefinitionNames/").append(manifest.getBusinessObjectDefinitionName());
    uriPathBuilder.append("/businessObjectFormatUsages/").append(manifest.getBusinessObjectFormatUsage());
    uriPathBuilder.append("/businessObjectFormatFileTypes/").append(manifest.getBusinessObjectFormatFileType());
    uriPathBuilder.append("/businessObjectFormatVersions/").append(manifest.getBusinessObjectFormatVersion());
    uriPathBuilder.append("/s3KeyPrefix");
    String uriPath = uriPathBuilder.toString();
    URIBuilder uriBuilder = new URIBuilder().setScheme(getUriScheme()).setHost(regServerAccessParamsDto.getRegServerHost()).setPort(regServerAccessParamsDto.getRegServerPort()).setPath(uriPath).setParameter("partitionKey", manifest.getPartitionKey()).setParameter("partitionValue", manifest.getPartitionValue()).setParameter("createNewVersion", createNewVersion.toString());
    if (!CollectionUtils.isEmpty(manifest.getSubPartitionValues())) {
        uriBuilder.setParameter("subPartitionValues", herdStringHelper.join(manifest.getSubPartitionValues(), "|", "\\"));
    }
    if (businessObjectDataVersion != null) {
        uriBuilder.setParameter("businessObjectDataVersion", businessObjectDataVersion.toString());
    }
    if (StringUtils.isNotBlank(manifest.getStorageName())) {
        uriBuilder.setParameter("storageName", manifest.getStorageName());
    }
    S3KeyPrefixInformation s3KeyPrefixInformation;
    try (CloseableHttpClient client = httpClientOperations.createHttpClient()) {
        HttpGet request = new HttpGet(uriBuilder.build());
        request.addHeader("Accepts", DEFAULT_ACCEPT);
        // If SSL is enabled, set the client authentication header.
        if (regServerAccessParamsDto.isUseSsl()) {
            request.addHeader(getAuthorizationHeader());
        }
        LOGGER.info(String.format("    HTTP GET URI: %s", request.getURI().toString()));
        LOGGER.info(String.format("    HTTP GET Headers: %s", Arrays.toString(request.getAllHeaders())));
        s3KeyPrefixInformation = getS3KeyPrefixInformation(httpClientOperations.execute(client, request));
    }
    LOGGER.info("Successfully retrieved S3 key prefix from the registration server.");
    LOGGER.info("    S3 key prefix: " + s3KeyPrefixInformation.getS3KeyPrefix());
    return s3KeyPrefixInformation;
}

Example 7 with S3KeyPrefixInformation

use of org.finra.herd.model.api.xml.S3KeyPrefixInformation in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefix.

@Test
public void testGetS3KeyPrefix() {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, SUBPARTITION_VALUES.size() + 1);
    // Get an S3 key prefix by passing all parameters including partition key, business object data version,
    // and "create new version" flag (has no effect when data version is specified).
    S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
    // Get the expected S3 key prefix value using the business object data version.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), DATA_VERSION);
    // Validate the results.
    assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
}

Example 8 with S3KeyPrefixInformation

use of org.finra.herd.model.api.xml.S3KeyPrefixInformation in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixUpperCaseParameters.

@Test
public void testGetS3KeyPrefixUpperCaseParameters() {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, SUBPARTITION_VALUES.size() + 1);
    // Get an S3 key prefix using upper case input parameters (except for case-sensitive partition values).
    S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE.toUpperCase(), BDEF_NAME.toUpperCase(), FORMAT_USAGE_CODE.toUpperCase(), FORMAT_FILE_TYPE_CODE.toUpperCase(), FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey.toUpperCase(), STORAGE_NAME.toUpperCase(), false);
    // Get the expected S3 key prefix value using the business object data version.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), DATA_VERSION);
    // Validate the results.
    assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
}

Example 9 with S3KeyPrefixInformation

use of org.finra.herd.model.api.xml.S3KeyPrefixInformation in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixNoDataVersionSpecifiedInitialDataVersionNoExists.

/**
 * <p> Tests cases where get S3 key prefix is called with create new version flag set to true and false, and the data does not exist. </p> <h1>Case 1</h1>
 * <ul> <li>Given: <ul> <li>A business object format</li> <li>No business object data</li> </ul> </li> <li>When: <ul> <li>Get S3 key prefix is called with
 * create new version flag set to false</li> </ul> </li> <li>Then: <ul> <li>The S3 key prefix should have data version set to 0</li> </ul> </li> </ul>
 * <h1>Case 2</h1> <ul> <li>Given: <ul> <li>Same as case 1</li> </ul> </li> <li>When: <ul> <li>Get S3 key prefix is called with create new version flag set
 * to true</li> </ul> </li> <li>Then: <ul> <li>The S3 key prefix should have data version set to 0</li> </ul> </li> </ul>
 */
@Test
public void testGetS3KeyPrefixNoDataVersionSpecifiedInitialDataVersionNoExists() throws Exception {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, SUBPARTITION_VALUES.size() + 1);
    // Get the expected S3 key prefix value using the initial business object data version.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), INITIAL_DATA_VERSION);
    // Get an S3 key prefix for the initial version of the business object data with and without createNewVersion flag set.
    for (Boolean createNewVersionFlag : new Boolean[] { false, true }) {
        S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, null), testPartitionKey, STORAGE_NAME, createNewVersionFlag);
        // Validate the results.
        assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
    }
}

Example 10 with S3KeyPrefixInformation

use of org.finra.herd.model.api.xml.S3KeyPrefixInformation in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixMissingOptionalParameters.

@Test
public void testGetS3KeyPrefixMissingOptionalParameters() {
    // Create a business object definition.
    businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION);
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // including passing all allowed number of subpartition values (from 0 to MAX_SUBPARTITIONS).
    for (int i = 0; i <= BusinessObjectDataEntity.MAX_SUBPARTITIONS; i++) {
        // Build a list of subpartition values.
        List<String> subPartitionValues = SUBPARTITION_VALUES.subList(0, i);
        // Get the test partition columns.
        List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
        String testPartitionKey = testPartitionColumns.get(0).getName();
        List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, subPartitionValues.size() + 1);
        // Get an S3 key prefix without passing any of the optional parameters.
        S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, subPartitionValues, null), BLANK_TEXT, BLANK_TEXT, null);
        // Get the expected S3 key prefix value using the initial business object data version.
        String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), INITIAL_DATA_VERSION);
        // Validate the results.
        assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
    }
}