use of gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions in project beneficiary-fhir-data by CMSgov.
the class CcwRifLoadJobIT method skipDataSetTest.
/**
* Tests {@link CcwRifLoadJob} when run against a bucket with a single data set that should be
* skipped (per {@link ExtractionOptions#getDataSetFilter()}).
*
* @throws Exception (exceptions indicate test failure)
*/
@Test
public void skipDataSetTest() throws Exception {
AmazonS3 s3Client = S3Utilities.createS3Client(new ExtractionOptions("foo"));
Bucket bucket = null;
try {
/*
* Create the (empty) bucket to run against, and populate it with a
* data set.
*/
bucket = DataSetTestUtilities.createTestBucket(s3Client);
ExtractionOptions options = new ExtractionOptions(bucket.getName(), Optional.of(RifFileType.PDE));
LOGGER.info("Bucket created: '{}:{}'", s3Client.getS3AccountOwner().getDisplayName(), bucket.getName());
DataSetManifest manifest = new DataSetManifest(Instant.now(), 0, new DataSetManifestEntry("beneficiaries.rif", RifFileType.BENEFICIARY), new DataSetManifestEntry("carrier.rif", RifFileType.CARRIER));
s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest));
s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest, manifest.getEntries().get(0), StaticRifResource.SAMPLE_A_BENES.getResourceUrl()));
s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest, manifest.getEntries().get(1), StaticRifResource.SAMPLE_A_CARRIER.getResourceUrl()));
// Run the job.
MockDataSetMonitorListener listener = new MockDataSetMonitorListener();
S3TaskManager s3TaskManager = new S3TaskManager(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), options);
CcwRifLoadJob ccwJob = new CcwRifLoadJob(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), options, s3TaskManager, listener);
ccwJob.call();
// Verify what was handed off to the DataSetMonitorListener.
assertEquals(1, listener.getNoDataAvailableEvents());
assertEquals(0, listener.getDataEvents().size());
assertEquals(0, listener.getErrorEvents().size());
// Verify that the data set was not renamed.
DataSetTestUtilities.waitForBucketObjectCount(s3Client, bucket, CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, 1 + manifest.getEntries().size(), java.time.Duration.ofSeconds(10));
DataSetTestUtilities.waitForBucketObjectCount(s3Client, bucket, CcwRifLoadJob.S3_PREFIX_COMPLETED_DATA_SETS, 0, java.time.Duration.ofSeconds(10));
} finally {
if (bucket != null)
DataSetTestUtilities.deleteObjectsAndBucket(s3Client, bucket);
}
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions in project beneficiary-fhir-data by CMSgov.
the class CcwRifLoadJobIT method emptyBucketTest.
/**
* Tests {@link CcwRifLoadJob} when run against an empty bucket.
*
* @throws Exception (exceptions indicate test failure)
*/
@Test
public void emptyBucketTest() throws Exception {
AmazonS3 s3Client = S3Utilities.createS3Client(new ExtractionOptions("foo"));
Bucket bucket = null;
try {
// Create the (empty) bucket to run against.
bucket = DataSetTestUtilities.createTestBucket(s3Client);
ExtractionOptions options = new ExtractionOptions(bucket.getName());
LOGGER.info("Bucket created: '{}:{}'", s3Client.getS3AccountOwner().getDisplayName(), bucket.getName());
// Run the job.
MockDataSetMonitorListener listener = new MockDataSetMonitorListener();
S3TaskManager s3TaskManager = new S3TaskManager(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), options);
CcwRifLoadJob ccwJob = new CcwRifLoadJob(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), options, s3TaskManager, listener);
ccwJob.call();
// Verify that no data sets were generated.
assertEquals(1, listener.getNoDataAvailableEvents());
assertEquals(0, listener.getDataEvents().size());
assertEquals(0, listener.getErrorEvents().size());
} finally {
if (bucket != null)
s3Client.deleteBucket(bucket.getName());
}
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions in project beneficiary-fhir-data by CMSgov.
the class AppConfiguration method readCcwRifLoadOptionsFromEnvironmentVariables.
@Nullable
static CcwRifLoadOptions readCcwRifLoadOptionsFromEnvironmentVariables(LoadAppOptions loadOptions) {
final boolean enabled = readEnvBooleanOptional(ENV_VAR_KEY_CCW_RIF_JOB_ENABLED).orElse(true);
if (!enabled) {
return null;
}
final String s3BucketName = readEnvStringRequired(ENV_VAR_KEY_BUCKET);
final Optional<String> rifFilterText = readEnvStringOptional(ENV_VAR_KEY_ALLOWED_RIF_TYPE);
final Optional<RifFileType> allowedRifFileType;
if (rifFilterText.isPresent()) {
try {
allowedRifFileType = Optional.of(RifFileType.valueOf(rifFilterText.get()));
} catch (IllegalArgumentException e) {
throw new AppConfigurationException(String.format("Invalid value for configuration environment variable '%s': '%s'", ENV_VAR_KEY_ALLOWED_RIF_TYPE, rifFilterText), e);
}
} else {
allowedRifFileType = Optional.empty();
}
/*
* Just for convenience: make sure DefaultAWSCredentialsProviderChain
* has whatever it needs.
*/
try {
DefaultAWSCredentialsProviderChain awsCredentialsProvider = new DefaultAWSCredentialsProviderChain();
awsCredentialsProvider.getCredentials();
} catch (AmazonClientException e) {
/*
* The credentials provider should throw this if it can't find what
* it needs.
*/
throw new AppConfigurationException(String.format("Missing configuration for AWS credentials (for %s).", DefaultAWSCredentialsProviderChain.class.getName()), e);
}
ExtractionOptions extractionOptions = new ExtractionOptions(s3BucketName, allowedRifFileType);
CcwRifLoadOptions ccwRifLoadOptions = new CcwRifLoadOptions(extractionOptions, loadOptions);
return ccwRifLoadOptions;
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions in project beneficiary-fhir-data by CMSgov.
the class SyntheticDataUploader method main.
/**
* Pushes the synthetic data up to S3, replacing any versions that are already there.
*
* @param args (not used)
* @throws Exception Any {@link Exception}s encountered will be bubbled up, halting the
* application.
*/
public static void main(String[] args) throws Exception {
ExtractionOptions options = new ExtractionOptions(String.format("bb-test-%d", new Random().nextInt(1000)));
AmazonS3 s3Client = S3Utilities.createS3Client(options);
LOGGER.info("Uploading original data...");
uploadSyntheticData(s3Client, TestDataSetLocation.SYNTHETIC_DATA.getS3KeyPrefix().replace("-fixed", ""), syntheticDataFile -> syntheticDataFile.getOriginalFilePath());
LOGGER.info("Uploading fixed data...");
uploadSyntheticData(s3Client, TestDataSetLocation.SYNTHETIC_DATA.getS3KeyPrefix(), syntheticDataFile -> syntheticDataFile.getFixedFilePath());
LOGGER.info("Uploaded all data.");
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions in project beneficiary-fhir-data by CMSgov.
the class SyntheticDataUploader2 method main.
/**
* Pushes the synthetic data from {@link SyntheticDataFixer2} up to S3, replacing any versions
* that are already there.
*
* @param args (not used)
* @throws Exception Any {@link Exception}s encountered will be bubbled up, halting the
* application.
*/
public static void main(String[] args) throws Exception {
ExtractionOptions options = new ExtractionOptions(String.format("bb-test-%d", new Random().nextInt(1000)));
AmazonS3 s3Client = S3Utilities.createS3Client(options);
LOGGER.info("Uploading fixed data...");
uploadSyntheticData(s3Client, TestDataSetLocation.SYNTHETIC_DATA.getS3KeyPrefix(), syntheticDataFile -> syntheticDataFile.getFixedFilePath());
LOGGER.info("Uploaded all data.");
}
Aggregations