use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.
the class S3RifFile method waitForDownload.
/**
* @return the completed {@link ManifestEntryDownloadResult} for {@link #manifestEntryDownload}
*/
private ManifestEntryDownloadResult waitForDownload() {
Timer.Context downloadWaitTimer = null;
if (!manifestEntryDownload.isDone()) {
downloadWaitTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "waitingForDownloads")).time();
LOGGER.info("Waiting for RIF file download: '{}'...", getDisplayName());
}
// Get the file download result, blocking and waiting if necessary.
ManifestEntryDownloadResult fileDownloadResult;
try {
fileDownloadResult = manifestEntryDownload.get(2, TimeUnit.HOURS);
} catch (InterruptedException e) {
// We're not expecting interrupts here, so go boom.
throw new BadCodeMonkeyException(e);
} catch (ExecutionException e) {
throw new AwsFailureException(e);
} catch (TimeoutException e) {
/*
* We expect downloads to complete within the (generous) timeout. If
* they don't, it's more likely than not that something's wrong, so
* the service should go boom.
*/
throw new AwsFailureException(e);
}
if (downloadWaitTimer != null) {
LOGGER.info("RIF file downloaded: '{}'.", getDisplayName());
downloadWaitTimer.close();
}
return fileDownloadResult;
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.
the class ManifestEntryDownloadTaskIT method testMD5ChkSum.
/**
* Test to ensure the MD5ChkSum of the downloaded S3 file matches the generated MD5ChkSum value
*/
@SuppressWarnings("deprecation")
@Test
public void testMD5ChkSum() throws Exception {
AmazonS3 s3Client = S3Utilities.createS3Client(new ExtractionOptions("foo"));
Bucket bucket = null;
try {
bucket = DataSetTestUtilities.createTestBucket(s3Client);
ExtractionOptions options = new ExtractionOptions(bucket.getName());
LOGGER.info("Bucket created: '{}:{}'", s3Client.getS3AccountOwner().getDisplayName(), bucket.getName());
DataSetManifest manifest = new DataSetManifest(Instant.now(), 0, new DataSetManifestEntry("beneficiaries.rif", RifFileType.BENEFICIARY));
// upload beneficiary sample file to S3 bucket created above
s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest));
s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest, manifest.getEntries().get(0), StaticRifResource.SAMPLE_A_BENES.getResourceUrl()));
// download file from S3 that was just uploaded above
GetObjectRequest objectRequest = new GetObjectRequest(bucket.getName(), String.format("%s/%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, manifest.getEntries().get(0).getParentManifest().getTimestampText(), manifest.getEntries().get(0).getName()));
Path localTempFile = Files.createTempFile("data-pipeline-s3-temp", ".rif");
s3TaskManager = new S3TaskManager(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), new ExtractionOptions(options.getS3BucketName()));
LOGGER.info("Downloading '{}' to '{}'...", objectRequest.getKey(), localTempFile.toAbsolutePath().toString());
Download downloadHandle = s3TaskManager.getS3TransferManager().download(objectRequest, localTempFile.toFile());
downloadHandle.waitForCompletion();
InputStream downloadedInputStream = new FileInputStream(localTempFile.toString());
String generatedMD5ChkSum = ManifestEntryDownloadTask.computeMD5ChkSum(downloadedInputStream);
LOGGER.info("The generated MD5 value from Java (Base64 encoded) is:" + generatedMD5ChkSum);
String downloadedFileMD5ChkSum = downloadHandle.getObjectMetadata().getUserMetaDataOf("md5chksum");
LOGGER.info("The MD5 value from AWS S3 file's metadata is: " + downloadedFileMD5ChkSum);
assertEquals(downloadedFileMD5ChkSum, generatedMD5ChkSum, "Checksum doesn't match on downloaded file " + objectRequest.getKey());
LOGGER.info("Downloaded '{}' to '{}'.", objectRequest.getKey(), localTempFile.toAbsolutePath().toString());
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (AmazonClientException e) {
throw new AwsFailureException(e);
} catch (InterruptedException e) {
// Shouldn't happen, as our apps don't use thread interrupts.
throw new BadCodeMonkeyException(e);
} finally {
if (bucket != null)
DataSetTestUtilities.deleteObjectsAndBucket(s3Client, bucket);
}
}
use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.
the class ManifestEntryDownloadTask method call.
/**
* @see java.util.concurrent.Callable#call()
*/
@Override
public ManifestEntryDownloadResult call() throws Exception {
try {
GetObjectRequest objectRequest = new GetObjectRequest(options.getS3BucketName(), String.format("%s/%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, manifestEntry.getParentManifest().getTimestampText(), manifestEntry.getName()));
Path localTempFile = Files.createTempFile("data-pipeline-s3-temp", ".rif");
Timer.Context downloadTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "downloadSystemTime")).time();
LOGGER.debug("Downloading '{}' to '{}'...", manifestEntry, localTempFile.toAbsolutePath().toString());
Download downloadHandle = s3TaskManager.getS3TransferManager().download(objectRequest, localTempFile.toFile());
downloadHandle.waitForCompletion();
LOGGER.debug("Downloaded '{}' to '{}'.", manifestEntry, localTempFile.toAbsolutePath().toString());
downloadTimer.close();
// generate MD5ChkSum value on file just downloaded
Timer.Context md5ChkSumTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "md5ChkSumSystemTime")).time();
InputStream downloadedInputStream = new FileInputStream(localTempFile.toString());
String generatedMD5ChkSum = ManifestEntryDownloadTask.computeMD5ChkSum(downloadedInputStream);
md5ChkSumTimer.close();
String downloadedFileMD5ChkSum = downloadHandle.getObjectMetadata().getUserMetaDataOf("md5chksum");
// TODO Remove null check below once Jira CBBD-368 is completed
if ((downloadedFileMD5ChkSum != null) && (!generatedMD5ChkSum.equals(downloadedFileMD5ChkSum)))
throw new ChecksumException("Checksum doesn't match on downloaded file " + localTempFile + " manifest entry is " + manifestEntry.toString());
return new ManifestEntryDownloadResult(manifestEntry, localTempFile);
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (AmazonClientException e) {
throw new AwsFailureException(e);
} catch (InterruptedException e) {
// Shouldn't happen, as our apps don't use thread interrupts.
throw new BadCodeMonkeyException(e);
}
}
Aggregations