Search in sources :

Example 1 with AwsFailureException

use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.

the class S3RifFile method waitForDownload.

/**
 * @return the completed {@link ManifestEntryDownloadResult} for {@link #manifestEntryDownload}
 */
private ManifestEntryDownloadResult waitForDownload() {
    Timer.Context downloadWaitTimer = null;
    if (!manifestEntryDownload.isDone()) {
        downloadWaitTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "waitingForDownloads")).time();
        LOGGER.info("Waiting for RIF file download: '{}'...", getDisplayName());
    }
    // Get the file download result, blocking and waiting if necessary.
    ManifestEntryDownloadResult fileDownloadResult;
    try {
        fileDownloadResult = manifestEntryDownload.get(2, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        // We're not expecting interrupts here, so go boom.
        throw new BadCodeMonkeyException(e);
    } catch (ExecutionException e) {
        throw new AwsFailureException(e);
    } catch (TimeoutException e) {
        /*
       * We expect downloads to complete within the (generous) timeout. If
       * they don't, it's more likely than not that something's wrong, so
       * the service should go boom.
       */
        throw new AwsFailureException(e);
    }
    if (downloadWaitTimer != null) {
        LOGGER.info("RIF file downloaded: '{}'.", getDisplayName());
        downloadWaitTimer.close();
    }
    return fileDownloadResult;
}
Also used : Timer(com.codahale.metrics.Timer) ManifestEntryDownloadResult(gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult) BadCodeMonkeyException(gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException) AwsFailureException(gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException) ExecutionException(java.util.concurrent.ExecutionException) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with AwsFailureException

use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.

the class ManifestEntryDownloadTaskIT method testMD5ChkSum.

/**
 * Test to ensure the MD5ChkSum of the downloaded S3 file matches the generated MD5ChkSum value
 */
@SuppressWarnings("deprecation")
@Test
public void testMD5ChkSum() throws Exception {
    AmazonS3 s3Client = S3Utilities.createS3Client(new ExtractionOptions("foo"));
    Bucket bucket = null;
    try {
        bucket = DataSetTestUtilities.createTestBucket(s3Client);
        ExtractionOptions options = new ExtractionOptions(bucket.getName());
        LOGGER.info("Bucket created: '{}:{}'", s3Client.getS3AccountOwner().getDisplayName(), bucket.getName());
        DataSetManifest manifest = new DataSetManifest(Instant.now(), 0, new DataSetManifestEntry("beneficiaries.rif", RifFileType.BENEFICIARY));
        // upload beneficiary sample file to S3 bucket created above
        s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest));
        s3Client.putObject(DataSetTestUtilities.createPutRequest(bucket, manifest, manifest.getEntries().get(0), StaticRifResource.SAMPLE_A_BENES.getResourceUrl()));
        // download file from S3 that was just uploaded above
        GetObjectRequest objectRequest = new GetObjectRequest(bucket.getName(), String.format("%s/%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, manifest.getEntries().get(0).getParentManifest().getTimestampText(), manifest.getEntries().get(0).getName()));
        Path localTempFile = Files.createTempFile("data-pipeline-s3-temp", ".rif");
        s3TaskManager = new S3TaskManager(PipelineTestUtils.get().getPipelineApplicationState().getMetrics(), new ExtractionOptions(options.getS3BucketName()));
        LOGGER.info("Downloading '{}' to '{}'...", objectRequest.getKey(), localTempFile.toAbsolutePath().toString());
        Download downloadHandle = s3TaskManager.getS3TransferManager().download(objectRequest, localTempFile.toFile());
        downloadHandle.waitForCompletion();
        InputStream downloadedInputStream = new FileInputStream(localTempFile.toString());
        String generatedMD5ChkSum = ManifestEntryDownloadTask.computeMD5ChkSum(downloadedInputStream);
        LOGGER.info("The generated MD5 value from Java (Base64 encoded) is:" + generatedMD5ChkSum);
        String downloadedFileMD5ChkSum = downloadHandle.getObjectMetadata().getUserMetaDataOf("md5chksum");
        LOGGER.info("The MD5 value from AWS S3 file's metadata is: " + downloadedFileMD5ChkSum);
        assertEquals(downloadedFileMD5ChkSum, generatedMD5ChkSum, "Checksum doesn't match on downloaded file " + objectRequest.getKey());
        LOGGER.info("Downloaded '{}' to '{}'.", objectRequest.getKey(), localTempFile.toAbsolutePath().toString());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } catch (AmazonClientException e) {
        throw new AwsFailureException(e);
    } catch (InterruptedException e) {
        // Shouldn't happen, as our apps don't use thread interrupts.
        throw new BadCodeMonkeyException(e);
    } finally {
        if (bucket != null)
            DataSetTestUtilities.deleteObjectsAndBucket(s3Client, bucket);
    }
}
Also used : Path(java.nio.file.Path) AmazonS3(com.amazonaws.services.s3.AmazonS3) DataSetManifest(gov.cms.bfd.pipeline.ccw.rif.extract.s3.DataSetManifest) BadCodeMonkeyException(gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) AmazonClientException(com.amazonaws.AmazonClientException) ExtractionOptions(gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) AwsFailureException(gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException) FileInputStream(java.io.FileInputStream) Bucket(com.amazonaws.services.s3.model.Bucket) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) Download(com.amazonaws.services.s3.transfer.Download) DataSetManifestEntry(gov.cms.bfd.pipeline.ccw.rif.extract.s3.DataSetManifest.DataSetManifestEntry) Test(org.junit.jupiter.api.Test)

Example 3 with AwsFailureException

use of gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException in project beneficiary-fhir-data by CMSgov.

the class ManifestEntryDownloadTask method call.

/**
 * @see java.util.concurrent.Callable#call()
 */
@Override
public ManifestEntryDownloadResult call() throws Exception {
    try {
        GetObjectRequest objectRequest = new GetObjectRequest(options.getS3BucketName(), String.format("%s/%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, manifestEntry.getParentManifest().getTimestampText(), manifestEntry.getName()));
        Path localTempFile = Files.createTempFile("data-pipeline-s3-temp", ".rif");
        Timer.Context downloadTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "downloadSystemTime")).time();
        LOGGER.debug("Downloading '{}' to '{}'...", manifestEntry, localTempFile.toAbsolutePath().toString());
        Download downloadHandle = s3TaskManager.getS3TransferManager().download(objectRequest, localTempFile.toFile());
        downloadHandle.waitForCompletion();
        LOGGER.debug("Downloaded '{}' to '{}'.", manifestEntry, localTempFile.toAbsolutePath().toString());
        downloadTimer.close();
        // generate MD5ChkSum value on file just downloaded
        Timer.Context md5ChkSumTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "md5ChkSumSystemTime")).time();
        InputStream downloadedInputStream = new FileInputStream(localTempFile.toString());
        String generatedMD5ChkSum = ManifestEntryDownloadTask.computeMD5ChkSum(downloadedInputStream);
        md5ChkSumTimer.close();
        String downloadedFileMD5ChkSum = downloadHandle.getObjectMetadata().getUserMetaDataOf("md5chksum");
        // TODO Remove null check below once Jira CBBD-368 is completed
        if ((downloadedFileMD5ChkSum != null) && (!generatedMD5ChkSum.equals(downloadedFileMD5ChkSum)))
            throw new ChecksumException("Checksum doesn't match on downloaded file " + localTempFile + " manifest entry is " + manifestEntry.toString());
        return new ManifestEntryDownloadResult(manifestEntry, localTempFile);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } catch (AmazonClientException e) {
        throw new AwsFailureException(e);
    } catch (InterruptedException e) {
        // Shouldn't happen, as our apps don't use thread interrupts.
        throw new BadCodeMonkeyException(e);
    }
}
Also used : Path(java.nio.file.Path) BadCodeMonkeyException(gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) ChecksumException(gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.ChecksumException) AmazonClientException(com.amazonaws.AmazonClientException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) AwsFailureException(gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException) FileInputStream(java.io.FileInputStream) Timer(com.codahale.metrics.Timer) ManifestEntryDownloadResult(gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) Download(com.amazonaws.services.s3.transfer.Download)

Aggregations

AwsFailureException (gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.AwsFailureException)3 BadCodeMonkeyException (gov.cms.bfd.sharedutils.exceptions.BadCodeMonkeyException)3 AmazonClientException (com.amazonaws.AmazonClientException)2 GetObjectRequest (com.amazonaws.services.s3.model.GetObjectRequest)2 Download (com.amazonaws.services.s3.transfer.Download)2 Timer (com.codahale.metrics.Timer)2 ManifestEntryDownloadResult (gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult)2 FileInputStream (java.io.FileInputStream)2 IOException (java.io.IOException)2 InputStream (java.io.InputStream)2 UncheckedIOException (java.io.UncheckedIOException)2 Path (java.nio.file.Path)2 AmazonS3 (com.amazonaws.services.s3.AmazonS3)1 Bucket (com.amazonaws.services.s3.model.Bucket)1 ExtractionOptions (gov.cms.bfd.pipeline.ccw.rif.extract.ExtractionOptions)1 ChecksumException (gov.cms.bfd.pipeline.ccw.rif.extract.exceptions.ChecksumException)1 DataSetManifest (gov.cms.bfd.pipeline.ccw.rif.extract.s3.DataSetManifest)1 DataSetManifestEntry (gov.cms.bfd.pipeline.ccw.rif.extract.s3.DataSetManifest.DataSetManifestEntry)1 ExecutionException (java.util.concurrent.ExecutionException)1 TimeoutException (java.util.concurrent.TimeoutException)1