Examples with ManifestEntryDownloadResult gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult used on opensource projects

Example 1 with ManifestEntryDownloadResult

use of gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult in project beneficiary-fhir-data by CMSgov.

the class S3RifFile method waitForDownload.

/**
 * @return the completed {@link ManifestEntryDownloadResult} for {@link #manifestEntryDownload}
 */
private ManifestEntryDownloadResult waitForDownload() {
    Timer.Context downloadWaitTimer = null;
    if (!manifestEntryDownload.isDone()) {
        downloadWaitTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "waitingForDownloads")).time();
        LOGGER.info("Waiting for RIF file download: '{}'...", getDisplayName());
    }
    // Get the file download result, blocking and waiting if necessary.
    ManifestEntryDownloadResult fileDownloadResult;
    try {
        fileDownloadResult = manifestEntryDownload.get(2, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        // We're not expecting interrupts here, so go boom.
        throw new BadCodeMonkeyException(e);
    } catch (ExecutionException e) {
        throw new AwsFailureException(e);
    } catch (TimeoutException e) {
        /*
       * We expect downloads to complete within the (generous) timeout. If
       * they don't, it's more likely than not that something's wrong, so
       * the service should go boom.
       */
        throw new AwsFailureException(e);
    }
    if (downloadWaitTimer != null) {
        LOGGER.info("RIF file downloaded: '{}'.", getDisplayName());
        downloadWaitTimer.close();
    }
    return fileDownloadResult;
}

Example 2 with ManifestEntryDownloadResult

use of gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult in project beneficiary-fhir-data by CMSgov.

the class ManifestEntryDownloadTask method call.

/**
 * @see java.util.concurrent.Callable#call()
 */
@Override
public ManifestEntryDownloadResult call() throws Exception {
    try {
        GetObjectRequest objectRequest = new GetObjectRequest(options.getS3BucketName(), String.format("%s/%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, manifestEntry.getParentManifest().getTimestampText(), manifestEntry.getName()));
        Path localTempFile = Files.createTempFile("data-pipeline-s3-temp", ".rif");
        Timer.Context downloadTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "downloadSystemTime")).time();
        LOGGER.debug("Downloading '{}' to '{}'...", manifestEntry, localTempFile.toAbsolutePath().toString());
        Download downloadHandle = s3TaskManager.getS3TransferManager().download(objectRequest, localTempFile.toFile());
        downloadHandle.waitForCompletion();
        LOGGER.debug("Downloaded '{}' to '{}'.", manifestEntry, localTempFile.toAbsolutePath().toString());
        downloadTimer.close();
        // generate MD5ChkSum value on file just downloaded
        Timer.Context md5ChkSumTimer = appMetrics.timer(MetricRegistry.name(getClass().getSimpleName(), "md5ChkSumSystemTime")).time();
        InputStream downloadedInputStream = new FileInputStream(localTempFile.toString());
        String generatedMD5ChkSum = ManifestEntryDownloadTask.computeMD5ChkSum(downloadedInputStream);
        md5ChkSumTimer.close();
        String downloadedFileMD5ChkSum = downloadHandle.getObjectMetadata().getUserMetaDataOf("md5chksum");
        // TODO Remove null check below once Jira CBBD-368 is completed
        if ((downloadedFileMD5ChkSum != null) && (!generatedMD5ChkSum.equals(downloadedFileMD5ChkSum)))
            throw new ChecksumException("Checksum doesn't match on downloaded file " + localTempFile + " manifest entry is " + manifestEntry.toString());
        return new ManifestEntryDownloadResult(manifestEntry, localTempFile);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } catch (AmazonClientException e) {
        throw new AwsFailureException(e);
    } catch (InterruptedException e) {
        // Shouldn't happen, as our apps don't use thread interrupts.
        throw new BadCodeMonkeyException(e);
    }
}

Example 3 with ManifestEntryDownloadResult

use of gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult in project beneficiary-fhir-data by CMSgov.

the class S3RifFile method cleanupTempFile.

/**
 * Removes the local temporary file that was used to cache this {@link S3RifFile}'s corresponding
 * S3 object data locally.
 */
public void cleanupTempFile() {
    LOGGER.debug("Cleaning up '{}'...", this);
    /*
     * We need to either cancel the download or wait for it to complete and then clean up the file.
     * However, canceling isn't a thread-safe operation (which is bonkers, but true), so we'll just
     * wait for completion.
     */
    try {
        ManifestEntryDownloadResult fileDownloadResult = waitForDownload();
        Files.deleteIfExists(fileDownloadResult.getLocalDownload());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } catch (CancellationException e) {
        LOGGER.debug("Download was cancelled and can't be cleaned up.");
    }
    LOGGER.debug("Cleaned up '{}'.", this);
}

Example 4 with ManifestEntryDownloadResult

use of gov.cms.bfd.pipeline.ccw.rif.extract.s3.task.ManifestEntryDownloadTask.ManifestEntryDownloadResult in project beneficiary-fhir-data by CMSgov.

the class S3RifFile method open.

/**
 * @see gov.cms.bfd.model.rif.RifFile#open()
 */
@Override
public InputStream open() {
    ManifestEntryDownloadResult fileDownloadResult = waitForDownload();
    // Open a stream for the file.
    InputStream fileDownloadStream;
    try {
        fileDownloadStream = new BufferedInputStream(new FileInputStream(fileDownloadResult.getLocalDownload().toFile()));
    } catch (FileNotFoundException e) {
        throw new UncheckedIOException(e);
    }
    return fileDownloadStream;
}