Examples with Download com.amazonaws.services.s3.transfer.Download used on opensource projects

Example 21 with Download

use of com.amazonaws.services.s3.transfer.Download in project carina by qaprosoft.

the class AbstractTest method updateS3AppPath.

/**
 * Method to update MOBILE_APP path in case if apk is located in s3 bucket.
 */
private void updateS3AppPath() {
    Pattern S3_BUCKET_PATTERN = Pattern.compile("s3:\\/\\/([a-zA-Z-0-9][^\\/]*)\\/(.*)");
    // get app path to be sure that we need(do not need) to download app from s3 bucket
    String mobileAppPath = Configuration.getMobileApp();
    Matcher matcher = S3_BUCKET_PATTERN.matcher(mobileAppPath);
    LOGGER.info("Analyzing if mobile app is located on S3...");
    if (matcher.find()) {
        LOGGER.info("app artifact is located on s3...");
        String bucketName = matcher.group(1);
        String key = matcher.group(2);
        Pattern pattern = Pattern.compile(key);
        // analyze if we have any pattern inside mobile_app to make extra
        // search in AWS
        int position = key.indexOf(".*");
        if (position > 0) {
            // /android/develop/dfgdfg.*/Mapmyrun.apk
            int slashPosition = key.substring(0, position).lastIndexOf("/");
            if (slashPosition > 0) {
                key = key.substring(0, slashPosition);
                S3ObjectSummary lastBuild = AmazonS3Manager.getInstance().getLatestBuildArtifact(bucketName, key, pattern);
                key = lastBuild.getKey();
            }
        }
        S3Object objBuild = AmazonS3Manager.getInstance().get(bucketName, key);
        String s3LocalStorage = Configuration.get(Parameter.S3_LOCAL_STORAGE);
        // download file from AWS to local storage
        String fileName = s3LocalStorage + "/" + StringUtils.substringAfterLast(objBuild.getKey(), "/");
        File file = new File(fileName);
        // download
        if (file.exists() && file.length() == objBuild.getObjectMetadata().getContentLength()) {
            LOGGER.info("build artifact with the same size already downloaded: " + file.getAbsolutePath());
        } else {
            LOGGER.info(String.format("Following data was extracted: bucket: %s, key: %s, local file: %s", bucketName, key, file.getAbsolutePath()));
            AmazonS3Manager.getInstance().download(bucketName, key, new File(fileName));
        }
        Configuration.setMobileApp(file.getAbsolutePath());
        // try to redefine app_version if it's value is latest or empty
        String appVersion = Configuration.get(Parameter.APP_VERSION);
        if (appVersion.equals("latest") || appVersion.isEmpty()) {
            R.CONFIG.put(Parameter.APP_VERSION.getKey(), file.getName());
        }
    }
}

Example 22 with Download

use of com.amazonaws.services.s3.transfer.Download in project herd by FINRAOS.

the class MockS3OperationsImpl method downloadDirectory.

/**
 * {@inheritDoc}
 * <p/>
 * This implementation creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, TransferManager transferManager) {
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix + ", destinationDirectory = " + destinationDirectory);
    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            if (mockS3Object.getKey().startsWith(keyPrefix)) {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                // Create any directory in the path that does not exist.
                file.getParentFile().mkdirs();
                try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null, new GetObjectRequest(bucketName, mockS3Object.getKey()), file, mockS3Object.getObjectMetadata(), false));
                } catch (IOException e) {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }
    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);
    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null, keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}

Example 23 with Download

use of com.amazonaws.services.s3.transfer.Download in project herd by FINRAOS.

the class MockS3OperationsImpl method putObject.

/**
 * {@inheritDoc}
 * <p/>
 * This implementation creates a new bucket if the bucket does not already exist.
 */
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest, AmazonS3 s3Client) {
    LOGGER.debug("putObject(): putObjectRequest.getBucketName() = " + putObjectRequest.getBucketName() + ", putObjectRequest.getKey() = " + putObjectRequest.getKey());
    String s3BucketName = putObjectRequest.getBucketName();
    InputStream inputStream = putObjectRequest.getInputStream();
    ObjectMetadata metadata = putObjectRequest.getMetadata();
    if (metadata == null) {
        metadata = new ObjectMetadata();
    }
    File file = putObjectRequest.getFile();
    if (file != null) {
        try {
            inputStream = new FileInputStream(file);
            metadata.setContentLength(file.length());
        } catch (FileNotFoundException e) {
            throw new IllegalArgumentException("File not found " + file, e);
        }
    }
    String s3ObjectKey = putObjectRequest.getKey();
    String s3ObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(putObjectRequest.getBucketName()) ? UUID.randomUUID().toString() : null;
    String s3ObjectKeyVersion = s3ObjectKey + (s3ObjectVersion != null ? s3ObjectVersion : "");
    byte[] s3ObjectData;
    try {
        s3ObjectData = IOUtils.toByteArray(inputStream);
        metadata.setContentLength(s3ObjectData.length);
    } catch (IOException e) {
        throw new IllegalArgumentException("Error converting input stream into byte array", e);
    } finally {
        try {
            inputStream.close();
        } catch (IOException e) {
            LOGGER.error("Error closing stream " + inputStream, e);
        }
    }
    // Update the Last-Modified header value. This value not being set causes NullPointerException in S3Dao download related unit tests.
    metadata.setLastModified(new Date());
    MockS3Bucket mockS3Bucket = getOrCreateBucket(s3BucketName);
    MockS3Object mockS3Object = new MockS3Object();
    mockS3Object.setKey(s3ObjectKey);
    mockS3Object.setVersion(s3ObjectVersion);
    mockS3Object.setData(s3ObjectData);
    mockS3Object.setObjectMetadata(metadata);
    if (putObjectRequest.getTagging() != null) {
        mockS3Object.setTags(putObjectRequest.getTagging().getTagSet());
    }
    mockS3Bucket.getObjects().put(s3ObjectKey, mockS3Object);
    mockS3Bucket.getVersions().put(s3ObjectKeyVersion, mockS3Object);
    return new PutObjectResult();
}

Example 24 with Download

use of com.amazonaws.services.s3.transfer.Download in project herd by FINRAOS.

the class S3DaoImpl method performTransfer.

/**
 * Performs a file/directory transfer.
 *
 * @param params the parameters.
 * @param transferer a transferer that knows how to perform the transfer.
 *
 * @return the results.
 * @throws InterruptedException if a problem is encountered.
 */
private S3FileTransferResultsDto performTransfer(final S3FileTransferRequestParamsDto params, Transferer transferer) throws InterruptedException {
    // Create a transfer manager.
    TransferManager transferManager = getTransferManager(params);
    try {
        // Start a stop watch to keep track of how long the transfer takes.
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        // Perform the transfer.
        Transfer transfer = transferer.performTransfer(transferManager);
        TransferProgress transferProgress = transfer.getProgress();
        logTransferProgress(transferProgress);
        long stepCount = 0;
        // Loop until the transfer is complete.
        do {
            Thread.sleep(sleepIntervalsMillis);
            stepCount++;
            // Log progress status every 30 seconds and when transfer is complete.
            if (transfer.isDone() || stepCount % 300 == 0) {
                logTransferProgress(transferProgress);
            }
        } while (!transfer.isDone());
        // Stop the stop watch and create a results object.
        stopWatch.stop();
        // If the transfer failed, throw the underlying AWS exception if we can determine one. Otherwise, throw our own exception.
        TransferState transferState = transfer.getState();
        if (transferState == TransferState.Failed) {
            // The waitForException method should return the underlying AWS exception since the state is "Failed". It should not block since the
            // transfer is already "done" per previous code checking "isDone".
            AmazonClientException amazonClientException = transfer.waitForException();
            // This is unlikely since the transfer failed, but it's better to handle the possibility just in case.
            if (amazonClientException == null) {
                throw new IllegalStateException("The transfer operation \"" + transfer.getDescription() + "\" failed for an unknown reason.");
            }
            // Throw the Amazon underlying exception.
            throw amazonClientException;
        } else // Ensure the transfer completed. If not, throw an exception.
        if (transferState != TransferState.Completed) {
            throw new IllegalStateException("The transfer operation \"" + transfer.getDescription() + "\" did not complete successfully. Current state: \"" + transferState + "\".");
        }
        // TransferProgress.getBytesTransferred() are not populated for S3 Copy objects.
        if (!(transfer instanceof Copy)) {
            // Sanity check for the number of bytes transferred.
            Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(), String.format("Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).", transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer()));
        }
        // Create the results object and populate it with the standard data.
        S3FileTransferResultsDto results = new S3FileTransferResultsDto();
        results.setDurationMillis(stopWatch.getTime());
        results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred());
        results.setTotalFilesTransferred(1L);
        if (transfer instanceof MultipleFileUpload) {
            // For upload directory, we need to calculate the total number of files transferred differently.
            results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size());
        } else if (transfer instanceof MultipleFileDownload) {
            // For download directory, we need to calculate the total number of files differently.
            results.setTotalFilesTransferred((long) listDirectory(params).size());
        }
        // Return the results.
        return results;
    } finally {
        // Shutdown the transfer manager to release resources. If this isn't done, the JVM may delay upon exiting.
        transferManager.shutdownNow();
    }
}

Example 25 with Download

use of com.amazonaws.services.s3.transfer.Download in project photon-model by vmware.

the class AWSCostStatsService method downloadParseAndCreateStats.

private void downloadParseAndCreateStats(AWSCostStatsCreationContext statsData, String awsBucketName) throws IOException {
    try {
        // Creating a working directory for downloading and processing the bill
        final Path workingDirPath = Paths.get(System.getProperty(TEMP_DIR_LOCATION), UUID.randomUUID().toString());
        Files.createDirectories(workingDirPath);
        AWSCsvBillParser parser = new AWSCsvBillParser();
        final String csvBillZipFileName = parser.getCsvBillFileName(statsData.billMonthToDownload, statsData.accountId, true);
        Path csvBillZipFilePath = Paths.get(workingDirPath.toString(), csvBillZipFileName);
        ProgressListener listener = new ProgressListener() {

            @Override
            public void progressChanged(ProgressEvent progressEvent) {
                try {
                    ProgressEventType eventType = progressEvent.getEventType();
                    if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(eventType)) {
                        OperationContext.restoreOperationContext(statsData.opContext);
                        LocalDate billMonth = new LocalDate(statsData.billMonthToDownload.getYear(), statsData.billMonthToDownload.getMonthOfYear(), 1);
                        logWithContext(statsData, Level.INFO, () -> String.format("Processing" + " bill for the month: %s.", billMonth));
                        parser.parseDetailedCsvBill(statsData.ignorableInvoiceCharge, csvBillZipFilePath, statsData.awsAccountIdToComputeStates.keySet(), getHourlyStatsConsumer(billMonth, statsData), getMonthlyStatsConsumer(billMonth, statsData));
                        deleteTempFiles();
                        // Continue downloading and processing the bills for past and current months' bills
                        statsData.billMonthToDownload = statsData.billMonthToDownload.plusMonths(1);
                        handleCostStatsCreationRequest(statsData);
                    } else if (ProgressEventType.TRANSFER_FAILED_EVENT.equals(eventType)) {
                        deleteTempFiles();
                        billDownloadFailureHandler(statsData, awsBucketName, new IOException("Download of AWS CSV Bill '" + csvBillZipFileName + "' failed."));
                    }
                } catch (Exception exception) {
                    deleteTempFiles();
                    billDownloadFailureHandler(statsData, awsBucketName, exception);
                }
            }

            private void deleteTempFiles() {
                try {
                    Files.deleteIfExists(csvBillZipFilePath);
                    Files.deleteIfExists(workingDirPath);
                } catch (IOException e) {
                // Ignore IO exception while cleaning files.
                }
            }
        };
        GetObjectRequest getObjectRequest = new GetObjectRequest(awsBucketName, csvBillZipFileName).withGeneralProgressListener(listener);
        statsData.s3Client.download(getObjectRequest, csvBillZipFilePath.toFile());
    } catch (AmazonS3Exception s3Exception) {
        billDownloadFailureHandler(statsData, awsBucketName, s3Exception);
    }
}