Examples with TransferManager com.amazonaws.services.s3.transfer.TransferManager used on opensource projects

Example 31 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project opencast by opencast.

the class AwsS3DistributionServiceImpl method activate.

@Override
public void activate(ComponentContext cc) {
    // Get the configuration
    if (cc != null) {
        if (!Boolean.valueOf(getAWSConfigKey(cc, AWS_S3_DISTRIBUTION_ENABLE))) {
            logger.info("AWS S3 distribution disabled");
            return;
        }
        // AWS S3 bucket name
        bucketName = getAWSConfigKey(cc, AWS_S3_BUCKET_CONFIG);
        logger.info("AWS S3 bucket name is {}", bucketName);
        // AWS region
        String regionStr = getAWSConfigKey(cc, AWS_S3_REGION_CONFIG);
        logger.info("AWS region is {}", regionStr);
        opencastDistributionUrl = getAWSConfigKey(cc, AWS_S3_DISTRIBUTION_BASE_CONFIG);
        if (!opencastDistributionUrl.endsWith("/")) {
            opencastDistributionUrl = opencastDistributionUrl + "/";
        }
        logger.info("AWS distribution url is {}", opencastDistributionUrl);
        // Explicit credentials are optional.
        AWSCredentialsProvider provider = null;
        Option<String> accessKeyIdOpt = OsgiUtil.getOptCfg(cc.getProperties(), AWS_S3_ACCESS_KEY_ID_CONFIG);
        Option<String> accessKeySecretOpt = OsgiUtil.getOptCfg(cc.getProperties(), AWS_S3_SECRET_ACCESS_KEY_CONFIG);
        // profile credentials
        if (accessKeyIdOpt.isNone() && accessKeySecretOpt.isNone())
            provider = new DefaultAWSCredentialsProviderChain();
        else
            provider = new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyIdOpt.get(), accessKeySecretOpt.get()));
        // Create AWS client.
        s3 = AmazonS3ClientBuilder.standard().withRegion(regionStr).withCredentials(provider).build();
        s3TransferManager = new TransferManager(s3);
        // Create AWS S3 bucket if not there yet
        createAWSBucket();
        distributionChannel = OsgiUtil.getComponentContextProperty(cc, CONFIG_KEY_STORE_TYPE);
        logger.info("AwsS3DistributionService activated!");
    }
}

Example 32 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project opencast by opencast.

the class AwsS3DistributionServiceImpl method distributeElement.

/**
 * Distribute a media package element to AWS S3.
 *
 * @param mediaPackage
 *          The media package that contains the element to distribute.
 * @param element
 *          The element that should be distributed contained within the media package.
 * @param checkAvailability
 *          Checks if the distributed element is available
 * @return A reference to the MediaPackageElement that has been distributed.
 * @throws DistributionException
 */
public MediaPackageElement distributeElement(String channelId, final MediaPackage mediaPackage, MediaPackageElement element, boolean checkAvailability) throws DistributionException {
    notNull(channelId, "channelId");
    notNull(mediaPackage, "mediapackage");
    notNull(element, "element");
    try {
        File source;
        try {
            source = workspace.get(element.getURI());
        } catch (NotFoundException e) {
            throw new DistributionException("Unable to find " + element.getURI() + " in the workspace", e);
        } catch (IOException e) {
            throw new DistributionException("Error loading " + element.getURI() + " from the workspace", e);
        }
        // Use TransferManager to take advantage of multipart upload.
        // TransferManager processes all transfers asynchronously, so this call will return immediately.
        String objectName = buildObjectName(channelId, mediaPackage.getIdentifier().toString(), element);
        logger.info("Uploading {} to bucket {}...", objectName, bucketName);
        Upload upload = s3TransferManager.upload(bucketName, objectName, source);
        long start = System.currentTimeMillis();
        try {
            // Block and wait for the upload to finish
            upload.waitForCompletion();
            logger.info("Upload of {} to bucket {} completed in {} seconds", objectName, bucketName, (System.currentTimeMillis() - start) / 1000);
        } catch (AmazonClientException e) {
            throw new DistributionException("AWS error: " + e.getMessage(), e);
        }
        // Create a representation of the distributed file in the media package
        MediaPackageElement distributedElement = (MediaPackageElement) element.clone();
        try {
            distributedElement.setURI(getDistributionUri(objectName));
        } catch (URISyntaxException e) {
            throw new DistributionException("Distributed element produces an invalid URI", e);
        }
        logger.info("Distributed element {}, object {}", element.getIdentifier(), objectName);
        if (checkAvailability) {
            URI uri = distributedElement.getURI();
            int tries = 0;
            CloseableHttpResponse response = null;
            boolean success = false;
            while (tries < MAX_TRIES) {
                try {
                    CloseableHttpClient httpClient = HttpClients.createDefault();
                    logger.trace("Trying to access {}", uri);
                    response = httpClient.execute(new HttpHead(uri));
                    if (response.getStatusLine().getStatusCode() == HttpServletResponse.SC_OK) {
                        logger.trace("Successfully got {}", uri);
                        success = true;
                        // Exit the loop, response is closed
                        break;
                    } else {
                        logger.debug("Http status code when checking distributed element {} is {}", objectName, response.getStatusLine().getStatusCode());
                    }
                } catch (Exception e) {
                    logger.info("Checking availability of {} threw exception {}. Trying again.", objectName, e.getMessage());
                // Just try again
                } finally {
                    if (null != response) {
                        response.close();
                    }
                }
                tries++;
                logger.trace("Sleeping for {} seconds...", SLEEP_INTERVAL / 1000);
                Thread.sleep(SLEEP_INTERVAL);
            }
            if (!success) {
                logger.warn("Could not check availability of distributed file {}", uri);
            // throw new DistributionException("Unable to load distributed file " + uri.toString());
            }
        }
        return distributedElement;
    } catch (Exception e) {
        logger.warn("Error distributing element " + element.getIdentifier() + " of media package " + mediaPackage, e);
        if (e instanceof DistributionException) {
            throw (DistributionException) e;
        } else {
            throw new DistributionException(e);
        }
    }
}

Example 33 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class HighLevelTrackMultipartUpload method main.

public static void main(String[] args) throws Exception {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    String keyName = "*** Object key ***";
    String filePath = "*** Path to file to upload ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(clientRegion).withCredentials(new ProfileCredentialsProvider()).build();
        TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3Client).build();
        PutObjectRequest request = new PutObjectRequest(bucketName, keyName, new File(filePath));
        // To receive notifications when bytes are transferred, add a
        // ProgressListener to your request.
        request.setGeneralProgressListener(new ProgressListener() {

            public void progressChanged(ProgressEvent progressEvent) {
                System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
            }
        });
        // TransferManager processes all transfers asynchronously,
        // so this call returns immediately.
        Upload upload = tm.upload(request);
        // Optionally, you can wait for the upload to finish before continuing.
        upload.waitForCompletion();
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 34 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class XferMgrCopy method copyObjectSimple.

public static void copyObjectSimple(String from_bucket, String from_key, String to_bucket, String to_key) {
    // snippet-start:[s3.java1.s3_xfer_mgr_copy.copy_object]
    System.out.println("Copying s3 object: " + from_key);
    System.out.println("      from bucket: " + from_bucket);
    System.out.println("     to s3 object: " + to_key);
    System.out.println("        in bucket: " + to_bucket);
    TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
    try {
        Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_copy.copy_object]
}

Example 35 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class XferMgrDownload method downloadDir.

public static void downloadDir(String bucket_name, String key_prefix, String dir_path, boolean pause) {
    System.out.println("downloading to directory: " + dir_path + (pause ? " (pause)" : ""));
    // snippet-start:[s3.java1.s3_xfer_mgr_download.directory]
    TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
    try {
        MultipleFileDownload xfer = xfer_mgr.downloadDirectory(bucket_name, key_prefix, new File(dir_path));
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_download.directory]
}