Examples with AmazonS3 com.talend.shaded.com.amazonaws.services.s3.AmazonS3 used on opensource projects

Example 16 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project ignite by apache.

the class S3CheckpointSpiSelfTest method afterSpiStopped.

/**
 * @throws Exception If error.
 */
@Override
protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(), IgniteS3TestSuite.getSecretKey());
    AmazonS3 s3 = new AmazonS3Client(cred);
    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) s3.deleteObject(bucketName, sum.getKey());
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}

Example 17 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project aws-xray-sdk-java by aws.

the class TracingHandlerTest method testS3CopyObjectSubsegmentContainsBucketName.

@Test
public void testS3CopyObjectSubsegmentContainsBucketName() {
    // Setup test
    final String copyResponse = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">" + "<LastModified>2018-01-21T10:09:54.000Z</LastModified><ETag>&quot;31748afd7b576119d3c2471f39fc7a55&quot;</ETag>" + "</CopyObjectResult>";
    AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRequestHandlers(new TracingHandler()).withRegion(Regions.US_EAST_1).withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("fake", "fake"))).build();
    mockHttpClient(s3, copyResponse);
    final String BUCKET = "source-bucket", KEY = "source-key", DST_BUCKET = "dest-bucket", DST_KEY = "dest-key";
    // Test logic
    Segment segment = AWSXRay.beginSegment("test");
    s3.copyObject(BUCKET, KEY, DST_BUCKET, DST_KEY);
    Assert.assertEquals(1, segment.getSubsegments().size());
    Assert.assertEquals("CopyObject", segment.getSubsegments().get(0).getAws().get("operation"));
    System.out.println(segment.getSubsegments().get(0).getAws());
    Assert.assertEquals(BUCKET, segment.getSubsegments().get(0).getAws().get("source_bucket_name"));
    Assert.assertEquals(KEY, segment.getSubsegments().get(0).getAws().get("source_key"));
    Assert.assertEquals(DST_BUCKET, segment.getSubsegments().get(0).getAws().get("destination_bucket_name"));
    Assert.assertEquals(DST_KEY, segment.getSubsegments().get(0).getAws().get("destination_key"));
}

Example 18 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project nifi by apache.

the class FetchS3Object method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue();
    final AmazonS3 client = getClient();
    final GetObjectRequest request;
    if (versionId == null) {
        request = new GetObjectRequest(bucket, key);
    } else {
        request = new GetObjectRequest(bucket, key, versionId);
    }
    final Map<String, String> attributes = new HashMap<>();
    try (final S3Object s3Object = client.getObject(request)) {
        flowFile = session.importFrom(s3Object.getObjectContent(), flowFile);
        attributes.put("s3.bucket", s3Object.getBucketName());
        final ObjectMetadata metadata = s3Object.getObjectMetadata();
        if (metadata.getContentDisposition() != null) {
            final String fullyQualified = metadata.getContentDisposition();
            final int lastSlash = fullyQualified.lastIndexOf("/");
            if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) {
                attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash));
                attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified);
                attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1));
            } else {
                attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
            }
        }
        if (metadata.getContentMD5() != null) {
            attributes.put("hash.value", metadata.getContentMD5());
            attributes.put("hash.algorithm", "MD5");
        }
        if (metadata.getContentType() != null) {
            attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType());
        }
        if (metadata.getETag() != null) {
            attributes.put("s3.etag", metadata.getETag());
        }
        if (metadata.getExpirationTime() != null) {
            attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime()));
        }
        if (metadata.getExpirationTimeRuleId() != null) {
            attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId());
        }
        if (metadata.getUserMetadata() != null) {
            attributes.putAll(metadata.getUserMetadata());
        }
        if (metadata.getSSEAlgorithm() != null) {
            attributes.put("s3.sseAlgorithm", metadata.getSSEAlgorithm());
        }
        if (metadata.getVersionId() != null) {
            attributes.put("s3.version", metadata.getVersionId());
        }
    } catch (final IOException | AmazonClientException ioe) {
        getLogger().error("Failed to retrieve S3 Object for {}; routing to failure", new Object[] { flowFile, ioe });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    if (!attributes.isEmpty()) {
        flowFile = session.putAllAttributes(flowFile, attributes);
    }
    session.transfer(flowFile, REL_SUCCESS);
    final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success", new Object[] { flowFile, transferMillis });
    session.getProvenanceReporter().fetch(flowFile, "http://" + bucket + ".amazonaws.com/" + key, transferMillis);
}

Example 19 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project aws-doc-sdk-examples by awsdocs.

the class GetBucketPolicy method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "Usage:\n" + "    GetBucketPolicy <bucket>\n\n" + "Where:\n" + "    bucket - the bucket to get the policy from.\n\n" + "Example:\n" + "    GetBucketPolicy testbucket\n\n";
    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    String policy_text = null;
    System.out.format("Getting policy for bucket: \"%s\"\n\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    try {
        BucketPolicy bucket_policy = s3.getBucketPolicy(bucket_name);
        policy_text = bucket_policy.getPolicyText();
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    if (policy_text == null) {
        System.out.println("The specified bucket has no bucket policy.");
    } else {
        System.out.println("Returned policy:");
        System.out.println("----");
        System.out.println(policy_text);
        System.out.println("----\n");
    }
    System.out.println("Done!");
}

Example 20 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project aws-doc-sdk-examples by awsdocs.

the class GetObject method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket and object to\n" + "download from it.\n" + "\n" + "Ex: GetObject <bucketname> <filename>\n";
    if (args.length < 2) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    String key_name = args[1];
    System.out.format("Downloading %s from S3 bucket %s...\n", key_name, bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    try {
        S3Object o = s3.getObject(bucket_name, key_name);
        S3ObjectInputStream s3is = o.getObjectContent();
        FileOutputStream fos = new FileOutputStream(new File(key_name));
        byte[] read_buf = new byte[1024];
        int read_len = 0;
        while ((read_len = s3is.read(read_buf)) > 0) {
            fos.write(read_buf, 0, read_len);
        }
        s3is.close();
        fos.close();
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    } catch (FileNotFoundException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    } catch (IOException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}