Examples with AmazonS3 com.talend.shaded.com.amazonaws.services.s3.AmazonS3 used on opensource projects

Example 71 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project nifi by apache.

the class ListS3 method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    try {
        restoreState(context);
    } catch (IOException ioe) {
        getLogger().error("Failed to restore processor state; yielding", ioe);
        context.yield();
        return;
    }
    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue();
    final long minAgeMilliseconds = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
    final long listingTimestamp = System.currentTimeMillis();
    final AmazonS3 client = getClient();
    int listCount = 0;
    long maxTimestamp = 0L;
    String delimiter = context.getProperty(DELIMITER).getValue();
    String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue();
    boolean useVersions = context.getProperty(USE_VERSIONS).asBoolean();
    int listType = context.getProperty(LIST_TYPE).asInteger();
    S3BucketLister bucketLister = useVersions ? new S3VersionBucketLister(client) : listType == 2 ? new S3ObjectBucketListerVersion2(client) : new S3ObjectBucketLister(client);
    bucketLister.setBucketName(bucket);
    if (delimiter != null && !delimiter.isEmpty()) {
        bucketLister.setDelimiter(delimiter);
    }
    if (prefix != null && !prefix.isEmpty()) {
        bucketLister.setPrefix(prefix);
    }
    VersionListing versionListing;
    do {
        versionListing = bucketLister.listVersions();
        for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
            long lastModified = versionSummary.getLastModified().getTime();
            if (lastModified < currentTimestamp || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey()) || lastModified > (listingTimestamp - minAgeMilliseconds)) {
                continue;
            }
            // Create the attributes
            final Map<String, String> attributes = new HashMap<>();
            attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getKey());
            attributes.put("s3.bucket", versionSummary.getBucketName());
            if (versionSummary.getOwner() != null) {
                // We may not have permission to read the owner
                attributes.put("s3.owner", versionSummary.getOwner().getId());
            }
            attributes.put("s3.etag", versionSummary.getETag());
            attributes.put("s3.lastModified", String.valueOf(lastModified));
            attributes.put("s3.length", String.valueOf(versionSummary.getSize()));
            attributes.put("s3.storeClass", versionSummary.getStorageClass());
            attributes.put("s3.isLatest", String.valueOf(versionSummary.isLatest()));
            if (versionSummary.getVersionId() != null) {
                attributes.put("s3.version", versionSummary.getVersionId());
            }
            // Create the flowfile
            FlowFile flowFile = session.create();
            flowFile = session.putAllAttributes(flowFile, attributes);
            session.transfer(flowFile, REL_SUCCESS);
            // Update state
            if (lastModified > maxTimestamp) {
                maxTimestamp = lastModified;
                currentKeys.clear();
            }
            if (lastModified == maxTimestamp) {
                currentKeys.add(versionSummary.getKey());
            }
            listCount++;
        }
        bucketLister.setNextMarker();
        commit(context, session, listCount);
        listCount = 0;
    } while (bucketLister.isTruncated());
    currentTimestamp = maxTimestamp;
    final long listMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully listed S3 bucket {} in {} millis", new Object[] { bucket, listMillis });
    if (!commit(context, session, listCount)) {
        if (currentTimestamp > 0) {
            persistState(context);
        }
        getLogger().debug("No new objects in S3 bucket {} to list. Yielding.", new Object[] { bucket });
        context.yield();
    }
}

Example 72 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project athenz by yahoo.

the class AwsPrivateKeyStoreTest method testAwsPrivateKeyStore.

@Test
public void testAwsPrivateKeyStore() throws Exception {
    String bucketName = "my_bucket";
    String keyName = "my_key";
    String expected = "my_value";
    AmazonS3 s3 = Mockito.mock(AmazonS3.class);
    AWSKMS kms = Mockito.mock(AWSKMS.class);
    S3Object s3Object = Mockito.mock(S3Object.class);
    Mockito.when(s3.getObject(bucketName, keyName)).thenReturn(s3Object);
    InputStream is = new ByteArrayInputStream(expected.getBytes());
    S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(is, null);
    Mockito.when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);
    String result = expected;
    ByteBuffer buffer = ByteBuffer.wrap(result.getBytes());
    DecryptResult decryptResult = Mockito.mock(DecryptResult.class);
    Mockito.when(kms.decrypt(Mockito.any(DecryptRequest.class))).thenReturn(decryptResult);
    Mockito.when(decryptResult.getPlaintext()).thenReturn(buffer);
    AwsPrivateKeyStore awsPrivateKeyStore = new AwsPrivateKeyStore(s3, kms);
    String actual = awsPrivateKeyStore.getApplicationSecret(bucketName, keyName);
    Assert.assertEquals(actual, expected);
}

Example 73 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project aws-cf-templates by widdix.

the class ACloudFormationTest method createStack.

protected final void createStack(final String stackName, final String template, final Parameter... parameters) {
    CreateStackRequest req = new CreateStackRequest().withStackName(stackName).withParameters(parameters).withCapabilities(Capability.CAPABILITY_IAM);
    if (Config.has(Config.Key.TEMPLATE_DIR)) {
        final String dir = Config.get(Config.Key.TEMPLATE_DIR);
        if (Config.has(Config.Key.BUCKET_NAME)) {
            final String bucketName = Config.get(Config.Key.BUCKET_NAME);
            final String bucketRegion = Config.get(Config.Key.BUCKET_REGION);
            final AmazonS3 s3local = AmazonS3ClientBuilder.standard().withCredentials(this.credentialsProvider).withRegion(bucketRegion).build();
            s3local.putObject(bucketName, stackName, new File(dir + template));
            req = req.withTemplateURL("https://s3-" + bucketRegion + ".amazonaws.com/" + bucketName + "/" + stackName);
        } else {
            final String body = readFile(dir + template, Charset.forName("UTF-8"));
            req = req.withTemplateBody(body);
        }
    } else {
        req = req.withTemplateURL("https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates/" + template);
    }
    this.cf.createStack(req);
    this.waitForStack(stackName, FinalStatus.CREATE_COMPLETE);
}

Example 74 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project bayou by capergroup.

the class S3LoggerBase method putToS3.

/**
 * Performs an S3 Put Object operation storing the UTF-8 bytes of logMsg under the given key
 * using construction provided AWS credentials.
 *
 * @param objectKey the S3 object key. may not be null or whitespace only.
 * @param logMsg the message to store
 * @throws IllegalArgumentException if objectKey is whitespace only.
 */
void putToS3(String objectKey, String logMsg) {
    if (objectKey == null)
        throw new NullPointerException("objectKey");
    if (objectKey.trim().length() == 0)
        throw new IllegalArgumentException("objectKey may not be only whitespace.");
    /*
         * Make the client used to send the log msg to S3.
         */
    AmazonS3 client;
    {
        Regions regions = Regions.US_EAST_1;
        if (_credentials == null) {
            client = // get creds from environment
            AmazonS3ClientBuilder.standard().withRegion(regions).build();
        } else {
            client = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(_credentials)).withRegion(regions).build();
        }
    }
    /*
         * Store the log msg in S3.
         */
    byte[] logMsgBytes = logMsg.getBytes(StandardCharsets.UTF_8);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(logMsgBytes.length);
    client.putObject(_bucketName, objectKey, new ByteArrayInputStream(logMsgBytes), metadata);
    _logger.debug("exiting");
}

Example 75 with AmazonS3

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3 in project FP-PSP-SERVER by FundacionParaguaya.

the class ImageUploadServiceImpl method uploadImage.

@Override
public String uploadImage(ImageDTO imageDTO) {
    if (imageDTO == null) {
        return null;
    }
    String url;
    try {
        String strRegion = applicationProperties.getAws().getStrRegion();
        Regions region = Regions.valueOf(strRegion);
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(region).build();
        String bucketName = applicationProperties.getAws().getBucketName();
        String imageDirectory = imageDTO.getImageDirectory();
        String imageName = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
        String fileName = imageName + "." + imageDTO.getFormat();
        String keyName = imageDirectory + fileName;
        s3Client.putObject(new PutObjectRequest(bucketName, keyName, imageDTO.getFile()).withCannedAcl(CannedAccessControlList.PublicRead));
        url = "https://s3." + s3Client.getRegionName() + ".amazonaws.com/" + bucketName + "/" + keyName;
    } catch (SdkClientException sdkClientExc) {
        LOG.error(sdkClientExc.getMessage(), sdkClientExc);
        throw new AWSS3RuntimeException(sdkClientExc);
    }
    return url;
}