Examples with AmazonS3Client com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client used on opensource projects

Example 16 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project ignite by apache.

the class S3CheckpointSpi method spiStart.

/**
 * {@inheritDoc}
 */
@SuppressWarnings({ "BusyWait" })
@Override
public void spiStart(String igniteInstanceName) throws IgniteSpiException {
    // Start SPI start stopwatch.
    startStopwatch();
    assertParameter(cred != null, "awsCredentials != null");
    if (log.isDebugEnabled()) {
        log.debug(configInfo("awsCredentials", cred));
        log.debug(configInfo("clientConfiguration", cfg));
        log.debug(configInfo("bucketNameSuffix", bucketNameSuffix));
        log.debug(configInfo("bucketEndpoint", bucketEndpoint));
        log.debug(configInfo("SSEAlgorithm", sseAlg));
    }
    if (cfg == null)
        U.warn(log, "Amazon client configuration is not set (will use default).");
    if (F.isEmpty(bucketNameSuffix)) {
        U.warn(log, "Bucket name suffix is null or empty (will use default bucket name).");
        bucketName = BUCKET_NAME_PREFIX + DFLT_BUCKET_NAME_SUFFIX;
    } else
        bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix;
    s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred);
    if (!F.isEmpty(bucketEndpoint))
        s3.setEndpoint(bucketEndpoint);
    if (!s3.doesBucketExist(bucketName)) {
        try {
            s3.createBucket(bucketName);
            if (log.isDebugEnabled())
                log.debug("Created S3 bucket: " + bucketName);
            while (!s3.doesBucketExist(bucketName)) try {
                U.sleep(200);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteSpiException("Thread has been interrupted.", e);
            }
        } catch (AmazonClientException e) {
            try {
                if (!s3.doesBucketExist(bucketName))
                    throw new IgniteSpiException("Failed to create bucket: " + bucketName, e);
            } catch (AmazonClientException ignored) {
                throw new IgniteSpiException("Failed to create bucket: " + bucketName, e);
            }
        }
    }
    Collection<S3TimeData> s3TimeDataLst = new LinkedList<>();
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) {
                S3CheckpointData data = read(sum.getKey());
                if (data != null) {
                    s3TimeDataLst.add(new S3TimeData(data.getExpireTime(), data.getKey()));
                    if (log.isDebugEnabled())
                        log.debug("Registered existing checkpoint from key: " + data.getKey());
                }
            }
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    } catch (IgniteCheckedException e) {
        throw new IgniteSpiException("Failed to marshal/unmarshal objects in bucket: " + bucketName, e);
    }
    // Track expiration for only those data that are made by this node
    timeoutWrk = new S3TimeoutWorker();
    timeoutWrk.add(s3TimeDataLst);
    timeoutWrk.start();
    registerMBean(igniteInstanceName, new S3CheckpointSpiMBeanImpl(this), S3CheckpointSpiMBean.class);
    // Ack ok start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}

Example 17 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project ignite by apache.

the class S3CheckpointSpiSelfTest method afterSpiStopped.

/**
 * @throws Exception If error.
 */
@Override
protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(), IgniteS3TestSuite.getSecretKey());
    AmazonS3 s3 = new AmazonS3Client(cred);
    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) s3.deleteObject(bucketName, sum.getKey());
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}

Example 18 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class TestPutS3Object method testSignerOverrideOptions.

@Test
public void testSignerOverrideOptions() {
    final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
    final ClientConfiguration config = new ClientConfiguration();
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final List<AllowableValue> allowableSignerValues = PutS3Object.SIGNER_OVERRIDE.getAllowableValues();
    final String defaultSignerValue = PutS3Object.SIGNER_OVERRIDE.getDefaultValue();
    for (AllowableValue allowableSignerValue : allowableSignerValues) {
        String signerType = allowableSignerValue.getValue();
        if (!signerType.equals(defaultSignerValue)) {
            runner.setProperty(PutS3Object.SIGNER_OVERRIDE, signerType);
            ProcessContext context = runner.getProcessContext();
            try {
                AmazonS3Client s3Client = processor.createClient(context, credentialsProvider, config);
            } catch (IllegalArgumentException argEx) {
                Assert.fail(argEx.getMessage());
            }
        }
    }
}

Example 19 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class AbstractS3Processor method createClient.

/**
 * Create client using AWSCredentials
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonS3Client createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client with AWS credentials");
    initializeSignerOverride(context, config);
    final AmazonS3Client s3 = new AmazonS3Client(credentials, config);
    initalizeEndpointOverride(context, s3);
    return s3;
}

Example 20 with AmazonS3Client

use of com.talend.shaded.com.amazonaws.services.s3.AmazonS3Client in project nifi by apache.

the class MockAWSProcessor method createClient.

/**
 * Create client using AWSCredentials
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonS3Client createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client with awd credentials");
    final AmazonS3Client s3 = new AmazonS3Client(credentials, config);
    return s3;
}