Examples with Region com.amazonaws.services.s3.model.Region used on opensource projects

Example 11 with Region

use of com.amazonaws.services.s3.model.Region in project hadoop by apache.

the class ITestS3AConfiguration method testEndpoint.

/**
   * Test if custom endpoint is picked up.
   * <p>
   * The test expects {@link S3ATestConstants#CONFIGURATION_TEST_ENDPOINT}
   * to be defined in the Configuration
   * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
   * (i.e. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
   * Evidently, the bucket has to be hosted in the region denoted by the
   * endpoint for the test to succeed.
   * <p>
   * More info and the list of endpoint identifiers:
   * @see <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">endpoint list</a>.
   *
   * @throws Exception
   */
@Test
public void testEndpoint() throws Exception {
    conf = new Configuration();
    String endpoint = conf.getTrimmed(S3ATestConstants.CONFIGURATION_TEST_ENDPOINT, "");
    if (endpoint.isEmpty()) {
        LOG.warn("Custom endpoint test skipped as " + S3ATestConstants.CONFIGURATION_TEST_ENDPOINT + "config " + "setting was not detected");
    } else {
        conf.set(Constants.ENDPOINT, endpoint);
        fs = S3ATestUtils.createTestFileSystem(conf);
        AmazonS3 s3 = fs.getAmazonS3Client();
        String endPointRegion = "";
        // Differentiate handling of "s3-" and "s3." based endpoint identifiers
        String[] endpointParts = StringUtils.split(endpoint, '.');
        if (endpointParts.length == 3) {
            endPointRegion = endpointParts[0].substring(3);
        } else if (endpointParts.length == 4) {
            endPointRegion = endpointParts[1];
        } else {
            fail("Unexpected endpoint");
        }
        assertEquals("Endpoint config setting and bucket location differ: ", endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
    }
}

Example 12 with Region

use of com.amazonaws.services.s3.model.Region in project hadoop by apache.

the class ITestS3AConfiguration method shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty.

@Test
public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() throws Exception {
    conf = new Configuration();
    conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
    assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
    try {
        fs = S3ATestUtils.createTestFileSystem(conf);
        assertNotNull(fs);
        AmazonS3 s3 = fs.getAmazonS3Client();
        assertNotNull(s3);
        S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class, "clientOptions");
        assertTrue("Expected to find path style access to be switched on!", clientOptions.isPathStyleAccess());
        byte[] file = ContractTestUtils.toAsciiByteArray("test file");
        ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, (int) conf.getLongBytes(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
    } catch (final AWSS3IOException e) {
        LOG.error("Caught exception: ", e);
        // Catch/pass standard path style access behaviour when live bucket
        // isn't in the same region as the s3 client default. See
        // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
        assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
    }
}

Example 13 with Region

use of com.amazonaws.services.s3.model.Region in project exhibitor by soabase.

the class S3ClientImpl method createClient.

private AmazonS3Client createClient(AWSCredentialsProvider awsCredentialProvider, BasicAWSCredentials basicAWSCredentials, S3ClientConfig clientConfig) {
    AmazonS3Client localClient;
    if (awsCredentialProvider != null) {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(awsCredentialProvider, clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client(awsCredentialProvider);
        }
    } else if (basicAWSCredentials != null) {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(basicAWSCredentials, clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client(basicAWSCredentials);
        }
    } else {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client();
        }
    }
    if (s3Region != null) {
        String fixedRegion = s3Region.equals("us-east-1") ? "" : ("-" + s3Region);
        String endpoint = ENDPOINT_SPEC.replace("$REGION$", fixedRegion);
        localClient.setEndpoint(endpoint);
        log.info("Setting S3 endpoint to: " + endpoint);
    }
    return localClient;
}

Example 14 with Region

use of com.amazonaws.services.s3.model.Region in project jackrabbit by apache.

the class S3Backend method init.

public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        startTime = new Date();
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        LOG.debug("init");
        setDataStore(store);
        s3ReqDecorator = new S3RequestDecorator(prop);
        s3service = Utils.openService(prop);
        if (bucket == null || "".equals(bucket.trim())) {
            bucket = prop.getProperty(S3Constants.S3_BUCKET);
        }
        String region = prop.getProperty(S3Constants.S3_REGION);
        Region s3Region = null;
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
            if (ec2Region != null) {
                s3Region = Region.fromValue(ec2Region.getName());
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
            }
        } else {
            if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                s3Region = Region.US_Standard;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                s3Region = Region.EU_Ireland;
            } else {
                s3Region = Region.fromValue(region);
            }
        }
        if (!s3service.doesBucketExist(bucket)) {
            s3service.createBucket(bucket, s3Region);
            LOG.info("Created bucket [{}] in [{}] ", bucket, region);
        } else {
            LOG.info("Using bucket [{}] in [{}] ", bucket, region);
        }
        int writeThreads = 10;
        String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
        if (writeThreadsStr != null) {
            writeThreads = Integer.parseInt(writeThreadsStr);
        }
        LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
        tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
        int asyncWritePoolSize = 10;
        String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
        if (maxConnsStr != null) {
            asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
        }
        setAsyncWritePoolSize(asyncWritePoolSize);
        String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
        boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
        LOG.info("Rename keys [{}]", renameKeyBool);
        if (renameKeyBool) {
            renameKeys();
        }
        LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
    } catch (Exception e) {
        LOG.debug("  error ", e);
        throw new DataStoreException("Could not initialize S3 from " + prop, e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 15 with Region

use of com.amazonaws.services.s3.model.Region in project jackrabbit-oak by apache.

the class Utils method openService.

/**
     * Create AmazonS3Client from properties.
     * 
     * @param prop properties to configure @link {@link AmazonS3Client}
     * @return {@link AmazonS3Client}
     */
public static AmazonS3Client openService(final Properties prop) {
    String accessKey = prop.getProperty(S3Constants.ACCESS_KEY);
    String secretKey = prop.getProperty(S3Constants.SECRET_KEY);
    AmazonS3Client s3service = null;
    if (StringUtils.isNullOrEmpty(accessKey) || StringUtils.isNullOrEmpty(secretKey)) {
        LOG.info("Configuring Amazon Client from environment");
        s3service = new AmazonS3Client(getClientConfiguration(prop));
    } else {
        LOG.info("Configuring Amazon Client from property file.");
        AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
        s3service = new AmazonS3Client(credentials, getClientConfiguration(prop));
    }
    String region = prop.getProperty(S3Constants.S3_REGION);
    String endpoint = null;
    String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
    if ((propEndPoint != null) & !"".equals(propEndPoint)) {
        endpoint = propEndPoint;
    } else {
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region s3Region = Regions.getCurrentRegion();
            if (s3Region != null) {
                region = s3Region.getName();
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
            }
        }
        if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
            endpoint = S3 + DOT + AWSDOTCOM;
        } else if (Region.EU_Ireland.toString().equals(region)) {
            endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM;
        } else {
            endpoint = S3 + DASH + region + DOT + AWSDOTCOM;
        }
    }
    /*
         * setting endpoint to remove latency of redirection. If endpoint is
         * not set, invocation first goes us standard region, which
         * redirects it to correct location.
         */
    s3service.setEndpoint(endpoint);
    LOG.info("S3 service endpoint [{}] ", endpoint);
    return s3service;
}