Examples with RestS3Service org.jets3t.service.impl.rest.httpclient.RestS3Service used on opensource projects

Example 6 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testFindSelf.

@Test
public void testFindSelf() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(new S3Object[] { object0 }).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, keyPrefix)), pattern);
    EasyMock.verify(s3Client);
    URI expected = URI.create(String.format("s3://%s/%s", bucket, object0.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 7 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testSimpleLatestVersion.

@Test
public void testSimpleLatestVersion() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object(), object1 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object1.setBucketName(bucket);
    object1.setKey(keyPrefix + "/renames-1.gz");
    object1.setLastModifiedDate(new Date(1));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(new S3Object[] { object0, object1 }).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, keyPrefix)), pattern);
    EasyMock.verify(s3Client);
    URI expected = URI.create(String.format("s3://%s/%s", bucket, object1.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 8 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3DataSegmentPullerTest method testGZUncompressRetries.

@Test
public void testGZUncompressRetries() throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));
    File tmpDir = temporaryFolder.newFolder("gzTestDir");
    S3ServiceException exception = new S3ServiceException();
    exception.setErrorCode("NoSuchKey");
    exception.setResponseCode(404);
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(null).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andThrow(exception).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}

Example 9 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3DataSegmentPusherTest method testPush.

@Test
public void testPush() throws Exception {
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    EasyMock.expect(s3Client.putObject(EasyMock.anyString(), EasyMock.<S3Object>anyObject())).andReturn(null).atLeastOnce();
    EasyMock.replay(s3Client);
    S3DataSegmentPusherConfig config = new S3DataSegmentPusherConfig();
    config.setBucket("bucket");
    config.setBaseKey("key");
    S3DataSegmentPusher pusher = new S3DataSegmentPusher(s3Client, config, new DefaultObjectMapper());
    // Create a mock segment on disk
    File tmp = tempFolder.newFile("version.bin");
    final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 };
    Files.write(data, tmp);
    final long size = data.length;
    DataSegment segmentToPush = new DataSegment("foo", new Interval("2015/2016"), "0", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), NoneShardSpec.instance(), 0, size);
    DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush);
    Assert.assertEquals(segmentToPush.getSize(), segment.getSize());
    EasyMock.verify(s3Client);
}

Example 10 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project alluxio by Alluxio.

the class S3UnderFileSystem method createInstance.

/**
   * Constructs a new instance of {@link S3UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @return the created {@link S3UnderFileSystem} instance
   * @throws ServiceException when a connection to S3 could not be created
   */
public static S3UnderFileSystem createInstance(AlluxioURI uri) throws ServiceException {
    String bucketName = uri.getHost();
    Preconditions.checkArgument(Configuration.containsKey(PropertyKey.S3N_ACCESS_KEY), "Property " + PropertyKey.S3N_ACCESS_KEY + " is required to connect to S3");
    Preconditions.checkArgument(Configuration.containsKey(PropertyKey.S3N_SECRET_KEY), "Property " + PropertyKey.S3N_SECRET_KEY + " is required to connect to S3");
    AWSCredentials awsCredentials = new AWSCredentials(Configuration.get(PropertyKey.S3N_ACCESS_KEY), Configuration.get(PropertyKey.S3N_SECRET_KEY));
    Jets3tProperties props = new Jets3tProperties();
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        props.setProperty("httpclient.proxy-autodetect", "false");
        props.setProperty("httpclient.proxy-host", Configuration.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
        props.setProperty("httpclient.proxy-port", Configuration.get(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
        props.setProperty("s3service.https-only", Boolean.toString(Configuration.getBoolean(PropertyKey.UNDERFS_S3_PROXY_HTTPS_ONLY)));
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ENDPOINT)) {
        props.setProperty("s3service.s3-endpoint", Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT));
        if (Configuration.getBoolean(PropertyKey.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
            props.setProperty("s3service.s3-endpoint-https-port", Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
        } else {
            props.setProperty("s3service.s3-endpoint-http-port", Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT_HTTP_PORT));
        }
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
        props.setProperty("s3service.disable-dns-buckets", Configuration.get(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS));
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX)) {
        props.setProperty("threaded-service.max-thread-count", Configuration.get(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX));
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX)) {
        props.setProperty("threaded-service.admin-max-thread-count", Configuration.get(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX));
    }
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_THREADS_MAX)) {
        props.setProperty("httpclient.max-connections", Configuration.get(PropertyKey.UNDERFS_S3_THREADS_MAX));
    }
    LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
    RestS3Service restS3Service = new RestS3Service(awsCredentials, null, null, props);
    String accountOwnerId = restS3Service.getAccountOwner().getId();
    // Gets the owner from user-defined static mapping from S3 canonical user id to Alluxio
    // user name.
    String owner = CommonUtils.getValueFromStaticMapping(Configuration.get(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING), accountOwnerId);
    // If there is no user-defined mapping, use the display name.
    if (owner == null) {
        owner = restS3Service.getAccountOwner().getDisplayName();
    }
    String accountOwner = owner == null ? accountOwnerId : owner;
    AccessControlList acl = restS3Service.getBucketAcl(bucketName);
    short bucketMode = S3Utils.translateBucketAcl(acl, accountOwnerId);
    return new S3UnderFileSystem(uri, restS3Service, bucketName, bucketMode, accountOwner);
}