Examples with RestS3Service org.jets3t.service.impl.rest.httpclient.RestS3Service used on opensource projects

Example 1 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project hadoop by apache.

the class Jets3tNativeFileSystemStore method initialize.

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    S3Credentials s3Credentials = new S3Credentials();
    s3Credentials.initialize(uri, conf);
    try {
        AWSCredentials awsCredentials = new AWSCredentials(s3Credentials.getAccessKey(), s3Credentials.getSecretAccessKey());
        this.s3Service = new RestS3Service(awsCredentials);
    } catch (S3ServiceException e) {
        handleException(e);
    }
    multipartEnabled = conf.getBoolean("fs.s3n.multipart.uploads.enabled", false);
    multipartBlockSize = Math.min(conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024), MAX_PART_SIZE);
    multipartCopyBlockSize = Math.min(conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE), MAX_PART_SIZE);
    serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");
    bucket = new S3Bucket(uri.getHost());
}

Example 2 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3DataSegmentPullerTest method testSimpleGetVersion.

@Test
public void testSimpleGetVersion() throws ServiceException, IOException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    String version = puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())));
    EasyMock.verify(s3Client);
    Assert.assertEquals(String.format("%d", new Date(0).getTime()), version);
}

Example 3 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3DataSegmentPullerTest method testGZUncompress.

@Test
public void testGZUncompress() throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    final S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));
    final File tmpDir = temporaryFolder.newFolder("gzTestDir");
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(null).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}

Example 4 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testFindExact.

@Test
public void testFindExact() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(new S3Object[] { object0 }).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())), null);
    EasyMock.verify(s3Client);
    URI expected = URI.create(String.format("s3://%s/%s", bucket, object0.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 5 with RestS3Service

use of org.jets3t.service.impl.rest.httpclient.RestS3Service in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testMissing.

@Test
public void testMissing() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object(), object1 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object1.setBucketName(bucket);
    object1.setKey(keyPrefix + "/renames-1.gz");
    object1.setLastModifiedDate(new Date(1));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(null).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, keyPrefix)), pattern);
    EasyMock.verify(s3Client);
    Assert.assertEquals(null, latest);
}