Examples with S3Object org.jets3t.service.model.S3Object used on opensource projects

Example 21 with S3Object

use of org.jets3t.service.model.S3Object in project OpenTripPlanner by opentripplanner.

the class DegreeGridNEDTileSource method getPathToTile.

private File getPathToTile(int x, int y) {
    File path = new File(cacheDirectory, formatLatLon(x, y) + ".tiff");
    if (path.exists()) {
        return path;
    } else {
        path.getParentFile().mkdirs();
        if (awsAccessKey == null || awsSecretKey == null) {
            throw new RuntimeException("Cannot download NED tiles from S3: awsAccessKey or awsSecretKey properties are not set");
        }
        log.info("Downloading NED degree tile " + path);
        // download the file from S3.
        AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
        try {
            S3Service s3Service = new RestS3Service(awsCredentials);
            String key = formatLatLon(x, y) + ".tiff";
            S3Object object = s3Service.getObject(awsBucketName, key);
            InputStream istream = object.getDataInputStream();
            FileOutputStream ostream = new FileOutputStream(path);
            byte[] buffer = new byte[4096];
            while (true) {
                int read = istream.read(buffer);
                if (read == -1) {
                    break;
                }
                ostream.write(buffer, 0, read);
            }
            ostream.close();
            istream.close();
        } catch (S3ServiceException e) {
            path.deleteOnExit();
            throw new RuntimeException(e);
        } catch (ServiceException e) {
            path.deleteOnExit();
            throw new RuntimeException(e);
        } catch (FileNotFoundException e) {
            path.deleteOnExit();
            throw new RuntimeException(e);
        } catch (IOException e) {
            path.deleteOnExit();
            throw new RuntimeException(e);
        }
        return path;
    }
}

Example 22 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testSimpleLatestVersion.

@Test
public void testSimpleLatestVersion() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object(), object1 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object1.setBucketName(bucket);
    object1.setKey(keyPrefix + "/renames-1.gz");
    object1.setLastModifiedDate(new Date(1));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(new S3Object[] { object0, object1 }).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, keyPrefix)), pattern);
    EasyMock.verify(s3Client);
    URI expected = URI.create(String.format("s3://%s/%s", bucket, object1.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 23 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentFinderTest method setUp.

@Before
public void setUp() throws Exception {
    bucket = "bucket1";
    baseKey = "dataSource1";
    config = new S3DataSegmentPusherConfig();
    config.setBucket(bucket);
    config.setBaseKey(baseKey);
    mockS3Client = new MockStorageService(temporaryFolder.newFolder());
    descriptor1 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval1/v1/0/");
    descriptor2 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval2/v1/0/");
    descriptor3 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval3/v2/0/");
    descriptor4_0 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval4/v1/0/");
    descriptor4_1 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval4/v1/1/");
    indexZip1 = S3Utils.indexZipForSegmentPath(descriptor1);
    indexZip2 = S3Utils.indexZipForSegmentPath(descriptor2);
    indexZip3 = S3Utils.indexZipForSegmentPath(descriptor3);
    indexZip4_0 = S3Utils.indexZipForSegmentPath(descriptor4_0);
    indexZip4_1 = S3Utils.indexZipForSegmentPath(descriptor4_1);
    mockS3Client.putObject(bucket, new S3Object(descriptor1, mapper.writeValueAsString(SEGMENT_1)));
    mockS3Client.putObject(bucket, new S3Object(descriptor2, mapper.writeValueAsString(SEGMENT_2)));
    mockS3Client.putObject(bucket, new S3Object(descriptor3, mapper.writeValueAsString(SEGMENT_3)));
    mockS3Client.putObject(bucket, new S3Object(descriptor4_0, mapper.writeValueAsString(SEGMENT_4_0)));
    mockS3Client.putObject(bucket, new S3Object(descriptor4_1, mapper.writeValueAsString(SEGMENT_4_1)));
    mockS3Client.putObject(bucket, new S3Object(indexZip1, "dummy"));
    mockS3Client.putObject(bucket, new S3Object(indexZip2, "dummy"));
    mockS3Client.putObject(bucket, new S3Object(indexZip3, "dummy"));
    mockS3Client.putObject(bucket, new S3Object(indexZip4_0, "dummy"));
    mockS3Client.putObject(bucket, new S3Object(indexZip4_1, "dummy"));
}

Example 24 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentMoverTest method testMove.

@Test
public void testMove() throws Exception {
    MockStorageService mockS3Client = new MockStorageService();
    S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig());
    mockS3Client.putObject("main", new S3Object("baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip"));
    mockS3Client.putObject("main", new S3Object("baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json"));
    DataSegment movedSegment = mover.move(sourceSegment, ImmutableMap.<String, Object>of("baseKey", "targetBaseKey", "bucket", "archive"));
    Map<String, Object> targetLoadSpec = movedSegment.getLoadSpec();
    Assert.assertEquals("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", MapUtils.getString(targetLoadSpec, "key"));
    Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec, "bucket"));
    Assert.assertTrue(mockS3Client.didMove());
}

Example 25 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentPullerTest method testGZUncompressRetries.

@Test
public void testGZUncompressRetries() throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));
    File tmpDir = temporaryFolder.newFolder("gzTestDir");
    S3ServiceException exception = new S3ServiceException();
    exception.setErrorCode("NoSuchKey");
    exception.setResponseCode(404);
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(null).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andThrow(exception).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}