Examples with S3Object org.jets3t.service.model.S3Object used on opensource projects

Example 1 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentFinder method findSegments.

@Override
public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException {
    final Set<DataSegment> segments = Sets.newHashSet();
    try {
        Iterator<StorageObject> objectsIterator = S3Utils.storageObjectsIterator(s3Client, config.getBucket(), workingDirPath.length() == 0 ? config.getBaseKey() : workingDirPath, config.getMaxListingLength());
        while (objectsIterator.hasNext()) {
            StorageObject storageObject = objectsIterator.next();
            storageObject.closeDataInputStream();
            if (S3Utils.toFilename(storageObject.getKey()).equals("descriptor.json")) {
                final String descriptorJson = storageObject.getKey();
                String indexZip = S3Utils.indexZipForSegmentPath(descriptorJson);
                if (S3Utils.isObjectInBucket(s3Client, config.getBucket(), indexZip)) {
                    S3Object indexObject = s3Client.getObject(config.getBucket(), descriptorJson);
                    try (InputStream is = indexObject.getDataInputStream()) {
                        final DataSegment dataSegment = jsonMapper.readValue(is, DataSegment.class);
                        log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip);
                        final Map<String, Object> loadSpec = dataSegment.getLoadSpec();
                        if (!loadSpec.get("type").equals(S3StorageDruidModule.SCHEME) || !loadSpec.get("key").equals(indexZip)) {
                            loadSpec.put("type", S3StorageDruidModule.SCHEME);
                            loadSpec.put("key", indexZip);
                            if (updateDescriptor) {
                                log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", descriptorJson, indexObject);
                                S3Object newDescJsonObject = new S3Object(descriptorJson, jsonMapper.writeValueAsString(dataSegment));
                                s3Client.putObject(config.getBucket(), newDescJsonObject);
                            }
                        }
                        segments.add(dataSegment);
                    }
                } else {
                    throw new SegmentLoadingException("index.zip didn't exist at [%s] while descriptor.json exists!?", indexZip);
                }
            }
        }
    } catch (ServiceException e) {
        throw new SegmentLoadingException(e, "Problem interacting with S3");
    } catch (IOException e) {
        throw new SegmentLoadingException(e, "IO exception");
    } catch (Exception e) {
        Throwables.propagateIfInstanceOf(e, SegmentLoadingException.class);
        Throwables.propagate(e);
    }
    return segments;
}

Example 2 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentMoverTest method testMoveNoop.

@Test
public void testMoveNoop() throws Exception {
    MockStorageService mockS3Client = new MockStorageService();
    S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig());
    mockS3Client.putObject("archive", new S3Object("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip"));
    mockS3Client.putObject("archive", new S3Object("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json"));
    DataSegment movedSegment = mover.move(sourceSegment, ImmutableMap.<String, Object>of("baseKey", "targetBaseKey", "bucket", "archive"));
    Map<String, Object> targetLoadSpec = movedSegment.getLoadSpec();
    Assert.assertEquals("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", MapUtils.getString(targetLoadSpec, "key"));
    Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec, "bucket"));
    Assert.assertFalse(mockS3Client.didMove());
}

Example 3 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentPullerTest method testSimpleGetVersion.

@Test
public void testSimpleGetVersion() throws ServiceException, IOException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    String version = puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())));
    EasyMock.verify(s3Client);
    Assert.assertEquals(String.format("%d", new Date(0).getTime()), version);
}

Example 4 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentPullerTest method testGZUncompress.

@Test
public void testGZUncompress() throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");
    final File tmpFile = temporaryFolder.newFile("gzTest.gz");
    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
        outputStream.write(value);
    }
    final S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));
    final File tmpDir = temporaryFolder.newFolder("gzTestDir");
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(null).once();
    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result = puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);
    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
}

Example 5 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3TimestampVersionedDataFinderTest method testFindExact.

@Test
public void testFindExact() throws S3ServiceException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.<String>isNull())).andReturn(new S3Object[] { object0 }).once();
    S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client);
    EasyMock.replay(s3Client);
    URI latest = finder.getLatestVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())), null);
    EasyMock.verify(s3Client);
    URI expected = URI.create(String.format("s3://%s/%s", bucket, object0.getKey()));
    Assert.assertEquals(expected, latest);
}