Examples with S3Object org.jets3t.service.model.S3Object used on opensource projects

Example 26 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentPusherTest method testPush.

@Test
public void testPush() throws Exception {
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    EasyMock.expect(s3Client.putObject(EasyMock.anyString(), EasyMock.<S3Object>anyObject())).andReturn(null).atLeastOnce();
    EasyMock.replay(s3Client);
    S3DataSegmentPusherConfig config = new S3DataSegmentPusherConfig();
    config.setBucket("bucket");
    config.setBaseKey("key");
    S3DataSegmentPusher pusher = new S3DataSegmentPusher(s3Client, config, new DefaultObjectMapper());
    // Create a mock segment on disk
    File tmp = tempFolder.newFile("version.bin");
    final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 };
    Files.write(data, tmp);
    final long size = data.length;
    DataSegment segmentToPush = new DataSegment("foo", new Interval("2015/2016"), "0", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), NoneShardSpec.instance(), 0, size);
    DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush);
    Assert.assertEquals(segmentToPush.getSize(), segment.getSize());
    EasyMock.verify(s3Client);
}

Example 27 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class StaticS3FirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");
    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();
            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());
            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);
            try {
                final InputStream innerInputStream = s3Client.getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();
                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 28 with S3Object

use of org.jets3t.service.model.S3Object in project druid by druid-io.

the class S3DataSegmentPusher method push.

@Override
public DataSegment push(final File indexFilesDir, final DataSegment inSegment) throws IOException {
    final String s3Path = S3Utils.constructSegmentPath(config.getBaseKey(), inSegment);
    log.info("Copying segment[%s] to S3 at location[%s]", inSegment.getIdentifier(), s3Path);
    final File zipOutFile = File.createTempFile("druid", "index.zip");
    final long indexSize = CompressionUtils.zip(indexFilesDir, zipOutFile);
    try {
        return S3Utils.retryS3Operation(new Callable<DataSegment>() {

            @Override
            public DataSegment call() throws Exception {
                S3Object toPush = new S3Object(zipOutFile);
                final String outputBucket = config.getBucket();
                final String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path);
                toPush.setBucketName(outputBucket);
                toPush.setKey(s3Path);
                if (!config.getDisableAcl()) {
                    toPush.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL);
                }
                log.info("Pushing %s.", toPush);
                s3Client.putObject(outputBucket, toPush);
                final DataSegment outSegment = inSegment.withSize(indexSize).withLoadSpec(ImmutableMap.<String, Object>of("type", "s3_zip", "bucket", outputBucket, "key", toPush.getKey())).withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir));
                File descriptorFile = File.createTempFile("druid", "descriptor.json");
                Files.copy(ByteStreams.newInputStreamSupplier(jsonMapper.writeValueAsBytes(inSegment)), descriptorFile);
                S3Object descriptorObject = new S3Object(descriptorFile);
                descriptorObject.setBucketName(outputBucket);
                descriptorObject.setKey(s3DescriptorPath);
                if (!config.getDisableAcl()) {
                    descriptorObject.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL);
                }
                log.info("Pushing %s", descriptorObject);
                s3Client.putObject(outputBucket, descriptorObject);
                log.info("Deleting zipped index File[%s]", zipOutFile);
                zipOutFile.delete();
                log.info("Deleting descriptor file[%s]", descriptorFile);
                descriptorFile.delete();
                return outSegment;
            }
        });
    } catch (ServiceException e) {
        throw new IOException(e);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 29 with S3Object

use of org.jets3t.service.model.S3Object in project hadoop by apache.

the class Jets3tNativeFileSystemStore method storeEmptyFile.

@Override
public void storeEmptyFile(String key) throws IOException {
    try {
        S3Object object = new S3Object(key);
        object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
        object.setContentType("binary/octet-stream");
        object.setContentLength(0);
        object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
        s3Service.putObject(bucket, object);
    } catch (ServiceException e) {
        handleException(e, key);
    }
}

Example 30 with S3Object

use of org.jets3t.service.model.S3Object in project hadoop by apache.

the class Jets3tNativeFileSystemStore method dump.

@Override
public void dump() throws IOException {
    StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
    sb.append(bucket.getName()).append("\n");
    try {
        S3Object[] objects = s3Service.listObjects(bucket.getName());
        for (S3Object object : objects) {
            sb.append(object.getKey()).append("\n");
        }
    } catch (S3ServiceException e) {
        handleException(e);
    }
    System.out.println(sb);
}