Examples with S3Bucket org.jets3t.service.model.S3Bucket used on opensource projects

Example 6 with S3Bucket

use of org.jets3t.service.model.S3Bucket in project tutorials by eugenp.

the class JetS3tLiveTest method givenStreamDataUploaded_StreamDataIsDownloaded.

@Test
public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {
    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);
    // Create a stream
    ByteArrayInputStream testStream = new ByteArrayInputStream("test stream".getBytes());
    // Create and populate object
    S3Object streamObject = new S3Object("stream");
    streamObject.setDataInputStream(testStream);
    streamObject.setContentLength(testStream.available());
    streamObject.setContentType("text/plain");
    // Put it
    s3Service.putObject(BucketName, streamObject);
    // Get it
    S3Object newStreamObject = s3Service.getObject(BucketName, "stream");
    // Convert back to string
    String testString = new BufferedReader(new InputStreamReader(newStreamObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));
    assertTrue("test stream".equals(testString));
    // Clean up
    deleteObject("stream");
    deleteBucket();
}

Example 7 with S3Bucket

use of org.jets3t.service.model.S3Bucket in project tutorials by eugenp.

the class JetS3tLiveTest method createBucket.

private S3Bucket createBucket() throws Exception {
    S3Bucket bucket = s3Service.createBucket(BucketName);
    log.info(bucket);
    return bucket;
}

Example 8 with S3Bucket

use of org.jets3t.service.model.S3Bucket in project druid by druid-io.

the class StaticS3FirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");
    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();
            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());
            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);
            try {
                final InputStream innerInputStream = s3Client.getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();
                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 9 with S3Bucket

use of org.jets3t.service.model.S3Bucket in project twitter-2-weibo by rjyo.

the class S3BackupTask method run.

public void run() {
    String dump = getHelper().dump();
    try {
        AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretAccessKey);
        S3Service s3Service = new RestS3Service(awsCredentials);
        S3Bucket backup = s3Service.getBucket("h2weibo.backup");
        String dateString = DateFormatUtils.format(new Date(), "yyyy-MM-dd_HH");
        S3Object object = new S3Object(dateString + ".json", dump);
        object = s3Service.putObject(backup, object);
        System.out.println("S3Object after upload: " + object);
    } catch (Exception e) {
        log.error("Failed to upload to S3.");
        log.error(e);
    }
}

Example 10 with S3Bucket

use of org.jets3t.service.model.S3Bucket in project tutorials by eugenp.

the class JetS3tLiveTest method givenCreate_AndDeleteBucket_CountGoesUpThenDown.

@Test
public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {
    // List buckets, get a count
    S3Bucket[] myBuckets = s3Service.listAllBuckets();
    int count = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
    // Create a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);
    // List again
    myBuckets = s3Service.listAllBuckets();
    int newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
    // We should have one more
    assertEquals((count + 1), newCount);
    // Delete so next test doesn't fail
    deleteBucket();
    // Check the count again, just for laughs
    myBuckets = s3Service.listAllBuckets();
    newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
    assertEquals(count, newCount);
}