Examples with S3Object org.jets3t.service.model.S3Object used on opensource projects

Example 11 with S3Object

use of org.jets3t.service.model.S3Object in project hadoop by apache.

the class Jets3tNativeFileSystemStore method retrieve.

/**
   *
   * @param key
   * The key is the object name that is being retrieved from the S3 bucket
   * @return
   * This method returns null if the key is not found
   * @throws IOException
   */
@Override
public InputStream retrieve(String key, long byteRangeStart) throws IOException {
    try {
        LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}", key, bucket.getName(), byteRangeStart);
        S3Object object = s3Service.getObject(bucket, key, null, null, null, null, byteRangeStart, null);
        return object.getDataInputStream();
    } catch (ServiceException e) {
        handleException(e, key);
        return null;
    }
}

Example 12 with S3Object

use of org.jets3t.service.model.S3Object in project hadoop by apache.

the class Jets3tNativeFileSystemStore method purge.

@Override
public void purge(String prefix) throws IOException {
    String key = "";
    try {
        S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
        for (S3Object object : objects) {
            key = object.getKey();
            s3Service.deleteObject(bucket, key);
        }
    } catch (S3ServiceException e) {
        handleException(e, key);
    }
}

Example 13 with S3Object

use of org.jets3t.service.model.S3Object in project alluxio by Alluxio.

the class S3OutputStream method close.

@Override
public void close() throws IOException {
    if (mClosed.getAndSet(true)) {
        return;
    }
    mLocalOutputStream.close();
    try {
        S3Object obj = new S3Object(mKey);
        obj.setBucketName(mBucketName);
        obj.setDataInputFile(mFile);
        obj.setContentLength(mFile.length());
        obj.setContentEncoding(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);
        if (mHash != null) {
            obj.setMd5Hash(mHash.digest());
        } else {
            LOG.warn("MD5 was not computed for: {}", mKey);
        }
        if (MULTIPART_UTIL.isFileLargerThanMaxPartSize(mFile)) {
            // Big object will be split into parts and uploaded to S3 in parallel.
            List<StorageObject> objectsToUploadAsMultipart = new ArrayList<>();
            objectsToUploadAsMultipart.add(obj);
            MULTIPART_UTIL.uploadObjects(mBucketName, mClient, objectsToUploadAsMultipart, null);
        } else {
            // Avoid uploading file with Multipart if it's not necessary to save the
            // extra overhead.
            mClient.putObject(mBucketName, obj);
        }
        if (!mFile.delete()) {
            LOG.error("Failed to delete temporary file @ {}", mFile.getPath());
        }
    } catch (Exception e) {
        LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath());
        throw new IOException(e);
    }
}

Example 14 with S3Object

use of org.jets3t.service.model.S3Object in project alluxio by Alluxio.

the class S3UnderFileSystem method copyObject.

@Override
protected boolean copyObject(String src, String dst) {
    LOG.debug("Copying {} to {}", src, dst);
    S3Object obj = new S3Object(dst);
    // Retry copy for a few times, in case some Jets3t or AWS internal errors happened during copy.
    int retries = 3;
    for (int i = 0; i < retries; i++) {
        try {
            mClient.copyObject(mBucketName, src, mBucketName, obj, false);
            return true;
        } catch (ServiceException e) {
            LOG.error("Failed to copy file {} to {}", src, dst, e);
            if (i != retries - 1) {
                LOG.error("Retrying copying file {} to {}", src, dst);
            }
        }
    }
    LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries);
    return false;
}

Example 15 with S3Object

use of org.jets3t.service.model.S3Object in project alluxio by Alluxio.

the class S3UnderFileSystem method createEmptyObject.

@Override
protected boolean createEmptyObject(String key) {
    try {
        S3Object obj = new S3Object(key);
        obj.setDataInputStream(new ByteArrayInputStream(new byte[0]));
        obj.setContentLength(0);
        obj.setMd5Hash(DIR_HASH);
        obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);
        mClient.putObject(mBucketName, obj);
        return true;
    } catch (ServiceException e) {
        LOG.error("Failed to create object: {}", key, e);
        return false;
    }
}