Examples with PutObjectRequest com.amazonaws.s3.model.PutObjectRequest used on opensource projects

Example 96 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project stocator by CODAIT.

the class COSAPIClient method createObject.

@Override
public FSDataOutputStream createObject(String objName, String contentType, Map<String, String> metadata, Statistics statistics) throws IOException {
    LOG.debug("Create object {}", objName);
    try {
        String objNameWithoutBuket = objName;
        if (objName.startsWith(mBucket + "/")) {
            objNameWithoutBuket = objName.substring(mBucket.length() + 1);
        }
        if (blockUploadEnabled) {
            return new FSDataOutputStream(new COSBlockOutputStream(this, objNameWithoutBuket, new SemaphoredDelegatingExecutor(threadPoolExecutor, blockOutputActiveBlocks, true), partSize, blockFactory, contentType, new WriteOperationHelper(objNameWithoutBuket), metadata), null);
        }
        if (!contentType.equals(Constants.APPLICATION_DIRECTORY)) {
            return new FSDataOutputStream(new COSOutputStream(mBucket, objName, mClient, contentType, metadata, transfers, this), statistics);
        } else {
            final InputStream im = new InputStream() {

                @Override
                public int read() throws IOException {
                    return -1;
                }
            };
            final ObjectMetadata om = new ObjectMetadata();
            om.setContentLength(0L);
            om.setContentType(contentType);
            om.setUserMetadata(metadata);
            // Remove the bucket name prefix from key path
            if (objName.startsWith(mBucket + "/")) {
                objName = objName.substring(mBucket.length() + 1);
            }
            /*
        if (!objName.endsWith("/")) {
          objName = objName + "/";
        }*/
            LOG.debug("bucket: {}, key {}", mBucket, objName);
            PutObjectRequest putObjectRequest = new PutObjectRequest(mBucket, objName, im, om);
            Upload upload = transfers.upload(putObjectRequest);
            upload.waitForUploadResult();
            OutputStream fakeStream = new OutputStream() {

                @Override
                public void write(int b) throws IOException {
                }

                @Override
                public void close() throws IOException {
                    super.close();
                }
            };
            return new FSDataOutputStream(fakeStream, statistics);
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException("Interrupted creating " + objName);
    } catch (IOException e) {
        LOG.error(e.getMessage());
        throw e;
    }
}

Example 97 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project opentest by mcdcorp.

the class PutS3Object method run.

@Override
public void run() {
    super.run();
    String awsCredentialsProfile = this.readStringArgument("awsProfile", null);
    String accessKey = this.readStringArgument("accessKey", null);
    String secretKey = this.readStringArgument("secretKey", null);
    String bucket = this.readStringArgument("bucket");
    String objectKey = this.readStringArgument("objectKey");
    String sourceFilePath = this.readStringArgument("sourceFile");
    File sourceFile = new File(sourceFilePath);
    if (sourceFile.exists()) {
        try {
            AmazonS3 s3Client = null;
            if (awsCredentialsProfile != null) {
                s3Client = new AmazonS3Client(new ProfileCredentialsProvider(awsCredentialsProfile));
            } else if (accessKey != null && secretKey != null) {
                s3Client = new AmazonS3Client(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
            } else {
                s3Client = new AmazonS3Client();
            }
            s3Client.putObject(new PutObjectRequest(bucket, objectKey, sourceFile));
        } catch (Exception ex) {
            throw new RuntimeException(String.format("Failed to upload file \"%s\" to S3 bucket \"%s\"", sourceFilePath, bucket), ex);
        }
    } else {
        throw new RuntimeException(String.format("Source file \"%s\" doesn't exist", sourceFilePath));
    }
}

Example 98 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project uploader by smoketurner.

the class Uploader method upload.

/**
 * Upload a batch to S3
 *
 * @param batch Batch to upload
 */
public void upload(final Batch batch) {
    batchSize.update(batch.size());
    batchCount.update(batch.getCount());
    final ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder().put("count", String.valueOf(batch.getCount()));
    batch.getCustomerId().ifPresent(id -> builder.put("customer_id", id));
    final Map<String, String> metadata = builder.build();
    final String key;
    if (configuration.getPrefix().isPresent()) {
        key = configuration.getPrefix().get() + "/" + batch.getKey();
    } else {
        key = batch.getKey();
    }
    LOGGER.debug("Customer: {}, S3 key: {}", batch.getCustomerId().orElse(null), key);
    final PutObjectRequest request = PutObjectRequest.builder().bucket(configuration.getBucketName()).key(key).metadata(metadata).contentLength(batch.size()).contentType(MediaType.TEXT_PLAIN).contentEncoding("gzip").serverSideEncryption(ServerSideEncryption.AES256).build();
    final long start = currentTimeProvider.get();
    final CompletableFuture<PutObjectResponse> future = s3.putObject(request, new BatchRequestBody(batch));
    future.whenComplete((resp, err) -> {
        if (resp != null) {
            final long took = currentTimeProvider.get() - start;
            uploadTime.update(took, TimeUnit.NANOSECONDS);
            successCounter.inc();
            LOGGER.info("Finished uploading \"{}\" ({} events, {} bytes) in {}ms", key, batch.getCount(), batch.size(), (took / NANOS_IN_MILLIS));
        } else {
            failedCounter.inc();
            LOGGER.error(String.format("Failed to upload \"%s\"", key), err);
        }
    });
}

Example 99 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project sandbox by irof.

the class S3Test method putWithInputStream.

@Test
public void putWithInputStream() throws Exception {
    try (InputStream input = new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8))) {
        ObjectMetadata metaData = new ObjectMetadata();
        // 可能なら設定する。設定しなかったらデフォルト値が使われる。
        metaData.setContentType("text/plain");
        // 設定しなくても計算されるぽいのでスルーでよさげ
        // metaData.setContentMD5("CY9rzUYh03PK3k6DJie09g==");
        // オプションだけど、できるかぎり設定する。
        // 設定しないとWARNログ出ちゃうし、違う値を設定すると例外出る。
        // metaData.setContentLength(3);
        PutObjectRequest request = new PutObjectRequest("irof-sandbox", "byteFile.txt", input, metaData);
        PutObjectResult result = s3.putObject(request);
        assertThat(result.getETag(), is("098f6bcd4621d373cade4e832627b4f6"));
    }
}

Example 100 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project sandbox by irof.

the class S3MultipartUploadTest method 比較用の通常のputObject.

@Ignore
@Test
public void 比較用の通常のputObject() throws Exception {
    byte[] bytes = new byte[5 * Constants.MB + 1];
    ObjectMetadata meta = new ObjectMetadata();
    meta.setContentLength(bytes.length);
    PutObjectRequest putObjectRequest = new PutObjectRequest("irof-sandbox", "S3MultipartUpload/basic-5MB.dat", new ByteArrayInputStream(bytes), meta);
    PutObjectResult result = new AmazonS3Client().putObject(putObjectRequest);
    logger.info(result.getETag());
}