Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 41 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project YCSB by brianfrankcooper.

the class S3Client method writeToStorage.

/**
  * Upload a new object to S3 or update an object on S3.
  *
  * @param bucket
  *            The name of the bucket
  * @param key
  *            The file key of the object to upload/update.
  * @param values
  *            The data to be written on the object
  * @param updateMarker
  *            A boolean value. If true a new object will be uploaded
  *            to S3. If false an existing object will be re-uploaded
  *
  */
protected Status writeToStorage(String bucket, String key, HashMap<String, ByteIterator> values, Boolean updateMarker, String sseLocal, SSECustomerKey ssecLocal) {
    int totalSize = 0;
    //number of fields to concatenate
    int fieldCount = values.size();
    // getting the first field in the values
    Object keyToSearch = values.keySet().toArray()[0];
    // getting the content of just one field
    byte[] sourceArray = values.get(keyToSearch).toArray();
    //size of each array
    int sizeArray = sourceArray.length;
    if (updateMarker) {
        totalSize = sizeArray * fieldCount;
    } else {
        try {
            Map.Entry<S3Object, ObjectMetadata> objectAndMetadata = getS3ObjectAndMetadata(bucket, key, ssecLocal);
            int sizeOfFile = (int) objectAndMetadata.getValue().getContentLength();
            fieldCount = sizeOfFile / sizeArray;
            totalSize = sizeOfFile;
            objectAndMetadata.getKey().close();
        } catch (Exception e) {
            System.err.println("Not possible to get the object :" + key);
            e.printStackTrace();
            return Status.ERROR;
        }
    }
    byte[] destinationArray = new byte[totalSize];
    int offset = 0;
    for (int i = 0; i < fieldCount; i++) {
        System.arraycopy(sourceArray, 0, destinationArray, offset, sizeArray);
        offset += sizeArray;
    }
    try (InputStream input = new ByteArrayInputStream(destinationArray)) {
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(totalSize);
        PutObjectRequest putObjectRequest = null;
        if (sseLocal.equals("true")) {
            metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
            putObjectRequest = new PutObjectRequest(bucket, key, input, metadata);
        } else if (ssecLocal != null) {
            putObjectRequest = new PutObjectRequest(bucket, key, input, metadata).withSSECustomerKey(ssecLocal);
        } else {
            putObjectRequest = new PutObjectRequest(bucket, key, input, metadata);
        }
        try {
            PutObjectResult res = s3Client.putObject(putObjectRequest);
            if (res.getETag() == null) {
                return Status.ERROR;
            } else {
                if (sseLocal.equals("true")) {
                    System.out.println("Uploaded object encryption status is " + res.getSSEAlgorithm());
                } else if (ssecLocal != null) {
                    System.out.println("Uploaded object encryption status is " + res.getSSEAlgorithm());
                }
            }
        } catch (Exception e) {
            System.err.println("Not possible to write object :" + key);
            e.printStackTrace();
            return Status.ERROR;
        }
    } catch (Exception e) {
        System.err.println("Error in the creation of the stream :" + e.toString());
        e.printStackTrace();
        return Status.ERROR;
    }
    return Status.OK;
}

Example 42 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project gradle by gradle.

the class S3ResourceConnector method getMetaData.

public ExternalResourceMetaData getMetaData(URI location, boolean revalidate) {
    LOGGER.debug("Attempting to get resource metadata: {}", location);
    S3Object s3Object = s3Client.getMetaData(location);
    if (s3Object == null) {
        return null;
    }
    try {
        ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
        return new DefaultExternalResourceMetaData(location, objectMetadata.getLastModified().getTime(), objectMetadata.getContentLength(), objectMetadata.getContentType(), objectMetadata.getETag(), // Passing null for sha1 - TODO - consider using the etag which is an MD5 hash of the file (when less than 5Gb)
        null);
    } finally {
        IoActions.closeQuietly(s3Object);
    }
}

Example 43 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project archaius by Netflix.

the class S3ConfigurationSource method poll.

@Override
public PollResult poll(boolean initial, Object checkPoint) throws IOException, AmazonServiceException {
    GetObjectRequest s3request = new GetObjectRequest(bucketName, key);
    InputStream is = null;
    try {
        S3Object result = client.getObject(s3request);
        is = result.getObjectContent();
        Map<String, Object> resultMap = inputStreamToMap(is);
        return PollResult.createFull(resultMap);
    } finally {
        if (is != null)
            is.close();
    }
}

Example 44 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project exhibitor by soabase.

the class S3ConfigProvider method loadConfig.

@Override
public LoadedInstanceConfig loadConfig() throws Exception {
    Date lastModified;
    Properties properties = new Properties();
    S3Object object = getConfigObject();
    if (object != null) {
        try {
            lastModified = object.getObjectMetadata().getLastModified();
            properties.load(object.getObjectContent());
        } finally {
            CloseableUtils.closeQuietly(object.getObjectContent());
        }
    } else {
        lastModified = new Date(0L);
    }
    PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(properties, defaults);
    return new LoadedInstanceConfig(config, lastModified.getTime());
}

Example 45 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project camel by apache.

the class S3Consumer method poll.

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }
        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }
        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}