Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 11 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project h2o-2 by h2oai.

the class ImportS3 method processListing.

public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail) {
    for (S3ObjectSummary obj : listing.getObjectSummaries()) {
        try {
            Key k = PersistS3.loadKey(obj);
            JsonObject o = new JsonObject();
            o.addProperty(KEY, k.toString());
            o.addProperty(FILE, obj.getKey());
            o.addProperty(VALUE_SIZE, obj.getSize());
            succ.add(o);
        } catch (IOException e) {
            JsonObject o = new JsonObject();
            o.addProperty(FILE, obj.getKey());
            o.addProperty(ERROR, e.getMessage());
            fail.add(o);
        }
    }
}

Example 12 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project cloudstack by apache.

the class NfsSecondaryStorageResource method s3ListVolume.

Map<Long, TemplateProp> s3ListVolume(S3TO s3) {
    String bucket = s3.getBucketName();
    // List the objects in the source directory on S3
    final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR);
    if (objectSummaries == null) {
        return null;
    }
    Map<Long, TemplateProp> tmpltInfos = new HashMap<Long, TemplateProp>();
    for (S3ObjectSummary objectSummary : objectSummaries) {
        String key = objectSummary.getKey();
        // String installPath = StringUtils.substringBeforeLast(key,
        // S3Utils.SEPARATOR);
        Long id = determineS3VolumeIdFromKey(key);
        // TODO: how to get volume template name
        TemplateProp tInfo = new TemplateProp(id.toString(), key, objectSummary.getSize(), objectSummary.getSize(), true, false);
        tmpltInfos.put(id, tInfo);
    }
    return tmpltInfos;
}

Example 13 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project cloudstack by apache.

the class S3Utils method deleteDirectory.

public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName, final String directoryName) {
    LOGGER.debug(format("Deleting S3 Directory %1$s in bucket %2$s", directoryName, bucketName));
    final List<S3ObjectSummary> objects = listDirectory(clientOptions, bucketName, directoryName);
    for (final S3ObjectSummary object : objects) {
        deleteObject(clientOptions, bucketName, object.getKey());
    }
    deleteObject(clientOptions, bucketName, directoryName);
}

Example 14 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project cloudstack by apache.

the class S3Utils method listDirectory.

public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
    LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));
    List<S3ObjectSummary> objects = new ArrayList<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.withBucketName(bucketName);
    listObjectsRequest.withPrefix(directory);
    ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
    if (ol.isTruncated()) {
        do {
            objects.addAll(ol.getObjectSummaries());
            listObjectsRequest.setMarker(ol.getNextMarker());
            ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
        } while (ol.isTruncated());
    } else {
        objects.addAll(ol.getObjectSummaries());
    }
    if (objects.isEmpty()) {
        return emptyList();
    }
    return unmodifiableList(objects);
}

Example 15 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project ignite by apache.

the class S3CheckpointSpi method spiStart.

/** {@inheritDoc} */
@SuppressWarnings({ "BusyWait" })
@Override
public void spiStart(String igniteInstanceName) throws IgniteSpiException {
    // Start SPI start stopwatch.
    startStopwatch();
    assertParameter(cred != null, "awsCredentials != null");
    if (log.isDebugEnabled()) {
        log.debug(configInfo("awsCredentials", cred));
        log.debug(configInfo("clientConfiguration", cfg));
        log.debug(configInfo("bucketNameSuffix", bucketNameSuffix));
    }
    if (cfg == null)
        U.warn(log, "Amazon client configuration is not set (will use default).");
    if (F.isEmpty(bucketNameSuffix)) {
        U.warn(log, "Bucket name suffix is null or empty (will use default bucket name).");
        bucketName = BUCKET_NAME_PREFIX + DFLT_BUCKET_NAME_SUFFIX;
    } else
        bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix;
    s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred);
    if (!s3.doesBucketExist(bucketName)) {
        try {
            s3.createBucket(bucketName);
            if (log.isDebugEnabled())
                log.debug("Created S3 bucket: " + bucketName);
            while (!s3.doesBucketExist(bucketName)) try {
                U.sleep(200);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteSpiException("Thread has been interrupted.", e);
            }
        } catch (AmazonClientException e) {
            try {
                if (!s3.doesBucketExist(bucketName))
                    throw new IgniteSpiException("Failed to create bucket: " + bucketName, e);
            } catch (AmazonClientException ignored) {
                throw new IgniteSpiException("Failed to create bucket: " + bucketName, e);
            }
        }
    }
    Collection<S3TimeData> s3TimeDataLst = new LinkedList<>();
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) {
                S3CheckpointData data = read(sum.getKey());
                if (data != null) {
                    s3TimeDataLst.add(new S3TimeData(data.getExpireTime(), data.getKey()));
                    if (log.isDebugEnabled())
                        log.debug("Registered existing checkpoint from key: " + data.getKey());
                }
            }
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    } catch (IgniteCheckedException e) {
        throw new IgniteSpiException("Failed to marshal/unmarshal objects in bucket: " + bucketName, e);
    }
    // Track expiration for only those data that are made by this node
    timeoutWrk = new S3TimeoutWorker();
    timeoutWrk.add(s3TimeDataLst);
    timeoutWrk.start();
    registerMBean(igniteInstanceName, new S3CheckpointSpiMBeanImpl(this), S3CheckpointSpiMBean.class);
    // Ack ok start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}