Examples with Bucket com.talend.shaded.com.amazonaws.services.s3.model.Bucket used on opensource projects

Example 21 with Bucket

use of com.talend.shaded.com.amazonaws.services.s3.model.Bucket in project photon-model by vmware.

the class AWSS3StorageEnumerationAdapterService method createDiskStates.

/**
 * Creates the disk states that represent the buckets received from AWS during
 * enumeration. Fields currently being enumerated for S3 are all immutable on AWS side, hence we only create
 * disks and don't patch to them in subsequent except for changes in tagLinks.
 */
private void createDiskStates(S3StorageEnumerationContext aws, S3StorageEnumerationSubStage next) {
    // For all the disks to be created, we filter them based on whether we were able to find the correct
    // region for the disk using getBucketTaggingConfiguration() call and then map them and create operations.
    // Filtering is done to avoid creating disk states with null region (since we don't PATCH region field
    // after creating the disk, we need to ensure that disk state is initially created with the correct region).
    // kick off the operation using a JOIN
    List<DiskState> diskStatesToBeCreated = new ArrayList<>();
    aws.bucketsToBeCreated.stream().filter(bucket -> aws.regionsByBucketName.containsKey(bucket.getName())).forEach(bucket -> {
        diskStatesToBeCreated.add(mapBucketToDiskState(bucket, aws));
    });
    diskStatesToBeCreated.forEach(diskState -> aws.enumerationOperations.add(createPostOperation(this, diskState, DiskService.FACTORY_LINK)));
    this.logFine(() -> String.format("Creating %d S3 disks", aws.bucketsToBeCreated.size()));
    // For those disk states which do not have the tagLink, add the tagLink by PATCHing those states.
    if (aws.internalTypeTagSelfLink != null) {
        aws.diskStatesToBeUpdatedByBucketName.entrySet().stream().filter(diskMap -> diskMap.getValue().tagLinks == null || !diskMap.getValue().tagLinks.contains(aws.internalTypeTagSelfLink)).forEach(diskMap -> {
            Map<String, Collection<Object>> collectionsToAddMap = Collections.singletonMap(DiskState.FIELD_NAME_TAG_LINKS, Collections.singletonList(aws.internalTypeTagSelfLink));
            Map<String, Collection<Object>> collectionsToRemoveMap = Collections.singletonMap(DiskState.FIELD_NAME_TAG_LINKS, Collections.emptyList());
            ServiceStateCollectionUpdateRequest updateTagLinksRequest = ServiceStateCollectionUpdateRequest.create(collectionsToAddMap, collectionsToRemoveMap);
            aws.enumerationOperations.add(Operation.createPatch(this.getHost(), diskMap.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(updateTagLinksRequest));
        });
    }
    // update endpointLinks
    aws.diskStatesToBeUpdatedByBucketName.entrySet().stream().filter(diskMap -> diskMap.getValue().endpointLinks == null || !diskMap.getValue().endpointLinks.contains(aws.request.original.endpointLink)).forEach(diskMap -> {
        Map<String, Collection<Object>> collectionsToAddMap = Collections.singletonMap(DiskState.FIELD_NAME_ENDPOINT_LINKS, Collections.singletonList(aws.request.original.endpointLink));
        Map<String, Collection<Object>> collectionsToRemoveMap = Collections.singletonMap(DiskState.FIELD_NAME_ENDPOINT_LINKS, Collections.emptyList());
        ServiceStateCollectionUpdateRequest updateEndpointLinksRequest = ServiceStateCollectionUpdateRequest.create(collectionsToAddMap, collectionsToRemoveMap);
        aws.enumerationOperations.add(Operation.createPatch(this.getHost(), diskMap.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(updateEndpointLinksRequest));
    });
    OperationJoin.JoinedCompletionHandler joinCompletion = (ox, exc) -> {
        if (exc != null) {
            this.logSevere(() -> String.format("Error creating/updating disk %s", Utils.toString(exc)));
            aws.subStage = S3StorageEnumerationSubStage.DELETE_DISKS;
            handleReceivedEnumerationData(aws);
            return;
        }
        ox.entrySet().stream().forEach(operationEntry -> {
            aws.diskStatesEnumerated.add(operationEntry.getValue().getBody(DiskState.class));
        });
        this.logFine(() -> "Successfully created and updated all the disk states.");
        aws.subStage = next;
        handleReceivedEnumerationData(aws);
    };
    if (aws.enumerationOperations.isEmpty()) {
        aws.subStage = next;
        handleReceivedEnumerationData(aws);
        return;
    }
    OperationJoin joinOp = OperationJoin.create(aws.enumerationOperations);
    joinOp.setCompletion(joinCompletion);
    joinOp.sendWith(this.getHost());
}

Example 22 with Bucket

use of com.talend.shaded.com.amazonaws.services.s3.model.Bucket in project photon-model by vmware.

the class AWSRemoteCleanup method cleanUpAWSS3.

@Test
public void cleanUpAWSS3() {
    if (this.isMock) {
        return;
    }
    List<Bucket> buckets = this.s3Clients.get(Regions.DEFAULT_REGION.getName()).listBuckets();
    for (Bucket bucket : buckets) {
        long bucketCreationTimeMicros = TimeUnit.MILLISECONDS.toMicros(bucket.getCreationDate().getTime());
        long timeDifference = Utils.getNowMicrosUtc() - bucketCreationTimeMicros;
        if (bucket.getName().contains(ENUMTEST_BUCKET) && timeDifference > TimeUnit.HOURS.toMicros(1) && !bucket.getName().contains(ENUMTEST_BUCKET_TAG)) {
            for (AmazonS3Client s3Client : this.s3Clients.values()) {
                try {
                    s3Client.deleteBucket(bucket.getName());
                    this.host.log(Level.INFO, "Deleting stale bucket %s", bucket.getName());
                } catch (Exception e) {
                    continue;
                }
            }
        }
    }
}

Example 23 with Bucket

use of com.talend.shaded.com.amazonaws.services.s3.model.Bucket in project components by Talend.

the class S3DatasetRuntime method listBuckets.

@Override
public Set<String> listBuckets() {
    AmazonS3 conn = S3Connection.createClient(properties.getDatastoreProperties());
    String region = properties.region.getValue().getValue();
    if (S3Region.OTHER.getValue().equals(region)) {
        region = properties.unknownRegion.getValue();
    }
    conn.setEndpoint(regionToEndpoint(region));
    LOG.debug("Start to find buckets in region {}", region);
    List<Bucket> buckets = conn.listBuckets();
    Set<String> bucketsName = new HashSet<>();
    for (Bucket bucket : buckets) {
        String bucketName = bucket.getName();
        try {
            String bucketLocation = conn.getBucketLocation(bucketName);
            String bucketRegion = locationToRegion(bucketLocation);
            LOG.debug("Bucket is {} and location is {}({})", bucketName, bucketLocation, bucketRegion);
            if (region.equals(bucketRegion)) {
                bucketsName.add(bucketName);
            }
        } catch (Exception e) {
            // Ignore any exception when calling getBucketLocation, try next
            LOG.debug("Exception when check bucket location: {}", e.getMessage());
        }
    }
    return bucketsName;
}