Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 31 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project ignite by apache.

the class S3CheckpointSpiSelfTest method afterSpiStopped.

/**
 * @throws Exception If error.
 */
@Override
protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(), IgniteS3TestSuite.getSecretKey());
    AmazonS3 s3 = new AmazonS3Client(cred);
    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";
    try {
        ObjectListing list = s3.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) s3.deleteObject(bucketName, sum.getKey());
            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}

Example 32 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project elasticsearch by elastic.

the class AbstractS3SnapshotRestoreTest method cleanRepositoryFiles.

/**
     * Deletes content of the repository files in the bucket
     */
public void cleanRepositoryFiles(String basePath) {
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings[] buckets = { settings.getByPrefix("repositories.s3."), settings.getByPrefix("repositories.s3.private-bucket."), settings.getByPrefix("repositories.s3.remote-bucket."), settings.getByPrefix("repositories.s3.external-bucket.") };
    for (Settings bucket : buckets) {
        String bucketName = bucket.get("bucket");
        // We check that settings has been set in elasticsearch.yml integration test file
        // as described in README
        assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
        AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, null, randomBoolean(), null);
        try {
            ObjectListing prevListing = null;
            //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
            //we can do at most 1K objects per delete
            //We don't know the bucket name until first object listing
            DeleteObjectsRequest multiObjectDeleteRequest = null;
            ArrayList<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    list = client.listObjects(bucketName, basePath);
                    multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                    //Every 500 objects batch the delete request
                    if (keys.size() > 500) {
                        multiObjectDeleteRequest.setKeys(keys);
                        client.deleteObjects(multiObjectDeleteRequest);
                        multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                        keys.clear();
                    }
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
            if (!keys.isEmpty()) {
                multiObjectDeleteRequest.setKeys(keys);
                client.deleteObjects(multiObjectDeleteRequest);
            }
        } catch (Exception ex) {
            logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex);
        }
    }
}

Example 33 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project elasticsearch by elastic.

the class AbstractS3SnapshotRestoreTest method testEncryption.

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testEncryption() {
    Client client = client();
    logger.info("-->  creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
    Settings repositorySettings = Settings.builder().put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath).put(S3Repository.Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000)).put(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true).build();
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("s3").setSettings(repositorySettings).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings bucket = settings.getByPrefix("repositories.s3.");
    AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
    String bucketName = bucket.get("bucket");
    logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
    List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
    }
    logger.info("--> delete some data");
    for (int i = 0; i < 50; i++) {
        client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
    }
    for (int i = 50; i < 100; i++) {
        client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
    }
    for (int i = 0; i < 100; i += 2) {
        client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    logger.info("--> close indices");
    client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
    logger.info("--> restore all indices from the snapshot");
    RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    // Test restore after index deletion
    logger.info("--> delete indices");
    cluster().wipeIndices("test-idx-1", "test-idx-2");
    logger.info("--> restore one index after deletion");
    restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
    assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
    assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
}

Example 34 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project elasticsearch by elastic.

the class MockAmazonS3 method listObjects.

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);
    String blobName;
    String prefix = listObjectsRequest.getPrefix();
    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();
    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();
        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);
            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " + "failed for blob [" + blob.getKey() + "]");
            }
            mockObjectSummaries.add(objectSummary);
        }
    }
    list.setObjectSummaries(mockObjectSummaries);
    return list;
}

Example 35 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project zeppelin by apache.

the class S3NotebookRepo method remove.

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
    String key = user + "/" + "notebook" + "/" + noteId;
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
    try {
        ObjectListing objects = s3client.listObjects(listObjectsRequest);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                s3client.deleteObject(bucketName, objectSummary.getKey());
            }
            objects = s3client.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
}