Examples with Bucket com.amazonaws.services.s3.model.Bucket used on opensource projects

Example 86 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project elasticsearch by elastic.

the class AbstractS3SnapshotRestoreTest method testEncryption.

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testEncryption() {
    Client client = client();
    logger.info("-->  creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
    Settings repositorySettings = Settings.builder().put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath).put(S3Repository.Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000)).put(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true).build();
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("s3").setSettings(repositorySettings).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings bucket = settings.getByPrefix("repositories.s3.");
    AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
    String bucketName = bucket.get("bucket");
    logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
    List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
    }
    logger.info("--> delete some data");
    for (int i = 0; i < 50; i++) {
        client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
    }
    for (int i = 50; i < 100; i++) {
        client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
    }
    for (int i = 0; i < 100; i += 2) {
        client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    logger.info("--> close indices");
    client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
    logger.info("--> restore all indices from the snapshot");
    RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    // Test restore after index deletion
    logger.info("--> delete indices");
    cluster().wipeIndices("test-idx-1", "test-idx-2");
    logger.info("--> restore one index after deletion");
    restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
    assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
    assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
}

Example 87 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project elasticsearch by elastic.

the class TestAmazonS3 method uploadPart.

@Override
public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException {
    if (shouldFail(request.getBucketName(), request.getKey(), writeFailureRate)) {
        long length = request.getPartSize();
        long partToRead = (long) (length * randomDouble());
        byte[] buffer = new byte[1024];
        for (long cur = 0; cur < partToRead; cur += buffer.length) {
            try (InputStream input = request.getInputStream()) {
                input.read(buffer, 0, (int) (partToRead - cur > buffer.length ? buffer.length : partToRead - cur));
            } catch (IOException ex) {
                throw new ElasticsearchException("cannot read input stream", ex);
            }
        }
        logger.info("--> random write failure on uploadPart method: throwing an exception for [bucket={}, key={}]", request.getBucketName(), request.getKey());
        AmazonS3Exception ex = new AmazonS3Exception("Random S3 write exception");
        ex.setStatusCode(400);
        ex.setErrorCode("RequestTimeout");
        throw ex;
    } else {
        return super.uploadPart(request);
    }
}

Example 88 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project elasticsearch by elastic.

the class TestAmazonS3 method getObject.

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    if (shouldFail(bucketName, key, readFailureRate)) {
        logger.info("--> random read failure on getObject method: throwing an exception for [bucket={}, key={}]", bucketName, key);
        AmazonS3Exception ex = new AmazonS3Exception("Random S3 read exception");
        ex.setStatusCode(404);
        throw ex;
    } else {
        return super.getObject(bucketName, key);
    }
}

Example 89 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project h2o-2 by h2oai.

the class TypeaheadFileRequest method serveS3.

protected JsonArray serveS3(String filter, int limit) {
    JsonArray array = new JsonArray();
    try {
        AmazonS3 s3 = PersistS3.getClient();
        filter = Strings.nullToEmpty(filter);
        for (Bucket b : s3.listBuckets()) {
            if (b.getName().startsWith(filter))
                array.add(new JsonPrimitive(b.getName()));
            if (array.size() == limit)
                break;
        }
    } catch (IllegalArgumentException xe) {
    }
    return array;
}

Example 90 with Bucket

use of com.amazonaws.services.s3.model.Bucket in project h2o-2 by h2oai.

the class TypeaheadS3BucketRequest method serve.

@Override
protected JsonArray serve(String filter, int limit) {
    JsonArray array = new JsonArray();
    try {
        AmazonS3 s3 = PersistS3.getClient();
        filter = Strings.nullToEmpty(filter);
        for (Bucket b : s3.listBuckets()) {
            if (b.getName().startsWith(filter))
                array.add(new JsonPrimitive(b.getName()));
            if (array.size() == limit)
                break;
        }
    } catch (IllegalArgumentException xe) {
    }
    return array;
}