Examples with GetObjectRequest software.amazon.awssdk.services.s3.model.GetObjectRequest used on opensource projects

Example 26 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project XRTB by benmfaul.

the class Configuration method processDirectory.

public void processDirectory(AmazonS3Client s3, ObjectListing listing, String bucket) throws Exception {
    for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
        long size = objectSummary.getSize();
        logger.info("*** Processing S3 {}, size: {}", objectSummary.getKey(), size);
        S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));
        String bucketName = object.getBucketName();
        String keyName = object.getKey();
        GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
        GetObjectTaggingResult result = s3.getObjectTagging(request);
        List<Tag> tags = result.getTagSet();
        String type = null;
        String name = null;
        if (tags.isEmpty()) {
            System.err.println("Error: " + keyName + " has no tags");
        } else {
            for (Tag tag : tags) {
                String key = tag.getKey();
                String value = tag.getValue();
                if (key.equals("type")) {
                    type = value;
                }
                if (key.equals("name")) {
                    name = value;
                }
            }
            if (name == null)
                throw new Exception("Error: " + keyName + " is missing a name tag");
            if (name.contains(" "))
                throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
            if (type == null)
                throw new Exception("Error: " + keyName + " has no type tag");
            if (!name.startsWith("$"))
                name = "$" + name;
            readData(type, name, object, size);
        }
    }
}

Example 27 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project airpal by airbnb.

the class ResultsPreviewResource method getS3Preview.

private Response getS3Preview(URI fileURI, int numLines) {
    val filename = getFilename(fileURI);
    val outputKey = getOutputKey(filename);
    // download ~100 kb (depending on your definition) of the file
    val request = new GetObjectRequest(outputBucket, outputKey).withRange(0, 100 * 1024);
    val object = s3Client.getObject(request);
    ObjectMetadata objectMetadata = object.getObjectMetadata();
    boolean gzip = "gzip".equalsIgnoreCase(objectMetadata.getContentEncoding());
    try (InputStream input = object.getObjectContent()) {
        InputStreamReader reader;
        if (gzip) {
            reader = new InputStreamReader(new GZIPInputStream(input));
        } else {
            reader = new InputStreamReader(input);
        }
        return getPreviewFromCSV(new CSVReader(reader), numLines);
    } catch (IOException e) {
        return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
    }
}

Example 28 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project pravega by pravega.

the class S3ChunkStorage method doRead.

@Override
protected int doRead(ChunkHandle handle, long fromOffset, int length, byte[] buffer, int bufferOffset) throws ChunkStorageException {
    try {
        GetObjectRequest objectRequest = GetObjectRequest.builder().key(getObjectPath(handle.getChunkName())).range(getRangeWithLength(fromOffset, length)).bucket(config.getBucket()).build();
        ResponseBytes<GetObjectResponse> objectBytes = client.getObjectAsBytes(objectRequest);
        try (val inputStream = objectBytes.asInputStream()) {
            return StreamHelpers.readAll(inputStream, buffer, bufferOffset, length);
        }
    } catch (Exception e) {
        throw convertException(handle.getChunkName(), "doRead", e);
    }
}

Example 29 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project verify-hub by alphagov.

the class S3ConfigSourceTest method getRemoteConfigOnlyRetrievesNewContentWhenLastModifiedChanges.

@Test
public void getRemoteConfigOnlyRetrievesNewContentWhenLastModifiedChanges() throws Exception {
    SelfServiceConfig selfServiceConfig = objectMapper.readValue(selfServiceConfigShortCacheJson, SelfServiceConfig.class);
    when(s3Client.getObject(new GetObjectRequest(BUCKET_NAME, OBJECT_KEY))).thenReturn(s3Object);
    when(s3Object.getObjectContent()).thenReturn(getObjectStream("/remote-test-config.json"));
    when(s3Object.getObjectMetadata()).thenReturn(objectMetadata);
    when(objectMetadata.getLastModified()).thenReturn(Date.from(Instant.now().minusMillis(10000)));
    S3ConfigSource testSource = new S3ConfigSource(selfServiceConfig, s3Client, objectMapper);
    var testCacheLoader = testSource.getCacheLoader();
    RemoteConfigCollection result1 = testSource.getRemoteConfig();
    ListenableFuture<RemoteConfigCollection> task = testCacheLoader.reload("test", result1);
    while (!task.isDone()) {
        Thread.yield();
    }
    RemoteConfigCollection result2 = task.get();
    assertThat((result1 == result2)).isTrue();
    verify(s3Object, times(1)).getObjectContent();
    when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(s3Object2);
    when(s3Object2.getObjectMetadata()).thenReturn(objectMetadata2);
    when(s3Object2.getObjectContent()).thenReturn(getObjectStream("/remote-test-config.json"));
    when(objectMetadata2.getLastModified()).thenReturn(Date.from(Instant.now()));
    ListenableFuture<RemoteConfigCollection> task2 = testCacheLoader.reload("test", result1);
    while (!task2.isDone()) {
        Thread.yield();
    }
    verify(s3Object2, times(1)).getObjectContent();
    verify(objectMetadata2, times(1)).getLastModified();
}

Example 30 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project verify-hub by alphagov.

the class S3ConfigSourceTest method getRemoteConfigReturnsRemoteConfigCollection.

@Test
public /**
 * Tests to make sure we can process the JSON to an object
 */
void getRemoteConfigReturnsRemoteConfigCollection() throws Exception {
    SelfServiceConfig selfServiceConfig = objectMapper.readValue(selfServiceConfigEnabledJson, SelfServiceConfig.class);
    when(s3Client.getObject(new GetObjectRequest(BUCKET_NAME, OBJECT_KEY))).thenReturn(s3Object);
    when(s3Object.getObjectContent()).thenReturn(getObjectStream("/remote-test-config.json"));
    when(s3Object.getObjectMetadata()).thenReturn(objectMetadata);
    when(objectMetadata.getLastModified()).thenReturn(new Date());
    S3ConfigSource testSource = new S3ConfigSource(selfServiceConfig, s3Client, objectMapper);
    RemoteConfigCollection result = testSource.getRemoteConfig();
    Map<String, RemoteMatchingServiceConfig> msConfigs = result.getMatchingServiceAdapters();
    assertThat(msConfigs.size()).isEqualTo(3);
    assertThat(msConfigs.get("https://msa.bananaregistry.test.com").getName()).isEqualTo("Banana Registry MSA");
    assertThat(msConfigs.get("https://msa.bananaregistry.test.com").getEncryptionCertificate()).contains(CERT_MSA_BANANA_ENCRYPTION);
    assertThat(msConfigs.get("https://msa.bananaregistry.test.com").getSignatureVerificationCertificates().size()).isEqualTo(1);
    assertThat(msConfigs.get("https://msa.bananaregistry.test.com").getSignatureVerificationCertificates().get(0)).contains(CERT_MSA_BANANA_SIGNING);
    Map<String, RemoteServiceProviderConfig> spConfigs = result.getServiceProviders();
    assertThat(spConfigs.size()).isEqualTo(3);
    RemoteServiceProviderConfig spConfig2 = spConfigs.get("2");
    assertThat(spConfig2.getName()).isEqualTo("Apple Registry VSP");
    assertThat(spConfig2.getSignatureVerificationCertificates().size()).isEqualTo(1);
    assertThat(spConfig2.getSignatureVerificationCertificates().get(0)).contains(CERT_VSP_APPLE_SIGNING);
    RemoteServiceProviderConfig spConfig3 = spConfigs.get("3");
    assertThat(spConfig3.getName()).isEqualTo("Banana Registry VSP");
    assertThat(spConfig3.getSignatureVerificationCertificates().size()).isEqualTo(1);
    assertThat(spConfig3.getSignatureVerificationCertificates().get(0)).contains(CERT_VSP_BANANA_SIGNING);
}