Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 76 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project ignite by apache.

the class S3CheckpointSpi method read.

/**
 * Reads checkpoint data.
 *
 * @param key Key name to read data from.
 * @return Checkpoint data object.
 * @throws IgniteCheckedException Thrown if an error occurs while unmarshalling.
 * @throws AmazonClientException If an error occurs while querying Amazon S3.
 */
@Nullable
private S3CheckpointData read(String key) throws IgniteCheckedException, AmazonClientException {
    assert !F.isEmpty(key);
    if (log.isDebugEnabled())
        log.debug("Reading data from S3 [bucket=" + bucketName + ", key=" + key + ']');
    try {
        S3Object obj = s3.getObject(bucketName, key);
        InputStream in = obj.getObjectContent();
        try {
            return S3CheckpointData.fromStream(in);
        } catch (IOException e) {
            throw new IgniteCheckedException("Failed to unmarshal S3CheckpointData [bucketName=" + bucketName + ", key=" + key + ']', e);
        } finally {
            U.closeQuiet(in);
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404)
            throw e;
    }
    return null;
}

Example 77 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project nifi by apache.

the class TestFetchS3Object method testGetObjectVersion.

@Test
public void testGetObjectVersion() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    attrs.put("s3.version", "request-version");
    runner.enqueue(new byte[0], attrs);
    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    Mockito.when(metadata.getVersionId()).thenReturn("response-version");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
    runner.run(1);
    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertEquals("request-version", request.getVersionId());
    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals("s3.version", "response-version");
    ff.assertContentEquals("Some Content");
}

Example 78 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project nifi by apache.

the class TestFetchS3Object method testGetObject.

@Test
public void testGetObject() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);
    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
    runner.run(1);
    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertNull(request.getVersionId());
    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}

Example 79 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetSignedDomainException.

@Test
public void testGetSignedDomainException() throws IOException {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    InputStream is = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null);
    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is);
    // first call we return null, second call we return success
    when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenThrow(new AmazonServiceException("test")).thenReturn(object);
    SignedDomain signedDomain = store.getSignedDomain("iaas");
    assertNotNull(signedDomain);
    DomainData domainData = signedDomain.getDomain();
    assertNotNull(domainData);
    assertEquals(domainData.getName(), "iaas");
    is.close();
}

Example 80 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetSignedDomain.

@Test
public void testGetSignedDomain() throws IOException {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    InputStream is = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null);
    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is);
    when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
    SignedDomain signedDomain = store.getSignedDomain("iaas");
    assertNotNull(signedDomain);
    DomainData domainData = signedDomain.getDomain();
    assertNotNull(domainData);
    assertEquals(domainData.getName(), "iaas");
    is.close();
}