Examples with ObjectMetadata com.ibm.watson.visual_recognition.v4.model.ObjectMetadata used on opensource projects

Example 76 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project druid by druid-io.

the class S3ServerSideEncryption method decorate.

@Override
public CopyObjectRequest decorate(CopyObjectRequest request) {
    final ObjectMetadata objectMetadata = request.getNewObjectMetadata() == null ? new ObjectMetadata() : request.getNewObjectMetadata().clone();
    objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    return request.withNewObjectMetadata(objectMetadata);
}

Example 77 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project beam by apache.

the class S3FileSystemTest method matchVariousInvokeThreadPoolWithS3Options.

@Test
public void matchVariousInvokeThreadPoolWithS3Options() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
    notFoundException.setStatusCode(404);
    S3ResourceId pathNotExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathNotExist.getBucket(), pathNotExist.getKey()))))).thenThrow(notFoundException);
    AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
    forbiddenException.setStatusCode(403);
    S3ResourceId pathForbidden = S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathForbidden.getBucket(), pathForbidden.getKey()))))).thenThrow(forbiddenException);
    S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
    s3ObjectMetadata.setContentLength(100);
    s3ObjectMetadata.setLastModified(new Date(1540000000000L));
    s3ObjectMetadata.setContentEncoding("not-gzip");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey()))))).thenReturn(s3ObjectMetadata);
    S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");
    S3ObjectSummary foundListObject = new S3ObjectSummary();
    foundListObject.setBucketName(pathGlob.getBucket());
    foundListObject.setKey("path/part-0");
    foundListObject.setSize(200);
    foundListObject.setLastModified(new Date(1541000000000L));
    ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
    listObjectsResult.setNextContinuationToken(null);
    listObjectsResult.getObjectSummaries().add(foundListObject);
    when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class))).thenReturn(listObjectsResult);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentEncoding("");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0"))))).thenReturn(metadata);
    assertThat(s3FileSystem.match(ImmutableList.of(pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains(MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create(200, 1541000000000L, S3ResourceId.fromComponents("s3", pathGlob.getBucket(), foundListObject.getKey()), true)));
}

Example 78 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project beam by apache.

the class S3FileSystemTest method matchVariousInvokeThreadPool.

@Test
public void matchVariousInvokeThreadPool() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("s3"));
    AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
    notFoundException.setStatusCode(404);
    S3ResourceId pathNotExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathNotExist.getBucket(), pathNotExist.getKey()))))).thenThrow(notFoundException);
    AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
    forbiddenException.setStatusCode(403);
    S3ResourceId pathForbidden = S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathForbidden.getBucket(), pathForbidden.getKey()))))).thenThrow(forbiddenException);
    S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
    s3ObjectMetadata.setContentLength(100);
    s3ObjectMetadata.setLastModified(new Date(1540000000000L));
    s3ObjectMetadata.setContentEncoding("not-gzip");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey()))))).thenReturn(s3ObjectMetadata);
    S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");
    S3ObjectSummary foundListObject = new S3ObjectSummary();
    foundListObject.setBucketName(pathGlob.getBucket());
    foundListObject.setKey("path/part-0");
    foundListObject.setSize(200);
    foundListObject.setLastModified(new Date(1541000000000L));
    ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
    listObjectsResult.setNextContinuationToken(null);
    listObjectsResult.getObjectSummaries().add(foundListObject);
    when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class))).thenReturn(listObjectsResult);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentEncoding("");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(argThat(new GetObjectMetadataRequestMatcher(new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0"))))).thenReturn(metadata);
    assertThat(s3FileSystem.match(ImmutableList.of(pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains(MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create(200, 1541000000000L, S3ResourceId.fromComponents("s3", pathGlob.getBucket(), foundListObject.getKey()), true)));
}

Example 79 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project beam by apache.

the class S3FileSystemTest method matchGlob.

@Test
public void matchGlob() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar*baz");
    ListObjectsV2Request firstRequest = new ListObjectsV2Request().withBucketName(path.getBucket()).withPrefix(path.getKeyNonWildcardPrefix()).withContinuationToken(null);
    // Expected to be returned; prefix and wildcard/regex match
    S3ObjectSummary firstMatch = new S3ObjectSummary();
    firstMatch.setBucketName(path.getBucket());
    firstMatch.setKey("foo/bar0baz");
    firstMatch.setSize(100);
    firstMatch.setLastModified(new Date(1540000000001L));
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3ObjectSummary secondMatch = new S3ObjectSummary();
    secondMatch.setBucketName(path.getBucket());
    secondMatch.setKey("foo/bar1qux");
    secondMatch.setSize(200);
    secondMatch.setLastModified(new Date(1540000000002L));
    // Expected first request returns continuation token
    ListObjectsV2Result firstResult = new ListObjectsV2Result();
    firstResult.setNextContinuationToken("token");
    firstResult.getObjectSummaries().add(firstMatch);
    firstResult.getObjectSummaries().add(secondMatch);
    when(s3FileSystem.getAmazonS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest)))).thenReturn(firstResult);
    // Expect second request with continuation token
    ListObjectsV2Request secondRequest = new ListObjectsV2Request().withBucketName(path.getBucket()).withPrefix(path.getKeyNonWildcardPrefix()).withContinuationToken("token");
    // Expected to be returned; prefix and wildcard/regex match
    S3ObjectSummary thirdMatch = new S3ObjectSummary();
    thirdMatch.setBucketName(path.getBucket());
    thirdMatch.setKey("foo/bar2baz");
    thirdMatch.setSize(300);
    thirdMatch.setLastModified(new Date(1540000000003L));
    // Expected second request returns third prefix match and no continuation token
    ListObjectsV2Result secondResult = new ListObjectsV2Result();
    secondResult.setNextContinuationToken(null);
    secondResult.getObjectSummaries().add(thirdMatch);
    when(s3FileSystem.getAmazonS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest)))).thenReturn(secondResult);
    // Expect object metadata queries for content encoding
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentEncoding("");
    when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", firstMatch.getBucketName(), firstMatch.getKey())).setSizeBytes(firstMatch.getSize()).setLastModifiedMillis(firstMatch.getLastModified().getTime()).build(), MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", thirdMatch.getBucketName(), thirdMatch.getKey())).setSizeBytes(thirdMatch.getSize()).setLastModifiedMillis(thirdMatch.getLastModified().getTime()).build())));
}

Example 80 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project beam by apache.

the class S3FileSystemTest method testCopy.

private void testCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey) throws IOException {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(0);
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        objectMetadata.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    assertGetObjectMetadata(s3FileSystem, createObjectMetadataRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, objectMetadata);
    s3FileSystem.copy(sourcePath, destinationPath);
    verify(s3FileSystem.getAmazonS3Client(), times(1)).copyObject(any(CopyObjectRequest.class));
    // we simulate a big object >= 5GB so it takes the multiPart path
    objectMetadata.setContentLength(5_368_709_120L);
    assertGetObjectMetadata(s3FileSystem, createObjectMetadataRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, objectMetadata);
    try {
        s3FileSystem.copy(sourcePath, destinationPath);
    } catch (NullPointerException e) {
    // ignore failing unmocked path, this is covered by testMultipartCopy test
    }
    verify(s3FileSystem.getAmazonS3Client(), never()).copyObject(null);
}