Examples with ObjectMetadata com.ibm.watson.visual_recognition.v4.model.ObjectMetadata used on opensource projects

Example 6 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project beneficiary-fhir-data by CMSgov.

the class DataSetMoveTask method call.

/**
 * @see java.util.concurrent.Callable#call()
 */
@Override
public Void call() throws Exception {
    LOGGER.debug("Renaming data set '{}' in S3, now that processing is complete...", manifest);
    /*
     * S3 doesn't support batch/transactional operations, or an atomic move
     * operation. Instead, we have to first copy all of the objects to their
     * new location, and then remove the old objects. If something blows up
     * in the middle of this method, orphaned S3 objects WILL be created.
     * That's an unlikely enough occurrence, though, that we're not going to
     * engineer around it right now.
     */
    // First, get a list of all the object keys to work on.
    List<String> s3KeySuffixesToMove = manifest.getEntries().stream().map(e -> String.format("%s/%s", manifest.getTimestampText(), e.getName())).collect(Collectors.toList());
    s3KeySuffixesToMove.add(String.format("%s/%d_manifest.xml", manifest.getTimestampText(), manifest.getSequenceId()));
    /*
     * Then, loop through each of those objects and copy them (S3 has no
     * bulk copy operation).
     */
    for (String s3KeySuffixToMove : s3KeySuffixesToMove) {
        String sourceKey = String.format("%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, s3KeySuffixToMove);
        String targetKey = String.format("%s/%s", CcwRifLoadJob.S3_PREFIX_COMPLETED_DATA_SETS, s3KeySuffixToMove);
        /*
       * Before copying, grab the metadata of the source object to ensure
       * that we maintain its encryption settings (by default, the copy
       * will maintain all metadata EXCEPT: server-side-encryption,
       * storage-class, and website-redirect-location).
       */
        ObjectMetadata objectMetadata = s3TaskManager.getS3Client().getObjectMetadata(options.getS3BucketName(), sourceKey);
        CopyObjectRequest copyRequest = new CopyObjectRequest(options.getS3BucketName(), sourceKey, options.getS3BucketName(), targetKey);
        if (objectMetadata.getSSEAwsKmsKeyId() != null) {
            copyRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(objectMetadata.getSSEAwsKmsKeyId()));
        }
        Copy copyOperation = s3TaskManager.getS3TransferManager().copy(copyRequest);
        try {
            copyOperation.waitForCopyResult();
            s3TaskManager.getS3Client().waiters().objectExists().run(new WaiterParameters<GetObjectMetadataRequest>(new GetObjectMetadataRequest(options.getS3BucketName(), targetKey)));
        } catch (InterruptedException e) {
            throw new BadCodeMonkeyException(e);
        }
    }
    LOGGER.debug("Data set copied in S3 (step 1 of move).");
    /*
     * After everything's been copied, loop over it all again and delete it the source objects. (We
     * could do it all in the same loop, but this is a bit easier to clean up from if it goes
     * sideways.)
     */
    for (String s3KeySuffixToMove : s3KeySuffixesToMove) {
        String sourceKey = String.format("%s/%s", CcwRifLoadJob.S3_PREFIX_PENDING_DATA_SETS, s3KeySuffixToMove);
        DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(options.getS3BucketName(), sourceKey);
        s3TaskManager.getS3Client().deleteObject(deleteObjectRequest);
        s3TaskManager.getS3Client().waiters().objectNotExists().run(new WaiterParameters<GetObjectMetadataRequest>(new GetObjectMetadataRequest(options.getS3BucketName(), sourceKey)));
    }
    LOGGER.debug("Data set deleted in S3 (step 2 of move).");
    LOGGER.debug("Renamed data set '{}' in S3, now that processing is complete.", manifest);
    return null;
}

Example 7 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project airavata-mft by apache.

the class S3IncomingConnector method downloadChunk.

@Override
public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception {
    GetObjectRequest rangeObjectRequest = new GetObjectRequest(resource.getS3Storage().getBucketName(), resource.getFile().getResourcePath());
    rangeObjectRequest.setRange(startByte, endByte - 1);
    ObjectMetadata objectMetadata = s3Client.getObject(rangeObjectRequest, new File(downloadFile));
    logger.info("Downloaded S3 chunk to path {} for resource id {}", downloadFile, resource.getResourceId());
}

Example 8 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project hetu-core by openlookeng.

the class PrestoS3FileSystem method getS3ObjectMetadata.

@VisibleForTesting
ObjectMetadata getS3ObjectMetadata(Path path) throws IOException {
    String bucketName = getBucketName(uri);
    String key = keyFromPath(path);
    ObjectMetadata s3ObjectMetadata = getS3ObjectMetadata(path, bucketName, key);
    if (s3ObjectMetadata == null && !key.isEmpty()) {
        return getS3ObjectMetadata(path, bucketName, key + PATH_SEPARATOR);
    }
    return s3ObjectMetadata;
}

Example 9 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project hetu-core by openlookeng.

the class TestPrestoS3FileSystem method testEmptyDirectory.

@Test
public void testEmptyDirectory() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3() {

            @Override
            public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) {
                if (getObjectMetadataRequest.getKey().equals("empty-dir/")) {
                    ObjectMetadata objectMetadata = new ObjectMetadata();
                    objectMetadata.setContentType(S3_DIRECTORY_OBJECT_CONTENT_TYPE);
                    return objectMetadata;
                }
                return super.getObjectMetadata(getObjectMetadataRequest);
            }
        };
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
        fs.setS3Client(s3);
        FileStatus fileStatus = fs.getFileStatus(new Path("s3n://test-bucket/empty-dir/"));
        assertTrue(fileStatus.isDirectory());
        fileStatus = fs.getFileStatus(new Path("s3n://test-bucket/empty-dir"));
        assertTrue(fileStatus.isDirectory());
    }
}

Example 10 with ObjectMetadata

use of com.ibm.watson.visual_recognition.v4.model.ObjectMetadata in project AlgaFoods by Dark1922.

the class S3FotoStorageService method armazenar.

@Override
public void armazenar(NovaFoto novaFoto) {
    try {
        String caminhoArquivo = getCaminhoArquivo(novaFoto.getNomeArquivo());
        // meta dados de um objeto
        var objectMetadata = new ObjectMetadata();
        // mostrar foto invez de fazer dowload
        objectMetadata.setContentType(novaFoto.getContentType());
        // submeter para api da amazon que agente ta fazendo uma requisição para colocar um objeto
        var putObjectRequest = new PutObjectRequest(storageProperties.getS3().getBucket(), caminhoArquivo, novaFoto.getInputStream(), objectMetadata).withCannedAcl(CannedAccessControlList.PublicRead);
        amazonS3.putObject(putObjectRequest);
    } catch (Exception e) {
        throw new StorageException("Não foi possivel enviar Arquivo para Amazon S3", e);
    }
}