Examples with GetObjectRequest software.amazon.awssdk.services.s3.model.GetObjectRequest used on opensource projects

Example 6 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project dataverse by IQSS.

the class S3AccessIO method open.

@Override
public void open(DataAccessOption... options) throws IOException {
    if (s3 == null) {
        throw new IOException("ERROR: s3 not initialised. ");
    }
    if (bucketName == null || !s3.doesBucketExist(bucketName)) {
        throw new IOException("ERROR: S3AccessIO - You must create and configure a bucket before creating datasets.");
    }
    DataAccessRequest req = this.getRequest();
    if (isWriteAccessRequested(options)) {
        isWriteAccess = true;
        isReadAccess = false;
    } else {
        isWriteAccess = false;
        isReadAccess = true;
    }
    if (dvObject instanceof DataFile) {
        String storageIdentifier = dvObject.getStorageIdentifier();
        DataFile dataFile = this.getDataFile();
        if (req != null && req.getParameter("noVarHeader") != null) {
            this.setNoVarHeader(true);
        }
        if (storageIdentifier == null || "".equals(storageIdentifier)) {
            throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile.");
        }
        if (isReadAccess) {
            key = getMainFileKey();
            S3Object s3object = s3.getObject(new GetObjectRequest(bucketName, key));
            InputStream in = s3object.getObjectContent();
            if (in == null) {
                throw new IOException("Cannot get Object" + key);
            }
            this.setInputStream(in);
            setChannel(Channels.newChannel(in));
            this.setSize(s3object.getObjectMetadata().getContentLength());
            if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
                List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
                String varHeaderLine = generateVariableHeader(datavariables);
                this.setVarHeader(varHeaderLine);
            }
        } else if (isWriteAccess) {
            key = dataFile.getOwner().getAuthority() + "/" + this.getDataFile().getOwner().getIdentifier();
            if (storageIdentifier.startsWith(S3_IDENTIFIER_PREFIX + "://")) {
                key += "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1);
            } else {
                key += "/" + storageIdentifier;
                dvObject.setStorageIdentifier(S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + storageIdentifier);
            }
        }
        this.setMimeType(dataFile.getContentType());
        try {
            this.setFileName(dataFile.getFileMetadata().getLabel());
        } catch (Exception ex) {
            this.setFileName("unknown");
        }
    } else if (dvObject instanceof Dataset) {
        Dataset dataset = this.getDataset();
        key = dataset.getAuthority() + "/" + dataset.getIdentifier();
        dataset.setStorageIdentifier(S3_IDENTIFIER_PREFIX + "://" + key);
    } else if (dvObject instanceof Dataverse) {
        throw new IOException("Data Access: Invalid DvObject type : Dataverse");
    } else {
        throw new IOException("Data Access: Invalid DvObject type");
    }
}

Example 7 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project opentest by mcdcorp.

the class GetS3Object method run.

@Override
public void run() {
    super.run();
    String awsCredentialsProfile = this.readStringArgument("awsProfile", "default");
    String bucket = this.readStringArgument("bucket");
    String objectKey = this.readStringArgument("objectKey");
    String targetFilePath = this.readStringArgument("targetFile");
    Boolean overwrite = this.readBooleanArgument("overwrite", false);
    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider(awsCredentialsProfile));
    S3Object object = s3Client.getObject(new GetObjectRequest(bucket, objectKey));
    InputStream objectDataStream = object.getObjectContent();
    if (targetFilePath != null) {
        File targetFile = new File(targetFilePath);
        if (!targetFile.isAbsolute()) {
            targetFile = Paths.get(this.getActor().getTempDir().getAbsolutePath(), targetFilePath).toFile();
        }
        targetFile.getParentFile().mkdirs();
        try {
            if (overwrite) {
                Files.copy(objectDataStream, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
            } else {
                Files.copy(objectDataStream, targetFile.toPath());
            }
        } catch (Exception ex) {
            throw new RuntimeException(String.format("Failed to transfer data from the input stream into file %s", targetFilePath), ex);
        }
        this.writeArgument("targetFile", targetFile.getAbsolutePath());
    } else {
    // TODO: Make targetFile arg optional so this branch can execute.
    // Read data in memory and write it to an output value
    }
}

Example 8 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project molgenis by molgenis.

the class AmazonBucketClientImpl method downloadFile.

@Override
public File downloadFile(AmazonS3 s3Client, FileStore fileStore, String jobIdentifier, String bucketName, String keyName, String extension, boolean isExpression, String targetEntityType) throws IOException {
    String key;
    // This is indicated by the "isExpression" boolean
    if (isExpression) {
        key = this.getMostRecentMatchingKey(s3Client, bucketName, keyName);
    } else {
        key = keyName;
    }
    S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key));
    InputStream in = s3Object.getObjectContent();
    return storeFile(fileStore, key, extension, targetEntityType, jobIdentifier, in);
}

Example 9 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project qpp-conversion-tool by CMSgov.

the class StorageServiceImpl method getFileByLocationId.

/**
 * Performs a {@link GetObjectRequest} to the S3 bucket by file id for the file
 *
 * @param fileLocationId Id of the file to search for
 * @return file found from S3
 */
@Override
public InputStream getFileByLocationId(String fileLocationId) {
    final String bucketName = environment.getProperty(Constants.BUCKET_NAME_ENV_VARIABLE);
    if (Strings.isNullOrEmpty(bucketName)) {
        API_LOG.warn("No bucket name is specified.");
        return null;
    }
    API_LOG.info("Retrieving file {} from bucket {}", fileLocationId, bucketName);
    GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileLocationId);
    S3Object s3Object = amazonS3.getObject(getObjectRequest);
    API_LOG.info("Successfully retrieved file {} from S3 bucket {}", getObjectRequest.getKey(), getObjectRequest.getBucketName());
    return s3Object.getObjectContent();
}

Example 10 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project presto by prestodb.

the class TestPrestoS3FileSystem method testPrestoS3InputStreamEOS.

@Test
public void testPrestoS3InputStreamEOS() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        AtomicInteger readableBytes = new AtomicInteger(1);
        MockAmazonS3 s3 = new MockAmazonS3() {

            @Override
            public S3Object getObject(GetObjectRequest req) {
                return new S3Object() {

                    @Override
                    public S3ObjectInputStream getObjectContent() {
                        return new S3ObjectInputStream(new ByteArrayInputStream(new byte[readableBytes.get()]), null);
                    }
                };
            }
        };
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            assertEquals(inputStream.read(0, new byte[2], 0, 2), 1);
            readableBytes.set(0);
            assertEquals(inputStream.read(0, new byte[1], 0, 1), -1);
        }
    }
}