Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 21 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project onebusaway-application-modules by camsys.

the class S3FileServiceImpl method get.

@Override
public /**
 * Retrieve the specified key from S3 and store in the given directory.
 */
String get(String key, String tmpDir) {
    _log.debug("get(" + key + ", " + tmpDir + ")");
    NYCFileUtils fs = new NYCFileUtils();
    String filename = fs.parseFileName(key);
    _log.debug("filename=" + filename);
    GetObjectRequest request = new GetObjectRequest(this._bucketName, key);
    S3Object file = _s3.getObject(request);
    String pathAndFileName = tmpDir + File.separator + filename;
    fs.copy(file.getObjectContent(), pathAndFileName);
    return pathAndFileName;
}

Example 22 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project dataverse by IQSS.

the class S3AccessIO method open.

@Override
public void open(DataAccessOption... options) throws IOException {
    if (s3 == null) {
        throw new IOException("ERROR: s3 not initialised. ");
    }
    if (bucketName == null || !s3.doesBucketExist(bucketName)) {
        throw new IOException("ERROR: S3AccessIO - You must create and configure a bucket before creating datasets.");
    }
    DataAccessRequest req = this.getRequest();
    if (isWriteAccessRequested(options)) {
        isWriteAccess = true;
        isReadAccess = false;
    } else {
        isWriteAccess = false;
        isReadAccess = true;
    }
    if (dvObject instanceof DataFile) {
        String storageIdentifier = dvObject.getStorageIdentifier();
        DataFile dataFile = this.getDataFile();
        if (req != null && req.getParameter("noVarHeader") != null) {
            this.setNoVarHeader(true);
        }
        if (storageIdentifier == null || "".equals(storageIdentifier)) {
            throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile.");
        }
        if (isReadAccess) {
            key = getMainFileKey();
            S3Object s3object = s3.getObject(new GetObjectRequest(bucketName, key));
            InputStream in = s3object.getObjectContent();
            if (in == null) {
                throw new IOException("Cannot get Object" + key);
            }
            this.setInputStream(in);
            setChannel(Channels.newChannel(in));
            this.setSize(s3object.getObjectMetadata().getContentLength());
            if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
                List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
                String varHeaderLine = generateVariableHeader(datavariables);
                this.setVarHeader(varHeaderLine);
            }
        } else if (isWriteAccess) {
            key = dataFile.getOwner().getAuthority() + "/" + this.getDataFile().getOwner().getIdentifier();
            if (storageIdentifier.startsWith(S3_IDENTIFIER_PREFIX + "://")) {
                key += "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1);
            } else {
                key += "/" + storageIdentifier;
                dvObject.setStorageIdentifier(S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + storageIdentifier);
            }
        }
        this.setMimeType(dataFile.getContentType());
        try {
            this.setFileName(dataFile.getFileMetadata().getLabel());
        } catch (Exception ex) {
            this.setFileName("unknown");
        }
    } else if (dvObject instanceof Dataset) {
        Dataset dataset = this.getDataset();
        key = dataset.getAuthority() + "/" + dataset.getIdentifier();
        dataset.setStorageIdentifier(S3_IDENTIFIER_PREFIX + "://" + key);
    } else if (dvObject instanceof Dataverse) {
        throw new IOException("Data Access: Invalid DvObject type : Dataverse");
    } else {
        throw new IOException("Data Access: Invalid DvObject type");
    }
}

Example 23 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project bender by Nextdoor.

the class S3EventIterator method updateCursor.

private void updateCursor() {
    if (this.currentIndex == 0 || (this.currentIndex < this.records.size() && !this.lineIterator.hasNext())) {
        /*
       * The previous reader must be closed in order to prevent S3 connection leaking
       */
        closeCurrentReader();
        /*
       * Use the S3 trigger event time for arrival time of records in file. This is less precise but
       * avoids making a call to the S3 api to find file creation time. Note that if the
       * deserializer creates a {@link com.nextdoor.bender.deserializer.DeserializedTimeSeriesEvent}
       * then this arrival time is not used.
       */
        S3EventNotificationRecord event = this.records.get(currentIndex);
        this.arrivalTime = event.getEventTime().toDate().getTime();
        this.currentS3Entity = event.getS3();
        /*
       * The S3 Object key is URL encoded and must be decoded before it can be used by the
       * AmazonS3Client
       */
        String key;
        try {
            key = URLDecoder.decode(this.currentS3Entity.getObject().getKey(), "UTF-8");
        } catch (UnsupportedEncodingException e) {
            throw new RuntimeException(e);
        }
        /*
       * Stream object back from S3 into a reader
       */
        String bucketName = this.currentS3Entity.getBucket().getName();
        logger.debug("opening s3://" + bucketName + "/" + key);
        GetObjectRequest req = new GetObjectRequest(bucketName, key);
        S3Object obj = client.getObject(req);
        logger.trace("s3 get request id: " + client.getCachedResponseMetadata(req).getRequestId() + " host: " + client.getCachedResponseMetadata(req).getHostId() + " cloudfrontid: " + client.getCachedResponseMetadata(req).getCloudFrontId());
        // TODO: support different types of compressions
        if (key.endsWith(".gz")) {
            GZIPInputStream gzip;
            try {
                gzip = new GZIPInputStream(obj.getObjectContent());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            reader = new BufferedReader(new InputStreamReader(gzip));
        } else {
            reader = new BufferedReader(new InputStreamReader(obj.getObjectContent()));
        }
        /*
       * Note the BufferedReader is lazy and so is the iterator. The object is directly streamed
       * from S3, fed into an input stream and consumed line by line by the iterator.
       */
        this.lineIterator = reader.lines().iterator();
        currentIndex++;
    }
}

Example 24 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project testcontainers-java by testcontainers.

the class SimpleLocalstackS3Test method s3Test.

@Test
public void s3Test() throws IOException {
    AmazonS3 s3 = AmazonS3ClientBuilder.standard().withEndpointConfiguration(localstack.getEndpointConfiguration(S3)).withCredentials(localstack.getDefaultCredentialsProvider()).build();
    s3.createBucket("foo");
    s3.putObject("foo", "bar", "baz");
    final List<Bucket> buckets = s3.listBuckets();
    assertEquals("The created bucket is present", 1, buckets.size());
    final Bucket bucket = buckets.get(0);
    assertEquals("The created bucket has the right name", "foo", bucket.getName());
    assertEquals("The created bucket has the right name", "foo", bucket.getName());
    final ObjectListing objectListing = s3.listObjects("foo");
    assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size());
    final S3Object object = s3.getObject("foo", "bar");
    final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8"));
    assertEquals("The object can be retrieved", "baz", content);
}

Example 25 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project OpenTripPlanner by opentripplanner.

the class ClusterGraphService method downloadGraphSourceFiles.

private void downloadGraphSourceFiles(String graphId, File dir) throws IOException {
    File graphCacheDir = dir;
    if (!graphCacheDir.exists())
        graphCacheDir.mkdirs();
    File graphZipFile = new File(graphCacheDir, graphId + ".zip");
    File extractedGraphDir = new File(graphCacheDir, graphId);
    if (extractedGraphDir.exists()) {
        FileUtils.deleteDirectory(extractedGraphDir);
    }
    extractedGraphDir.mkdirs();
    S3Object graphZip = s3.getObject(graphBucket, graphId + ".zip");
    InputStream zipFileIn = graphZip.getObjectContent();
    OutputStream zipFileOut = new FileOutputStream(graphZipFile);
    IOUtils.copy(zipFileIn, zipFileOut);
    IOUtils.closeQuietly(zipFileIn);
    IOUtils.closeQuietly(zipFileOut);
    unpackGraphZip(graphZipFile, extractedGraphDir);
}