Examples with InputStreamFromOutputStream com.gc.iotools.stream.is.InputStreamFromOutputStream used on opensource projects

Example 1 with InputStreamFromOutputStream

use of com.gc.iotools.stream.is.InputStreamFromOutputStream in project bender by Nextdoor.

the class S3Transport method sendBatch.

@Override
public void sendBatch(TransportBuffer buffer, LinkedHashMap<String, String> partitions, Context context) throws TransportException {
    S3TransportBuffer buf = (S3TransportBuffer) buffer;
    /*
     * Create s3 key (filepath + filename)
     */
    LinkedHashMap<String, String> parts = new LinkedHashMap<String, String>(partitions);
    String filename = parts.remove(FILENAME_KEY);
    if (filename == null) {
        filename = context.getAwsRequestId();
    }
    String key = parts.entrySet().stream().map(s -> s.getKey() + "=" + s.getValue()).collect(Collectors.joining("/"));
    key = (key.equals("") ? filename : key + '/' + filename);
    if (this.basePath.endsWith("/")) {
        key = this.basePath + key;
    } else {
        key = this.basePath + '/' + key;
    }
    // TODO: make this dynamic
    if (key.endsWith(".gz")) {
        key = key.substring(0, key.length() - 3);
    }
    /*
     * Add or strip out compression format extension
     *
     * TODO: get this based on the compression codec
     */
    if (this.compress || buf.isCompressed()) {
        key += ".bz2";
    }
    ByteArrayOutputStream os = buf.getInternalBuffer();
    /*
     * Compress stream if needed. Don't compress a compressed stream.
     */
    ByteArrayOutputStream payload;
    if (this.compress && !buf.isCompressed()) {
        payload = compress(os);
    } else {
        payload = os;
    }
    /*
     * For memory efficiency convert the output stream into an InputStream. This is done using the
     * easystream library but under the hood it uses piped streams to facilitate this process. This
     * avoids copying the entire contents of the OutputStream to populate the InputStream. Note that
     * this process creates another thread to consume from the InputStream.
     */
    final String s3Key = key;
    /*
     * Write to OutputStream
     */
    final InputStreamFromOutputStream<String> isos = new InputStreamFromOutputStream<String>() {

        public String produce(final OutputStream dataSink) throws Exception {
            /*
         * Note this is executed in a different thread
         */
            payload.writeTo(dataSink);
            return null;
        }
    };
    /*
     * Consume InputStream
     */
    try {
        sendStream(isos, s3Key, payload.size());
    } finally {
        try {
            isos.close();
        } catch (IOException e) {
            throw new TransportException(e);
        } finally {
            buf.close();
        }
    }
}