Examples with BufferFileInputStream backtype.storm.utils.BufferFileInputStream used on opensource projects

Example 1 with BufferFileInputStream

use of backtype.storm.utils.BufferFileInputStream in project jstorm by alibaba.

the class StormSubmitter method submitJar.

public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " + "StormSubmitter knows which jar to upload.");
    }
    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }
        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with BufferFileInputStream

use of backtype.storm.utils.BufferFileInputStream in project jstorm by alibaba.

the class ServiceHandler method beginFileDownload.

@Override
public String beginFileDownload(String file) throws TException {
    BufferFileInputStream is;
    String id;
    try {
        int bufferSize = JStormUtils.parseInt(conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE), 1024 * 1024) / 2;
        is = new BufferFileInputStream(file, bufferSize);
        id = UUID.randomUUID().toString();
        data.getDownloaders().put(id, is);
    } catch (FileNotFoundException e) {
        LOG.error(e + "file:" + file + " not found");
        throw new TException(e);
    }
    return id;
}

Example 3 with BufferFileInputStream

use of backtype.storm.utils.BufferFileInputStream in project jstorm by alibaba.

the class NimbusData method createFileHandler.

public void createFileHandler() {
    ExpiredCallback<Object, Object> expiredCallback = new ExpiredCallback<Object, Object>() {

        @Override
        public void expire(Object key, Object val) {
            try {
                LOG.info("Close file " + String.valueOf(key));
                if (val != null) {
                    if (val instanceof Channel) {
                        Channel channel = (Channel) val;
                        channel.close();
                    } else if (val instanceof BufferFileInputStream) {
                        BufferFileInputStream is = (BufferFileInputStream) val;
                        is.close();
                    }
                }
            } catch (IOException e) {
                LOG.error(e.getMessage(), e);
            }
        }
    };
    int file_copy_expiration_secs = JStormUtils.parseInt(conf.get(Config.NIMBUS_FILE_COPY_EXPIRATION_SECS), 30);
    uploaders = new TimeCacheMap<>(file_copy_expiration_secs, expiredCallback);
    downloaders = new TimeCacheMap<>(file_copy_expiration_secs, expiredCallback);
}

Example 4 with BufferFileInputStream

use of backtype.storm.utils.BufferFileInputStream in project storm by nathanmarz.

the class StormSubmitter method submitJar.

public static String submitJar(Map conf, String localJar) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String uploadLocation = client.getClient().beginFileUpload();
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        BufferFileInputStream is = new BufferFileInputStream(localJar);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}

Example 5 with BufferFileInputStream

use of backtype.storm.utils.BufferFileInputStream in project jstorm by alibaba.

the class ServiceHandler method downloadChunk.

@Override
public ByteBuffer downloadChunk(String id) throws TException {
    TimeCacheMap<Object, Object> downloaders = data.getDownloaders();
    Object obj = downloaders.get(id);
    if (obj == null) {
        throw new TException("Could not find input stream for that id");
    }
    try {
        if (obj instanceof BufferFileInputStream) {
            BufferFileInputStream is = (BufferFileInputStream) obj;
            byte[] ret = is.read();
            if (ret != null) {
                downloaders.put(id, is);
                return ByteBuffer.wrap(ret);
            }
        } else {
            throw new TException("Object isn't BufferFileInputStream for " + id);
        }
    } catch (IOException e) {
        LOG.error("BufferFileInputStream read failed when downloadChunk ", e);
        throw new TException(e);
    }
    byte[] empty = {};
    return ByteBuffer.wrap(empty);
}