Examples with RateLimitedOutputStream org.apache.accumulo.core.file.streams.RateLimitedOutputStream used on opensource projects

Example 1 with RateLimitedOutputStream

use of org.apache.accumulo.core.file.streams.RateLimitedOutputStream in project accumulo by apache.

the class RFileOperations method openWriter.

@Override
protected FileSKVWriter openWriter(OpenWriterOperation options) throws IOException {
    AccumuloConfiguration acuconf = options.getTableConfiguration();
    long blockSize = acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE);
    Preconditions.checkArgument((blockSize < Integer.MAX_VALUE && blockSize > 0), "table.file.compress.blocksize must be greater than 0 and less than " + Integer.MAX_VALUE);
    long indexBlockSize = acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX);
    Preconditions.checkArgument((indexBlockSize < Integer.MAX_VALUE && indexBlockSize > 0), "table.file.compress.blocksize.index must be greater than 0 and less than " + Integer.MAX_VALUE);
    SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl.newSamplerConfig(acuconf);
    Sampler sampler = null;
    if (samplerConfig != null) {
        sampler = SamplerFactory.newSampler(samplerConfig, acuconf, options.isAccumuloStartEnabled());
    }
    String compression = options.getCompression();
    compression = compression == null ? options.getTableConfiguration().get(Property.TABLE_FILE_COMPRESSION_TYPE) : compression;
    FSDataOutputStream outputStream = options.getOutputStream();
    Configuration conf = options.getConfiguration();
    if (outputStream == null) {
        int hrep = conf.getInt("dfs.replication", -1);
        int trep = acuconf.getCount(Property.TABLE_FILE_REPLICATION);
        int rep = hrep;
        if (trep > 0 && trep != hrep) {
            rep = trep;
        }
        long hblock = conf.getLong("dfs.block.size", 1 << 26);
        long tblock = acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE);
        long block = hblock;
        if (tblock > 0)
            block = tblock;
        int bufferSize = conf.getInt("io.file.buffer.size", 4096);
        String file = options.getFilename();
        FileSystem fs = options.getFileSystem();
        outputStream = fs.create(new Path(file), false, bufferSize, (short) rep, block);
    }
    CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(new RateLimitedOutputStream(outputStream, options.getRateLimiter()), compression, conf, acuconf);
    RFile.Writer writer = new RFile.Writer(_cbw, (int) blockSize, (int) indexBlockSize, samplerConfig, sampler);
    return writer;
}