Examples with CrailBufferedOutputStream org.apache.crail.CrailBufferedOutputStream used on opensource projects

Example 1 with CrailBufferedOutputStream

use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.

the class CrailHadoopFileSystem method create.

@Override
public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
    CrailFile fileInfo = null;
    try {
        fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            fileInfo = null;
        } else {
            throw new IOException(e);
        }
    }
    if (fileInfo == null) {
        Path parent = path.getParent();
        this.mkdirs(parent, FsPermission.getDirDefault());
        try {
            fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    CrailBufferedOutputStream outputStream = null;
    if (fileInfo != null) {
        try {
            fileInfo.syncDir();
            outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    if (outputStream != null) {
        return new CrailHDFSOutputStream(outputStream, statistics);
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }
}

Example 2 with CrailBufferedOutputStream

use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.

the class CrailBenchmark method write.

void write(String filename, int size, int loop, int storageClass, int locationClass, boolean buffered, boolean skipDir) throws Exception {
    System.out.println("write, filename " + filename + ", size " + size + ", loop " + loop + ", storageClass " + storageClass + ", locationClass " + locationClass + ", buffered " + buffered);
    CrailBuffer buf = null;
    if (size == CrailConstants.BUFFER_SIZE) {
        buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer _buf = fs.allocateBuffer();
        _buf.clear().limit(size);
        buf = _buf.slice();
    } else {
        buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    long _loop = (long) loop;
    long _bufsize = (long) CrailConstants.BUFFER_SIZE;
    long _capacity = _loop * _bufsize;
    double sumbytes = 0;
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass), !skipDir).get().asFile();
    CrailBufferedOutputStream bufferedStream = buffered ? file.getBufferedOutputStream(_capacity) : null;
    CrailOutputStream directStream = !buffered ? file.getDirectOutputStream(_capacity) : null;
    long start = System.currentTimeMillis();
    while (ops < loop) {
        buf.clear();
        if (buffered) {
            bufferedStream.write(buf.getByteBuffer());
        } else {
            directStream.write(buf).get();
        }
        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    if (buffered) {
        bufferedStream.close();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    fs.getStatistics().print("close");
}

Example 3 with CrailBufferedOutputStream

use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.

the class CrailBenchmark method warmUp.

private void warmUp(String filename, int operations, ConcurrentLinkedQueue<CrailBuffer> bufferList) throws Exception {
    Random random = new Random();
    String warmupFilename = filename + random.nextInt();
    System.out.println("warmUp, warmupFile " + warmupFilename + ", operations " + operations);
    if (operations > 0) {
        CrailFile warmupFile = fs.create(warmupFilename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile();
        CrailBufferedOutputStream warmupStream = warmupFile.getBufferedOutputStream(0);
        for (int i = 0; i < operations; i++) {
            CrailBuffer buf = bufferList.poll();
            buf.clear();
            warmupStream.write(buf.getByteBuffer());
            bufferList.add(buf);
        }
        warmupStream.purge().get();
        warmupStream.close();
        fs.delete(warmupFilename, false).get().syncDir();
    }
}

Example 4 with CrailBufferedOutputStream

use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.

the class CrailHDFS method createInternal.

@Override
public FSDataOutputStream createInternal(Path path, EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException {
    CrailFile fileInfo = null;
    try {
        fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            fileInfo = null;
        } else {
            throw new IOException(e);
        }
    }
    if (fileInfo == null) {
        Path parent = path.getParent();
        this.mkdir(parent, FsPermission.getDirDefault(), true);
        try {
            fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    CrailBufferedOutputStream outputStream = null;
    if (fileInfo != null) {
        try {
            fileInfo.syncDir();
            outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
        } catch (Exception e) {
            throw new IOException(e);
        }
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }
    if (outputStream != null) {
        return new CrailHDFSOutputStream(outputStream, statistics);
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }
}

Example 5 with CrailBufferedOutputStream

use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.

the class CrailBenchmark method early.

void early(String filename) throws Exception {
    ByteBuffer buf = ByteBuffer.allocateDirect(32);
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).early().asFile();
    CrailBufferedOutputStream stream = file.getBufferedOutputStream(0);
    System.out.println("buffered stream initialized");
    Thread.sleep(1000);
    stream.write(buf);
    System.out.println("buffered stream written");
    Thread.sleep(1000);
    stream.write(buf);
    System.out.println("buffered stream written");
    stream.purge();
    stream.close();
    System.out.println("buffered stream closed");
    fs.getStatistics().print("close");
}