Examples with CrailBuffer org.apache.crail.CrailBuffer used on opensource projects

Example 6 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method readSequential.

void readSequential(String filename, int size, int loop, boolean buffered) throws Exception {
    System.out.println("readSequential, filename " + filename + ", size " + size + ", loop " + loop + ", buffered " + buffered);
    CrailBuffer buf = null;
    if (size == CrailConstants.BUFFER_SIZE) {
        buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer _buf = fs.allocateBuffer();
        _buf.clear().limit(size);
        buf = _buf.slice();
    } else {
        buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailBufferedInputStream bufferedStream = file.getBufferedInputStream(file.getCapacity());
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    double sumbytes = 0;
    double ops = 0;
    long start = System.currentTimeMillis();
    while (ops < loop) {
        if (buffered) {
            buf.clear();
            double ret = (double) bufferedStream.read(buf.getByteBuffer());
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                ops = ops + 1.0;
                if (bufferedStream.position() == 0) {
                    break;
                } else {
                    bufferedStream.seek(0);
                }
            }
        } else {
            buf.clear();
            double ret = (double) directStream.read(buf).get().getLen();
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                ops = ops + 1.0;
                if (directStream.position() == 0) {
                    break;
                } else {
                    directStream.seek(0);
                }
            }
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    bufferedStream.close();
    directStream.close();
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    fs.getStatistics().print("close");
}

Example 7 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method getKey.

void getKey(String filename, int size, int loop) throws Exception {
    System.out.println("getKey, path " + filename + ", size " + size + ", loop " + loop);
    CrailBuffer buf = fs.allocateBuffer().clear().limit(size).slice();
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile();
    file.syncDir();
    CrailOutputStream directOutputStream = file.getDirectOutputStream(0);
    directOutputStream.write(buf).get();
    directOutputStream.close();
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        CrailInputStream directInputStream = fs.lookup(filename).get().asFile().getDirectInputStream(0);
        buf.clear();
        directInputStream.read(buf).get();
        directInputStream.close();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.getStatistics().print("close");
}

Example 8 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method readMultiStream.

void readMultiStream(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readMultiStream, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < warmup; i++) {
        CrailBuffer buf = fs.allocateBuffer().limit(size).slice();
        bufferQueue.add(buf);
    }
    warmUp(filename, warmup, bufferQueue);
    while (!bufferQueue.isEmpty()) {
        CrailBuffer buf = bufferQueue.poll();
        fs.freeBuffer(buf);
    }
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    CrailBuffer _buf = null;
    if (size == CrailConstants.BUFFER_SIZE) {
        _buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer __buf = fs.allocateBuffer();
        __buf.clear().limit(size);
        _buf = __buf.slice();
    } else {
        _buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }
    ByteBuffer buf = _buf.getByteBuffer();
    for (int i = 0; i < loop; i++) {
        CrailBufferedInputStream multiStream = fs.lookup(filename).get().asMultiFile().getMultiStream(batch);
        double sumbytes = 0;
        long _sumbytes = 0;
        double ops = 0;
        buf.clear();
        long start = System.currentTimeMillis();
        int ret = multiStream.read(buf);
        while (ret >= 0) {
            sumbytes = sumbytes + ret;
            long _ret = (long) ret;
            _sumbytes += _ret;
            ops = ops + 1.0;
            buf.clear();
            ret = multiStream.read(buf);
        }
        long end = System.currentTimeMillis();
        multiStream.close();
        double executionTime = ((double) (end - start)) / 1000.0;
        double throughput = 0.0;
        double latency = 0.0;
        double sumbits = sumbytes * 8.0;
        if (executionTime > 0) {
            throughput = sumbits / executionTime / 1000.0 / 1000.0;
            latency = 1000000.0 * executionTime / ops;
        }
        System.out.println("round " + i + ":");
        System.out.println("bytes read " + _sumbytes);
        System.out.println("execution time " + executionTime);
        System.out.println("ops " + ops);
        System.out.println("throughput " + throughput);
        System.out.println("latency " + latency);
    }
    fs.getStatistics().print("close");
}

Example 9 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class MappedBufferCache method slice.

private CrailBuffer slice(CrailBuffer mappedBuffer, int position) {
    int limit = position + CrailConstants.BUFFER_SIZE;
    mappedBuffer.clear();
    mappedBuffer.position(position);
    mappedBuffer.limit(limit);
    CrailBuffer buffer = mappedBuffer.slice();
    return buffer;
}

Example 10 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class MappedBufferCache method allocateRegion.

public CrailBuffer allocateRegion() throws IOException {
    if (currentRegion >= allocationCount) {
        return null;
    }
    String path = directory + "/" + currentRegion++;
    RandomAccessFile randomFile = new RandomAccessFile(path, "rw");
    randomFile.setLength(CrailConstants.REGION_SIZE);
    FileChannel channel = randomFile.getChannel();
    MappedByteBuffer _mappedBuffer = channel.map(MapMode.READ_WRITE, 0, CrailConstants.REGION_SIZE);
    CrailBuffer mappedBuffer = OffHeapBuffer.wrap(_mappedBuffer);
    randomFile.close();
    channel.close();
    CrailBuffer firstBuffer = slice(mappedBuffer, 0);
    for (int j = 1; j < bufferCount; j++) {
        int position = j * CrailConstants.BUFFER_SIZE;
        CrailBuffer sliceBuffer = slice(mappedBuffer, position);
        this.putBufferInternal(sliceBuffer);
    }
    mappedBuffer.clear();
    return firstBuffer;
}