Examples with CrailBuffer org.apache.crail.CrailBuffer used on opensource projects

Example 16 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method enumerateDir.

void enumerateDir(String filename, int loop) throws Exception {
    System.out.println("reading enumarate dir, path " + filename);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        // single operation == loop
        Iterator<String> iter = fs.lookup(filename).get().asDirectory().listEntries();
        while (iter.hasNext()) {
            iter.next();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.getStatistics().print("close");
}

Example 17 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method readSequentialAsync.

void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch);
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }
    // warmup
    warmUp(filename, warmup, bufferQueue);
    // benchmark
    System.out.println("starting benchmark...");
    double sumbytes = 0;
    double ops = 0;
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        future = futureQueue.poll();
        CrailResult result = future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        CrailResult result = future.get();
        futureMap.get(future.hashCode());
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    fs.getStatistics().print("close");
}

Example 18 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class CrailBenchmark method createFileAsync.

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir();
    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        // single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.delete(filename, true).get().syncDir();
    fs.getStatistics().print("close");
}

Example 19 with CrailBuffer

use of org.apache.crail.CrailBuffer in project incubator-crail by apache.

the class BufferCheckpoint method checkIn.

public void checkIn(CrailBuffer buffer) throws Exception {
    long address = buffer.address();
    CrailBuffer oldBuf = checkpoint.putIfAbsent(address, buffer);
    if (oldBuf != null) {
        LOG.info("ERROR Buffer already in use!");
        StackTraceElement[] stack = Thread.currentThread().getStackTrace();
        for (StackTraceElement element : stack) {
            LOG.info(element.toString());
        }
        throw new Exception("ERROR Buffer already in use!");
    }
}