Examples with CrailNode org.apache.crail.CrailNode used on opensource projects

Example 6 with CrailNode

use of org.apache.crail.CrailNode in project incubator-crail by apache.

the class CrailBenchmark method getFileAsync.

void getFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("getFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> fileQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        // single operation == loop
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fs.lookup(filename);
            fileQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fileQueue.poll();
            future.get();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) batch);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.getStatistics().print("close");
}

Example 7 with CrailNode

use of org.apache.crail.CrailNode in project incubator-crail by apache.

the class CrailBenchmark method browseDir.

void browseDir(String filename) throws Exception {
    System.out.println("reading enumarate dir, path " + filename);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    CrailNode node = fs.lookup(filename).get();
    System.out.println("node type is " + node.getType());
    Iterator<String> iter = node.getType() == CrailNodeType.DIRECTORY ? node.asDirectory().listEntries() : node.asMultiFile().listEntries();
    while (iter.hasNext()) {
        String name = iter.next();
        System.out.println(name);
    }
    fs.getStatistics().print("close");
}

Example 8 with CrailNode

use of org.apache.crail.CrailNode in project incubator-crail by apache.

the class CrailBenchmark method createFileAsync.

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir();
    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        // single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.delete(filename, true).get().syncDir();
    fs.getStatistics().print("close");
}