Examples with CrailFile org.apache.crail.CrailFile used on opensource projects

Example 16 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method createFileAsync.

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);
    // warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);
    // benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir();
    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        // single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);
    fs.delete(filename, true).get().syncDir();
    fs.getStatistics().print("close");
}

Example 17 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method writeInt.

void writeInt(String filename, int loop) throws Exception {
    System.out.println("writeInt, filename " + filename + ", loop " + loop);
    // benchmark
    System.out.println("starting benchmark...");
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile();
    CrailBufferedOutputStream outputStream = file.getBufferedOutputStream(loop * 4);
    int intValue = 0;
    System.out.println("starting write at position " + outputStream.position());
    while (ops < loop) {
        System.out.println("writing position " + outputStream.position() + ", value " + intValue);
        outputStream.writeInt(intValue);
        intValue++;
        ops++;
    }
    outputStream.purge().get();
    outputStream.sync().get();
    fs.getStatistics().print("close");
}