Examples with CrailFile org.apache.crail.CrailFile used on opensource projects

Example 11 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailHadoopFileSystem method open.

public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    CrailFile fileInfo = null;
    try {
        fileInfo = dfs.lookup(path.toUri().getRawPath()).get().asFile();
        CrailBufferedInputStream inputStream = fileInfo.getBufferedInputStream(fileInfo.getCapacity());
        return new CrailHDFSInputStream(inputStream);
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 12 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method early.

void early(String filename) throws Exception {
    ByteBuffer buf = ByteBuffer.allocateDirect(32);
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).early().asFile();
    CrailBufferedOutputStream stream = file.getBufferedOutputStream(0);
    System.out.println("buffered stream initialized");
    Thread.sleep(1000);
    stream.write(buf);
    System.out.println("buffered stream written");
    Thread.sleep(1000);
    stream.write(buf);
    System.out.println("buffered stream written");
    stream.purge();
    stream.close();
    System.out.println("buffered stream closed");
    fs.getStatistics().print("close");
}

Example 13 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method readInt.

void readInt(String filename, int loop) throws Exception {
    System.out.println("seek, filename " + filename + ", loop " + loop);
    // benchmark
    System.out.println("starting benchmark...");
    double ops = 0;
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailBufferedInputStream inputStream = file.getBufferedInputStream(loop * 4);
    System.out.println("starting read at position " + inputStream.position());
    while (ops < loop) {
        System.out.print("reading position " + inputStream.position() + ", expected " + inputStream.position() / 4 + " ");
        int intValue = inputStream.readInt();
        System.out.println(", value " + intValue);
        ops++;
    }
    inputStream.close();
    fs.getStatistics().print("close");
}

Example 14 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method seekInt.

void seekInt(String filename, int loop) throws Exception {
    System.out.println("seek, filename " + filename + ", loop " + loop);
    // benchmark
    System.out.println("starting benchmark...");
    double ops = 0;
    CrailFile file = fs.lookup(filename).get().asFile();
    Random random = new Random();
    long nbrOfInts = file.getCapacity() / 4;
    CrailBufferedInputStream seekStream = file.getBufferedInputStream(loop * 4);
    System.out.println("starting seek phase, nbrOfInts " + nbrOfInts + ", position " + seekStream.position());
    long falseMatches = 0;
    while (ops < loop) {
        int intIndex = random.nextInt((int) nbrOfInts);
        int pos = intIndex * 4;
        seekStream.seek((long) pos);
        int intValue = seekStream.readInt();
        if (intIndex != intValue) {
            falseMatches++;
            System.out.println("reading, position " + pos + ", expected " + pos / 4 + ", ########## value " + intValue);
        } else {
            System.out.println("reading, position " + pos + ", expected " + pos / 4 + ", value " + intValue);
        }
        ops++;
    }
    seekStream.close();
    long end = System.currentTimeMillis();
    System.out.println("falseMatches " + falseMatches);
    fs.getStatistics().print("close");
}

Example 15 with CrailFile

use of org.apache.crail.CrailFile in project incubator-crail by apache.

the class CrailBenchmark method readSequentialAsync.

void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch);
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }
    // warmup
    warmUp(filename, warmup, bufferQueue);
    // benchmark
    System.out.println("starting benchmark...");
    double sumbytes = 0;
    double ops = 0;
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        future = futureQueue.poll();
        CrailResult result = future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        CrailResult result = future.get();
        futureMap.get(future.hashCode());
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);
    fs.getStatistics().print("close");
}