use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.
the class CrailHadoopFileSystem method create.
@Override
public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
CrailFile fileInfo = null;
try {
fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
} catch (Exception e) {
if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
fileInfo = null;
} else {
throw new IOException(e);
}
}
if (fileInfo == null) {
Path parent = path.getParent();
this.mkdirs(parent, FsPermission.getDirDefault());
try {
fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
} catch (Exception e) {
throw new IOException(e);
}
}
CrailBufferedOutputStream outputStream = null;
if (fileInfo != null) {
try {
fileInfo.syncDir();
outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
} catch (Exception e) {
throw new IOException(e);
}
}
if (outputStream != null) {
return new CrailHDFSOutputStream(outputStream, statistics);
} else {
throw new IOException("Failed to create file, path " + path.toString());
}
}
use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.
the class CrailBenchmark method write.
void write(String filename, int size, int loop, int storageClass, int locationClass, boolean buffered, boolean skipDir) throws Exception {
System.out.println("write, filename " + filename + ", size " + size + ", loop " + loop + ", storageClass " + storageClass + ", locationClass " + locationClass + ", buffered " + buffered);
CrailBuffer buf = null;
if (size == CrailConstants.BUFFER_SIZE) {
buf = fs.allocateBuffer();
} else if (size < CrailConstants.BUFFER_SIZE) {
CrailBuffer _buf = fs.allocateBuffer();
_buf.clear().limit(size);
buf = _buf.slice();
} else {
buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
}
// warmup
ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
bufferQueue.add(buf);
warmUp(filename, warmup, bufferQueue);
// benchmark
System.out.println("starting benchmark...");
fs.getStatistics().reset();
long _loop = (long) loop;
long _bufsize = (long) CrailConstants.BUFFER_SIZE;
long _capacity = _loop * _bufsize;
double sumbytes = 0;
double ops = 0;
CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass), !skipDir).get().asFile();
CrailBufferedOutputStream bufferedStream = buffered ? file.getBufferedOutputStream(_capacity) : null;
CrailOutputStream directStream = !buffered ? file.getDirectOutputStream(_capacity) : null;
long start = System.currentTimeMillis();
while (ops < loop) {
buf.clear();
if (buffered) {
bufferedStream.write(buf.getByteBuffer());
} else {
directStream.write(buf).get();
}
sumbytes = sumbytes + buf.capacity();
ops = ops + 1.0;
}
if (buffered) {
bufferedStream.close();
}
long end = System.currentTimeMillis();
double executionTime = ((double) (end - start)) / 1000.0;
double throughput = 0.0;
double latency = 0.0;
double sumbits = sumbytes * 8.0;
if (executionTime > 0) {
throughput = sumbits / executionTime / 1000.0 / 1000.0;
latency = 1000000.0 * executionTime / ops;
}
System.out.println("execution time " + executionTime);
System.out.println("ops " + ops);
System.out.println("sumbytes " + sumbytes);
System.out.println("throughput " + throughput);
System.out.println("latency " + latency);
fs.getStatistics().print("close");
}
use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.
the class CrailBenchmark method warmUp.
private void warmUp(String filename, int operations, ConcurrentLinkedQueue<CrailBuffer> bufferList) throws Exception {
Random random = new Random();
String warmupFilename = filename + random.nextInt();
System.out.println("warmUp, warmupFile " + warmupFilename + ", operations " + operations);
if (operations > 0) {
CrailFile warmupFile = fs.create(warmupFilename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile();
CrailBufferedOutputStream warmupStream = warmupFile.getBufferedOutputStream(0);
for (int i = 0; i < operations; i++) {
CrailBuffer buf = bufferList.poll();
buf.clear();
warmupStream.write(buf.getByteBuffer());
bufferList.add(buf);
}
warmupStream.purge().get();
warmupStream.close();
fs.delete(warmupFilename, false).get().syncDir();
}
}
use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.
the class CrailHDFS method createInternal.
@Override
public FSDataOutputStream createInternal(Path path, EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException {
CrailFile fileInfo = null;
try {
fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
} catch (Exception e) {
if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
fileInfo = null;
} else {
throw new IOException(e);
}
}
if (fileInfo == null) {
Path parent = path.getParent();
this.mkdir(parent, FsPermission.getDirDefault(), true);
try {
fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile();
} catch (Exception e) {
throw new IOException(e);
}
}
CrailBufferedOutputStream outputStream = null;
if (fileInfo != null) {
try {
fileInfo.syncDir();
outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
} catch (Exception e) {
throw new IOException(e);
}
} else {
throw new IOException("Failed to create file, path " + path.toString());
}
if (outputStream != null) {
return new CrailHDFSOutputStream(outputStream, statistics);
} else {
throw new IOException("Failed to create file, path " + path.toString());
}
}
use of org.apache.crail.CrailBufferedOutputStream in project incubator-crail by apache.
the class CrailBenchmark method early.
void early(String filename) throws Exception {
ByteBuffer buf = ByteBuffer.allocateDirect(32);
CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).early().asFile();
CrailBufferedOutputStream stream = file.getBufferedOutputStream(0);
System.out.println("buffered stream initialized");
Thread.sleep(1000);
stream.write(buf);
System.out.println("buffered stream written");
Thread.sleep(1000);
stream.write(buf);
System.out.println("buffered stream written");
stream.purge();
stream.close();
System.out.println("buffered stream closed");
fs.getStatistics().print("close");
}
Aggregations