Examples with RpcGetBlock org.apache.crail.rpc.RpcGetBlock used on opensource projects

Example 1 with RpcGetBlock

use of org.apache.crail.rpc.RpcGetBlock in project incubator-crail by apache.

the class CoreStream method dataOperation.

final CoreDataOperation dataOperation(CrailBuffer dataBuf) throws Exception {
    blockMap.clear();
    pendingBlocks.clear();
    CoreDataOperation multiOperation = new CoreDataOperation(this, dataBuf);
    // compute off, len for the fragments, start transfer or start RPC if block info is missing
    while (multiOperation.remaining() > 0) {
        long blockRemaining = blockRemaining();
        int opLen = CrailUtils.minFileBuf(blockRemaining, multiOperation.remaining());
        CoreSubOperation subOperation = new CoreSubOperation(fileInfo.getFd(), position, multiOperation.getCurrentBufferPosition(), opLen);
        // LOG.info("OpDesc: " + opDesc.toString());
        ioStats.incTotalOps((long) opLen);
        if (blockCache.containsKey(subOperation.key())) {
            BlockInfo block = blockCache.get(subOperation.key());
            StorageFuture subFuture = this.prepareAndTrigger(subOperation, dataBuf, block);
            multiOperation.add(subFuture);
            this.ioStats.incCachedOps();
        } else if (nextBlockCache.containsKey(subOperation.key())) {
            RpcFuture<RpcGetBlock> rpcFuture = nextBlockCache.get(subOperation.key());
            blockMap.put(rpcFuture.getTicket(), subOperation);
            pendingBlocks.add(rpcFuture);
        } else {
            this.syncedCapacity = fileInfo.getCapacity();
            RpcFuture<RpcGetBlock> rpcFuture = namenodeClientRpc.getBlock(fileInfo.getFd(), fileInfo.getToken(), position, syncedCapacity);
            blockMap.put(rpcFuture.getTicket(), subOperation);
            pendingBlocks.add(rpcFuture);
        }
        position += opLen;
        multiOperation.incProcessedLen(opLen);
    }
    // wait for RPC results and start reads for those blocks as well
    for (RpcFuture<RpcGetBlock> rpcFuture = pendingBlocks.poll(); rpcFuture != null; rpcFuture = pendingBlocks.poll()) {
        if (!rpcFuture.isDone()) {
            this.ioStats.incBlockingOps();
            if (rpcFuture.isPrefetched()) {
                this.ioStats.incPrefetchedBlockingOps();
            }
        } else {
            this.ioStats.incNonblockingOps();
            if (rpcFuture.isPrefetched()) {
                this.ioStats.incPrefetchedNonblockingOps();
            }
        }
        RpcGetBlock getBlockRes = rpcFuture.get(CrailConstants.RPC_TIMEOUT, TimeUnit.MILLISECONDS);
        if (!rpcFuture.isDone()) {
            throw new IOException("rpc timeout ");
        }
        if (getBlockRes.getError() != RpcErrors.ERR_OK) {
            LOG.info("inputStream: " + RpcErrors.messages[getBlockRes.getError()]);
            throw new IOException(RpcErrors.messages[getBlockRes.getError()]);
        }
        BlockInfo block = getBlockRes.getBlockInfo();
        CoreSubOperation subOperation = blockMap.get(rpcFuture.getTicket());
        StorageFuture subFuture = prepareAndTrigger(subOperation, dataBuf, block);
        multiOperation.add(subFuture);
        blockCache.put(subOperation.key(), block);
    }
    if (!multiOperation.isProcessed()) {
        throw new IOException("Internal error, processed data != operation length");
    }
    dataBuf.limit(multiOperation.getBufferLimit());
    dataBuf.position(multiOperation.getCurrentBufferPosition());
    return multiOperation;
}