Examples with CellScannable org.apache.hadoop.hbase.CellScannable used on opensource projects

Example 1 with CellScannable

use of org.apache.hadoop.hbase.CellScannable in project hbase by apache.

the class AsyncBatchRpcRetryingCaller method send.

private void send(Map<ServerName, ServerRequest> actionsByServer, int tries) {
    long remainingNs;
    if (operationTimeoutNs > 0) {
        remainingNs = remainingTimeNs();
        if (remainingNs <= 0) {
            failAll(actionsByServer.values().stream().flatMap(m -> m.actionsByRegion.values().stream()).flatMap(r -> r.actions.stream()), tries);
            return;
        }
    } else {
        remainingNs = Long.MAX_VALUE;
    }
    actionsByServer.forEach((sn, serverReq) -> {
        ClientService.Interface stub;
        try {
            stub = conn.getRegionServerStub(sn);
        } catch (IOException e) {
            onError(serverReq.actionsByRegion, tries, e, sn);
            return;
        }
        ClientProtos.MultiRequest req;
        List<CellScannable> cells = new ArrayList<>();
        try {
            req = buildReq(serverReq.actionsByRegion, cells);
        } catch (IOException e) {
            onError(serverReq.actionsByRegion, tries, e, sn);
            return;
        }
        HBaseRpcController controller = conn.rpcControllerFactory.newController();
        resetController(controller, Math.min(rpcTimeoutNs, remainingNs));
        if (!cells.isEmpty()) {
            controller.setCellScanner(createCellScanner(cells));
        }
        stub.multi(controller, req, resp -> {
            if (controller.failed()) {
                onError(serverReq.actionsByRegion, tries, controller.getFailed(), sn);
            } else {
                try {
                    onComplete(serverReq.actionsByRegion, tries, sn, ResponseConverter.getResults(req, resp, controller.cellScanner()));
                } catch (Exception e) {
                    onError(serverReq.actionsByRegion, tries, e, sn);
                    return;
                }
            }
        });
    });
}

Example 2 with CellScannable

use of org.apache.hadoop.hbase.CellScannable in project hbase by apache.

the class RSRpcServices method multi.

/**
 * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
 * @param rpcc the RPC controller
 * @param request the multi request
 * @throws ServiceException
 */
@Override
public MultiResponse multi(final RpcController rpcc, final MultiRequest request) throws ServiceException {
    try {
        checkOpen();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    }
    checkBatchSizeAndLogLargeSize(request);
    // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
    // It is also the conduit via which we pass back data.
    HBaseRpcController controller = (HBaseRpcController) rpcc;
    CellScanner cellScanner = controller != null ? controller.cellScanner() : null;
    if (controller != null) {
        controller.setCellScanner(null);
    }
    long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
    MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
    RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
    this.rpcMultiRequestCount.increment();
    this.requestCount.increment();
    ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
    // MultiRequest#condition in case of checkAndMutate with RowMutations.
    if (request.hasCondition()) {
        if (request.getRegionActionList().isEmpty()) {
            // If the region action list is empty, do nothing.
            responseBuilder.setProcessed(true);
            return responseBuilder.build();
        }
        RegionAction regionAction = request.getRegionAction(0);
        // we can assume regionAction.getAtomic() is true here.
        assert regionAction.getAtomic();
        OperationQuota quota;
        HRegion region;
        RegionSpecifier regionSpecifier = regionAction.getRegion();
        try {
            region = getRegion(regionSpecifier);
            quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList());
        } catch (IOException e) {
            failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e);
            return responseBuilder.build();
        }
        try {
            boolean rejectIfFromClient = shouldRejectRequestsFromClient(region);
            // We only allow replication in standby state and it will not set the atomic flag.
            if (rejectIfFromClient) {
                failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                return responseBuilder.build();
            }
            try {
                CheckAndMutateResult result = checkAndMutate(region, regionAction.getActionList(), cellScanner, request.getCondition(), nonceGroup, spaceQuotaEnforcement);
                responseBuilder.setProcessed(result.isSuccess());
                ClientProtos.ResultOrException.Builder resultOrExceptionOrBuilder = ClientProtos.ResultOrException.newBuilder();
                for (int i = 0; i < regionAction.getActionCount(); i++) {
                    // To unify the response format with doNonAtomicRegionMutation and read through
                    // client's AsyncProcess we have to add an empty result instance per operation
                    resultOrExceptionOrBuilder.clear();
                    resultOrExceptionOrBuilder.setIndex(i);
                    regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                }
            } catch (IOException e) {
                rpcServer.getMetrics().exception(e);
                // As it's an atomic operation with a condition, we may expect it's a global failure.
                regionActionResultBuilder.setException(ResponseConverter.buildException(e));
            }
        } finally {
            quota.close();
        }
        responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
        ClientProtos.RegionLoadStats regionLoadStats = region.getLoadStatistics();
        if (regionLoadStats != null) {
            responseBuilder.setRegionStatistics(MultiRegionLoadStats.newBuilder().addRegion(regionSpecifier).addStat(regionLoadStats).build());
        }
        return responseBuilder.build();
    }
    // this will contain all the cells that we need to return. It's created later, if needed.
    List<CellScannable> cellsToReturn = null;
    RegionScannersCloseCallBack closeCallBack = null;
    RpcCallContext context = RpcServer.getCurrentCall().orElse(null);
    Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request.getRegionActionCount());
    for (RegionAction regionAction : request.getRegionActionList()) {
        OperationQuota quota;
        HRegion region;
        RegionSpecifier regionSpecifier = regionAction.getRegion();
        regionActionResultBuilder.clear();
        try {
            region = getRegion(regionSpecifier);
            quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList());
        } catch (IOException e) {
            failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e);
            // For this region it's a failure.
            continue;
        }
        try {
            boolean rejectIfFromClient = shouldRejectRequestsFromClient(region);
            if (regionAction.hasCondition()) {
                // We only allow replication in standby state and it will not set the atomic flag.
                if (rejectIfFromClient) {
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                try {
                    ClientProtos.ResultOrException.Builder resultOrExceptionOrBuilder = ClientProtos.ResultOrException.newBuilder();
                    if (regionAction.getActionCount() == 1) {
                        CheckAndMutateResult result = checkAndMutate(region, quota, regionAction.getAction(0).getMutation(), cellScanner, regionAction.getCondition(), nonceGroup, spaceQuotaEnforcement);
                        regionActionResultBuilder.setProcessed(result.isSuccess());
                        resultOrExceptionOrBuilder.setIndex(0);
                        if (result.getResult() != null) {
                            resultOrExceptionOrBuilder.setResult(ProtobufUtil.toResult(result.getResult()));
                        }
                        regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                    } else {
                        CheckAndMutateResult result = checkAndMutate(region, regionAction.getActionList(), cellScanner, regionAction.getCondition(), nonceGroup, spaceQuotaEnforcement);
                        regionActionResultBuilder.setProcessed(result.isSuccess());
                        for (int i = 0; i < regionAction.getActionCount(); i++) {
                            if (i == 0 && result.getResult() != null) {
                                // Set the result of the Increment/Append operations to the first element of the
                                // ResultOrException list
                                resultOrExceptionOrBuilder.setIndex(i);
                                regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.setResult(ProtobufUtil.toResult(result.getResult())).build());
                                continue;
                            }
                            // To unify the response format with doNonAtomicRegionMutation and read through
                            // client's AsyncProcess we have to add an empty result instance per operation
                            resultOrExceptionOrBuilder.clear();
                            resultOrExceptionOrBuilder.setIndex(i);
                            regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                        }
                    }
                } catch (IOException e) {
                    rpcServer.getMetrics().exception(e);
                    // As it's an atomic operation with a condition, we may expect it's a global failure.
                    regionActionResultBuilder.setException(ResponseConverter.buildException(e));
                }
            } else if (regionAction.hasAtomic() && regionAction.getAtomic()) {
                // We only allow replication in standby state and it will not set the atomic flag.
                if (rejectIfFromClient) {
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                try {
                    doAtomicBatchOp(regionActionResultBuilder, region, quota, regionAction.getActionList(), cellScanner, nonceGroup, spaceQuotaEnforcement);
                    regionActionResultBuilder.setProcessed(true);
                    // We no longer use MultiResponse#processed. Instead, we use
                    // RegionActionResult#processed. This is for backward compatibility for old clients.
                    responseBuilder.setProcessed(true);
                } catch (IOException e) {
                    rpcServer.getMetrics().exception(e);
                    // As it's atomic, we may expect it's a global failure.
                    regionActionResultBuilder.setException(ResponseConverter.buildException(e));
                }
            } else {
                if (rejectIfFromClient && regionAction.getActionCount() > 0 && !isReplicationRequest(regionAction.getAction(0))) {
                    // fail if it is not a replication request
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                // doNonAtomicRegionMutation manages the exception internally
                if (context != null && closeCallBack == null) {
                    // An RpcCallBack that creates a list of scanners that needs to perform callBack
                    // operation on completion of multiGets.
                    // Set this only once
                    closeCallBack = new RegionScannersCloseCallBack();
                    context.setCallBack(closeCallBack);
                }
                cellsToReturn = doNonAtomicRegionMutation(region, quota, regionAction, cellScanner, regionActionResultBuilder, cellsToReturn, nonceGroup, closeCallBack, context, spaceQuotaEnforcement);
            }
        } finally {
            quota.close();
        }
        responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
        ClientProtos.RegionLoadStats regionLoadStats = region.getLoadStatistics();
        if (regionLoadStats != null) {
            regionStats.put(regionSpecifier, regionLoadStats);
        }
    }
    // Load the controller with the Cells to return.
    if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
        controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
    }
    MultiRegionLoadStats.Builder builder = MultiRegionLoadStats.newBuilder();
    for (Entry<RegionSpecifier, ClientProtos.RegionLoadStats> stat : regionStats.entrySet()) {
        builder.addRegion(stat.getKey());
        builder.addStat(stat.getValue());
    }
    responseBuilder.setRegionStatistics(builder);
    return responseBuilder.build();
}

Example 3 with CellScannable

use of org.apache.hadoop.hbase.CellScannable in project hbase by apache.

the class MultiServerCallable method rpcCall.

@Override
protected MultiResponse rpcCall() throws Exception {
    int countOfActions = this.multiAction.size();
    if (countOfActions <= 0)
        throw new DoNotRetryIOException("No Actions");
    MultiRequest.Builder multiRequestBuilder = MultiRequest.newBuilder();
    RegionAction.Builder regionActionBuilder = RegionAction.newBuilder();
    ClientProtos.Action.Builder actionBuilder = ClientProtos.Action.newBuilder();
    MutationProto.Builder mutationBuilder = MutationProto.newBuilder();
    List<CellScannable> cells = null;
    // The multi object is a list of Actions by region.  Iterate by region.
    long nonceGroup = multiAction.getNonceGroup();
    if (nonceGroup != HConstants.NO_NONCE) {
        multiRequestBuilder.setNonceGroup(nonceGroup);
    }
    for (Map.Entry<byte[], List<Action>> e : this.multiAction.actions.entrySet()) {
        final byte[] regionName = e.getKey();
        final List<Action> actions = e.getValue();
        regionActionBuilder.clear();
        regionActionBuilder.setRegion(RequestConverter.buildRegionSpecifier(HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME, regionName));
        if (this.cellBlock) {
            // Pre-size. Presume at least a KV per Action.  There are likely more.
            if (cells == null)
                cells = new ArrayList<>(countOfActions);
            // Send data in cellblocks. The call to buildNoDataMultiRequest will skip RowMutations.
            // They have already been handled above. Guess at count of cells
            regionActionBuilder = RequestConverter.buildNoDataRegionAction(regionName, actions, cells, regionActionBuilder, actionBuilder, mutationBuilder);
        } else {
            regionActionBuilder = RequestConverter.buildRegionAction(regionName, actions, regionActionBuilder, actionBuilder, mutationBuilder);
        }
        multiRequestBuilder.addRegionAction(regionActionBuilder.build());
    }
    if (cells != null) {
        setRpcControllerCellScanner(CellUtil.createCellScanner(cells));
    }
    ClientProtos.MultiRequest requestProto = multiRequestBuilder.build();
    ClientProtos.MultiResponse responseProto = getStub().multi(getRpcController(), requestProto);
    // Occurs on cancel
    if (responseProto == null)
        return null;
    return ResponseConverter.getResults(requestProto, responseProto, getRpcControllerCellScanner());
}

Example 4 with CellScannable

use of org.apache.hadoop.hbase.CellScannable in project hbase by apache.

the class AsyncBatchRpcRetryingCaller method sendToServer.

private void sendToServer(ServerName serverName, ServerRequest serverReq, int tries) {
    long remainingNs;
    if (operationTimeoutNs > 0) {
        remainingNs = remainingTimeNs();
        if (remainingNs <= 0) {
            failAll(serverReq.actionsByRegion.values().stream().flatMap(r -> r.actions.stream()), tries);
            return;
        }
    } else {
        remainingNs = Long.MAX_VALUE;
    }
    ClientService.Interface stub;
    try {
        stub = conn.getRegionServerStub(serverName);
    } catch (IOException e) {
        onError(serverReq.actionsByRegion, tries, e, serverName);
        return;
    }
    ClientProtos.MultiRequest req;
    List<CellScannable> cells = new ArrayList<>();
    // Map from a created RegionAction to the original index for a RowMutations within
    // the original list of actions. This will be used to process the results when there
    // is RowMutations/CheckAndMutate in the action list.
    Map<Integer, Integer> indexMap = new HashMap<>();
    try {
        req = buildReq(serverReq.actionsByRegion, cells, indexMap);
    } catch (IOException e) {
        onError(serverReq.actionsByRegion, tries, e, serverName);
        return;
    }
    HBaseRpcController controller = conn.rpcControllerFactory.newController();
    resetController(controller, Math.min(rpcTimeoutNs, remainingNs), calcPriority(serverReq.getPriority(), tableName));
    if (!cells.isEmpty()) {
        controller.setCellScanner(createCellScanner(cells));
    }
    stub.multi(controller, req, resp -> {
        if (controller.failed()) {
            onError(serverReq.actionsByRegion, tries, controller.getFailed(), serverName);
        } else {
            try {
                onComplete(serverReq.actionsByRegion, tries, serverName, ResponseConverter.getResults(req, indexMap, resp, controller.cellScanner()));
            } catch (Exception e) {
                onError(serverReq.actionsByRegion, tries, e, serverName);
                return;
            }
        }
    });
}

Example 5 with CellScannable

use of org.apache.hadoop.hbase.CellScannable in project hbase by apache.

the class TestHBaseRpcControllerImpl method testListOfCellScannerables.

@Test
public void testListOfCellScannerables() throws IOException {
    final int count = 10;
    List<CellScannable> cells = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        cells.add(createCell(i));
    }
    HBaseRpcController controller = new HBaseRpcControllerImpl(null, cells);
    CellScanner cellScanner = controller.cellScanner();
    int index = 0;
    for (; cellScanner.advance(); index++) {
        Cell cell = cellScanner.current();
        byte[] indexBytes = Bytes.toBytes(index);
        assertTrue("" + index, Bytes.equals(indexBytes, 0, indexBytes.length, cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
    }
    assertEquals(count, index);
}