Examples with ActivePolicyEnforcement org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement used on opensource projects

Example 1 with ActivePolicyEnforcement

use of org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement in project hbase by apache.

the class RSRpcServices method bulkLoadHFile.

/**
 * Atomically bulk load several HFiles into an open region
 * @return true if successful, false is failed but recoverably (no action)
 * @throws ServiceException if failed unrecoverably
 */
@Override
public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException {
    long start = EnvironmentEdgeManager.currentTime();
    List<String> clusterIds = new ArrayList<>(request.getClusterIdsList());
    if (clusterIds.contains(this.server.getClusterId())) {
        return BulkLoadHFileResponse.newBuilder().setLoaded(true).build();
    } else {
        clusterIds.add(this.server.getClusterId());
    }
    try {
        checkOpen();
        requestCount.increment();
        HRegion region = getRegion(request.getRegion());
        final boolean spaceQuotaEnabled = QuotaUtil.isQuotaEnabled(getConfiguration());
        long sizeToBeLoaded = -1;
        // Check to see if this bulk load would exceed the space quota for this table
        if (spaceQuotaEnabled) {
            ActivePolicyEnforcement activeSpaceQuotas = getSpaceQuotaManager().getActiveEnforcements();
            SpaceViolationPolicyEnforcement enforcement = activeSpaceQuotas.getPolicyEnforcement(region);
            if (enforcement != null) {
                // Bulk loads must still be atomic. We must enact all or none.
                List<String> filePaths = new ArrayList<>(request.getFamilyPathCount());
                for (FamilyPath familyPath : request.getFamilyPathList()) {
                    filePaths.add(familyPath.getPath());
                }
                // Check if the batch of files exceeds the current quota
                sizeToBeLoaded = enforcement.computeBulkLoadSize(getFileSystem(filePaths), filePaths);
            }
        }
        // secure bulk load
        Map<byte[], List<Path>> map = server.getSecureBulkLoadManager().secureBulkLoadHFiles(region, request, clusterIds);
        BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
        builder.setLoaded(map != null);
        if (map != null) {
            // not possible to occur in real life (cannot bulk load a file with negative size)
            if (spaceQuotaEnabled && sizeToBeLoaded > 0) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Incrementing space use of " + region.getRegionInfo() + " by " + sizeToBeLoaded + " bytes");
                }
                // Inform space quotas of the new files for this region
                getSpaceQuotaManager().getRegionSizeStore().incrementRegionSize(region.getRegionInfo(), sizeToBeLoaded);
            }
        }
        return builder.build();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    } finally {
        final MetricsRegionServer metricsRegionServer = server.getMetrics();
        if (metricsRegionServer != null) {
            metricsRegionServer.updateBulkLoad(EnvironmentEdgeManager.currentTime() - start);
        }
    }
}

Example 2 with ActivePolicyEnforcement

use of org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement in project hbase by apache.

the class RSRpcServices method multi.

/**
 * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
 * @param rpcc the RPC controller
 * @param request the multi request
 * @throws ServiceException
 */
@Override
public MultiResponse multi(final RpcController rpcc, final MultiRequest request) throws ServiceException {
    try {
        checkOpen();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    }
    checkBatchSizeAndLogLargeSize(request);
    // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
    // It is also the conduit via which we pass back data.
    HBaseRpcController controller = (HBaseRpcController) rpcc;
    CellScanner cellScanner = controller != null ? controller.cellScanner() : null;
    if (controller != null) {
        controller.setCellScanner(null);
    }
    long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
    MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
    RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
    this.rpcMultiRequestCount.increment();
    this.requestCount.increment();
    ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
    // MultiRequest#condition in case of checkAndMutate with RowMutations.
    if (request.hasCondition()) {
        if (request.getRegionActionList().isEmpty()) {
            // If the region action list is empty, do nothing.
            responseBuilder.setProcessed(true);
            return responseBuilder.build();
        }
        RegionAction regionAction = request.getRegionAction(0);
        // we can assume regionAction.getAtomic() is true here.
        assert regionAction.getAtomic();
        OperationQuota quota;
        HRegion region;
        RegionSpecifier regionSpecifier = regionAction.getRegion();
        try {
            region = getRegion(regionSpecifier);
            quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList());
        } catch (IOException e) {
            failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e);
            return responseBuilder.build();
        }
        try {
            boolean rejectIfFromClient = shouldRejectRequestsFromClient(region);
            // We only allow replication in standby state and it will not set the atomic flag.
            if (rejectIfFromClient) {
                failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                return responseBuilder.build();
            }
            try {
                CheckAndMutateResult result = checkAndMutate(region, regionAction.getActionList(), cellScanner, request.getCondition(), nonceGroup, spaceQuotaEnforcement);
                responseBuilder.setProcessed(result.isSuccess());
                ClientProtos.ResultOrException.Builder resultOrExceptionOrBuilder = ClientProtos.ResultOrException.newBuilder();
                for (int i = 0; i < regionAction.getActionCount(); i++) {
                    // To unify the response format with doNonAtomicRegionMutation and read through
                    // client's AsyncProcess we have to add an empty result instance per operation
                    resultOrExceptionOrBuilder.clear();
                    resultOrExceptionOrBuilder.setIndex(i);
                    regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                }
            } catch (IOException e) {
                rpcServer.getMetrics().exception(e);
                // As it's an atomic operation with a condition, we may expect it's a global failure.
                regionActionResultBuilder.setException(ResponseConverter.buildException(e));
            }
        } finally {
            quota.close();
        }
        responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
        ClientProtos.RegionLoadStats regionLoadStats = region.getLoadStatistics();
        if (regionLoadStats != null) {
            responseBuilder.setRegionStatistics(MultiRegionLoadStats.newBuilder().addRegion(regionSpecifier).addStat(regionLoadStats).build());
        }
        return responseBuilder.build();
    }
    // this will contain all the cells that we need to return. It's created later, if needed.
    List<CellScannable> cellsToReturn = null;
    RegionScannersCloseCallBack closeCallBack = null;
    RpcCallContext context = RpcServer.getCurrentCall().orElse(null);
    Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request.getRegionActionCount());
    for (RegionAction regionAction : request.getRegionActionList()) {
        OperationQuota quota;
        HRegion region;
        RegionSpecifier regionSpecifier = regionAction.getRegion();
        regionActionResultBuilder.clear();
        try {
            region = getRegion(regionSpecifier);
            quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList());
        } catch (IOException e) {
            failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e);
            // For this region it's a failure.
            continue;
        }
        try {
            boolean rejectIfFromClient = shouldRejectRequestsFromClient(region);
            if (regionAction.hasCondition()) {
                // We only allow replication in standby state and it will not set the atomic flag.
                if (rejectIfFromClient) {
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                try {
                    ClientProtos.ResultOrException.Builder resultOrExceptionOrBuilder = ClientProtos.ResultOrException.newBuilder();
                    if (regionAction.getActionCount() == 1) {
                        CheckAndMutateResult result = checkAndMutate(region, quota, regionAction.getAction(0).getMutation(), cellScanner, regionAction.getCondition(), nonceGroup, spaceQuotaEnforcement);
                        regionActionResultBuilder.setProcessed(result.isSuccess());
                        resultOrExceptionOrBuilder.setIndex(0);
                        if (result.getResult() != null) {
                            resultOrExceptionOrBuilder.setResult(ProtobufUtil.toResult(result.getResult()));
                        }
                        regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                    } else {
                        CheckAndMutateResult result = checkAndMutate(region, regionAction.getActionList(), cellScanner, regionAction.getCondition(), nonceGroup, spaceQuotaEnforcement);
                        regionActionResultBuilder.setProcessed(result.isSuccess());
                        for (int i = 0; i < regionAction.getActionCount(); i++) {
                            if (i == 0 && result.getResult() != null) {
                                // Set the result of the Increment/Append operations to the first element of the
                                // ResultOrException list
                                resultOrExceptionOrBuilder.setIndex(i);
                                regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.setResult(ProtobufUtil.toResult(result.getResult())).build());
                                continue;
                            }
                            // To unify the response format with doNonAtomicRegionMutation and read through
                            // client's AsyncProcess we have to add an empty result instance per operation
                            resultOrExceptionOrBuilder.clear();
                            resultOrExceptionOrBuilder.setIndex(i);
                            regionActionResultBuilder.addResultOrException(resultOrExceptionOrBuilder.build());
                        }
                    }
                } catch (IOException e) {
                    rpcServer.getMetrics().exception(e);
                    // As it's an atomic operation with a condition, we may expect it's a global failure.
                    regionActionResultBuilder.setException(ResponseConverter.buildException(e));
                }
            } else if (regionAction.hasAtomic() && regionAction.getAtomic()) {
                // We only allow replication in standby state and it will not set the atomic flag.
                if (rejectIfFromClient) {
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                try {
                    doAtomicBatchOp(regionActionResultBuilder, region, quota, regionAction.getActionList(), cellScanner, nonceGroup, spaceQuotaEnforcement);
                    regionActionResultBuilder.setProcessed(true);
                    // We no longer use MultiResponse#processed. Instead, we use
                    // RegionActionResult#processed. This is for backward compatibility for old clients.
                    responseBuilder.setProcessed(true);
                } catch (IOException e) {
                    rpcServer.getMetrics().exception(e);
                    // As it's atomic, we may expect it's a global failure.
                    regionActionResultBuilder.setException(ResponseConverter.buildException(e));
                }
            } else {
                if (rejectIfFromClient && regionAction.getActionCount() > 0 && !isReplicationRequest(regionAction.getAction(0))) {
                    // fail if it is not a replication request
                    failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, new DoNotRetryIOException(region.getRegionInfo().getRegionNameAsString() + " is in STANDBY state"));
                    continue;
                }
                // doNonAtomicRegionMutation manages the exception internally
                if (context != null && closeCallBack == null) {
                    // An RpcCallBack that creates a list of scanners that needs to perform callBack
                    // operation on completion of multiGets.
                    // Set this only once
                    closeCallBack = new RegionScannersCloseCallBack();
                    context.setCallBack(closeCallBack);
                }
                cellsToReturn = doNonAtomicRegionMutation(region, quota, regionAction, cellScanner, regionActionResultBuilder, cellsToReturn, nonceGroup, closeCallBack, context, spaceQuotaEnforcement);
            }
        } finally {
            quota.close();
        }
        responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
        ClientProtos.RegionLoadStats regionLoadStats = region.getLoadStatistics();
        if (regionLoadStats != null) {
            regionStats.put(regionSpecifier, regionLoadStats);
        }
    }
    // Load the controller with the Cells to return.
    if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
        controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
    }
    MultiRegionLoadStats.Builder builder = MultiRegionLoadStats.newBuilder();
    for (Entry<RegionSpecifier, ClientProtos.RegionLoadStats> stat : regionStats.entrySet()) {
        builder.addRegion(stat.getKey());
        builder.addStat(stat.getValue());
    }
    responseBuilder.setRegionStatistics(builder);
    return responseBuilder.build();
}

Example 3 with ActivePolicyEnforcement

use of org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement in project hbase by apache.

the class RSRpcServices method mutate.

/**
 * Mutate data in a table.
 *
 * @param rpcc the RPC controller
 * @param request the mutate request
 */
@Override
public MutateResponse mutate(final RpcController rpcc, final MutateRequest request) throws ServiceException {
    // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
    // It is also the conduit via which we pass back data.
    HBaseRpcController controller = (HBaseRpcController) rpcc;
    CellScanner cellScanner = controller != null ? controller.cellScanner() : null;
    OperationQuota quota = null;
    RpcCallContext context = RpcServer.getCurrentCall().orElse(null);
    // Clear scanner so we are not holding on to reference across call.
    if (controller != null) {
        controller.setCellScanner(null);
    }
    try {
        checkOpen();
        requestCount.increment();
        rpcMutateRequestCount.increment();
        HRegion region = getRegion(request.getRegion());
        rejectIfInStandByState(region);
        MutateResponse.Builder builder = MutateResponse.newBuilder();
        MutationProto mutation = request.getMutation();
        if (!region.getRegionInfo().isMetaRegion()) {
            server.getMemStoreFlusher().reclaimMemStoreMemory();
        }
        long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
        quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.MUTATE);
        ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
        if (request.hasCondition()) {
            CheckAndMutateResult result = checkAndMutate(region, quota, mutation, cellScanner, request.getCondition(), nonceGroup, spaceQuotaEnforcement);
            builder.setProcessed(result.isSuccess());
            boolean clientCellBlockSupported = isClientCellBlockSupport(context);
            addResult(builder, result.getResult(), controller, clientCellBlockSupported);
            if (clientCellBlockSupported) {
                addSize(context, result.getResult(), null);
            }
        } else {
            Result r = null;
            Boolean processed = null;
            MutationType type = mutation.getMutateType();
            switch(type) {
                case APPEND:
                    // TODO: this doesn't actually check anything.
                    r = append(region, quota, mutation, cellScanner, nonceGroup, spaceQuotaEnforcement);
                    break;
                case INCREMENT:
                    // TODO: this doesn't actually check anything.
                    r = increment(region, quota, mutation, cellScanner, nonceGroup, spaceQuotaEnforcement);
                    break;
                case PUT:
                    put(region, quota, mutation, cellScanner, spaceQuotaEnforcement);
                    processed = Boolean.TRUE;
                    break;
                case DELETE:
                    delete(region, quota, mutation, cellScanner, spaceQuotaEnforcement);
                    processed = Boolean.TRUE;
                    break;
                default:
                    throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
            }
            if (processed != null) {
                builder.setProcessed(processed);
            }
            boolean clientCellBlockSupported = isClientCellBlockSupport(context);
            addResult(builder, r, controller, clientCellBlockSupported);
            if (clientCellBlockSupported) {
                addSize(context, r, null);
            }
        }
        return builder.build();
    } catch (IOException ie) {
        server.checkFileSystem();
        throw new ServiceException(ie);
    } finally {
        if (quota != null) {
            quota.close();
        }
    }
}