Search in sources :

Example 1 with SpaceViolationPolicyEnforcement

use of org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement in project hbase by apache.

the class RSRpcServices method bulkLoadHFile.

/**
 * Atomically bulk load several HFiles into an open region
 * @return true if successful, false is failed but recoverably (no action)
 * @throws ServiceException if failed unrecoverably
 */
@Override
public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException {
    long start = EnvironmentEdgeManager.currentTime();
    List<String> clusterIds = new ArrayList<>(request.getClusterIdsList());
    if (clusterIds.contains(this.server.getClusterId())) {
        return BulkLoadHFileResponse.newBuilder().setLoaded(true).build();
    } else {
        clusterIds.add(this.server.getClusterId());
    }
    try {
        checkOpen();
        requestCount.increment();
        HRegion region = getRegion(request.getRegion());
        final boolean spaceQuotaEnabled = QuotaUtil.isQuotaEnabled(getConfiguration());
        long sizeToBeLoaded = -1;
        // Check to see if this bulk load would exceed the space quota for this table
        if (spaceQuotaEnabled) {
            ActivePolicyEnforcement activeSpaceQuotas = getSpaceQuotaManager().getActiveEnforcements();
            SpaceViolationPolicyEnforcement enforcement = activeSpaceQuotas.getPolicyEnforcement(region);
            if (enforcement != null) {
                // Bulk loads must still be atomic. We must enact all or none.
                List<String> filePaths = new ArrayList<>(request.getFamilyPathCount());
                for (FamilyPath familyPath : request.getFamilyPathList()) {
                    filePaths.add(familyPath.getPath());
                }
                // Check if the batch of files exceeds the current quota
                sizeToBeLoaded = enforcement.computeBulkLoadSize(getFileSystem(filePaths), filePaths);
            }
        }
        // secure bulk load
        Map<byte[], List<Path>> map = server.getSecureBulkLoadManager().secureBulkLoadHFiles(region, request, clusterIds);
        BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
        builder.setLoaded(map != null);
        if (map != null) {
            // not possible to occur in real life (cannot bulk load a file with negative size)
            if (spaceQuotaEnabled && sizeToBeLoaded > 0) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Incrementing space use of " + region.getRegionInfo() + " by " + sizeToBeLoaded + " bytes");
                }
                // Inform space quotas of the new files for this region
                getSpaceQuotaManager().getRegionSizeStore().incrementRegionSize(region.getRegionInfo(), sizeToBeLoaded);
            }
        }
        return builder.build();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    } finally {
        final MetricsRegionServer metricsRegionServer = server.getMetrics();
        if (metricsRegionServer != null) {
            metricsRegionServer.updateBulkLoad(EnvironmentEdgeManager.currentTime() - start);
        }
    }
}
Also used : ActivePolicyEnforcement(org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement) FamilyPath(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath) ArrayList(java.util.ArrayList) ByteString(org.apache.hbase.thirdparty.com.google.protobuf.ByteString) IOException(java.io.IOException) DoNotRetryIOException(org.apache.hadoop.hbase.DoNotRetryIOException) HBaseIOException(org.apache.hadoop.hbase.HBaseIOException) UncheckedIOException(java.io.UncheckedIOException) SpaceViolationPolicyEnforcement(org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement) ServiceException(org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) BulkLoadHFileResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileResponse) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList)

Aggregations

IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 DoNotRetryIOException (org.apache.hadoop.hbase.DoNotRetryIOException)1 HBaseIOException (org.apache.hadoop.hbase.HBaseIOException)1 ActivePolicyEnforcement (org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement)1 SpaceViolationPolicyEnforcement (org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement)1 FamilyPath (org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath)1 BulkLoadHFileResponse (org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileResponse)1 ImmutableList (org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList)1 ByteString (org.apache.hbase.thirdparty.com.google.protobuf.ByteString)1 ServiceException (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException)1