Search in sources :

Example 11 with AggregateRequest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest in project hbase by apache.

the class AggregationClient method max.

/**
 * It gives the maximum value of a column for a given column family for the
 * given range. In case qualifier is null, a max of all values for the given
 * family is returned.
 * @param table table to scan.
 * @param ci the user's ColumnInterpreter implementation
 * @param scan the HBase scan object to use to read data from HBase
 * @return max val <>
 * @throws Throwable The caller is supposed to handle the exception as they are thrown
 *           & propagated to it.
 */
public <R, S, P extends Message, Q extends Message, T extends Message> R max(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, final Scan scan) throws Throwable {
    final AggregateRequest requestArg = validateArgAndGetPB(scan, ci, false);
    class MaxCallBack implements Batch.Callback<R> {

        R max = null;

        R getMax() {
            return max;
        }

        @Override
        public synchronized void update(byte[] region, byte[] row, R result) {
            max = (max == null || (result != null && ci.compare(max, result) < 0)) ? result : max;
        }
    }
    MaxCallBack aMaxCallBack = new MaxCallBack();
    table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, R>() {

        @Override
        public R call(AggregateService instance) throws IOException {
            RpcController controller = new AggregationClientRpcController();
            CoprocessorRpcUtils.BlockingRpcCallback<AggregateResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
            instance.getMax(controller, requestArg, rpcCallback);
            AggregateResponse response = rpcCallback.get();
            if (controller.failed()) {
                throw new IOException(controller.errorText());
            }
            if (response.getFirstPartCount() > 0) {
                ByteString b = response.getFirstPart(0);
                Q q = getParsedGenericInstance(ci.getClass(), 3, b);
                return ci.getCellValueFromProto(q);
            }
            return null;
        }
    }, aMaxCallBack);
    return aMaxCallBack.getMax();
}
Also used : AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) ByteString(org.apache.hbase.thirdparty.com.google.protobuf.ByteString) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) IOException(java.io.IOException) RpcController(org.apache.hbase.thirdparty.com.google.protobuf.RpcController) RpcCallback(org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback) CoprocessorRpcUtils(org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService)

Example 12 with AggregateRequest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest in project hbase by apache.

the class AggregationClient method getMedianArgs.

/**
 * It helps locate the region with median for a given column whose weight
 * is specified in an optional column.
 * From individual regions, it obtains sum of values and sum of weights.
 * @param table table to scan.
 * @param ci the user's ColumnInterpreter implementation
 * @param scan the HBase scan object to use to read data from HBase
 * @return pair whose first element is a map between start row of the region
 *   and (sum of values, sum of weights) for the region, the second element is
 *   (sum of values, sum of weights) for all the regions chosen
 * @throws Throwable The caller is supposed to handle the exception as they are thrown
 *           &amp; propagated to it.
 */
private <R, S, P extends Message, Q extends Message, T extends Message> Pair<NavigableMap<byte[], List<S>>, List<S>> getMedianArgs(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, final Scan scan) throws Throwable {
    final AggregateRequest requestArg = validateArgAndGetPB(scan, ci, false);
    final NavigableMap<byte[], List<S>> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    class StdCallback implements Batch.Callback<List<S>> {

        S sumVal = null, sumWeights = null;

        public synchronized Pair<NavigableMap<byte[], List<S>>, List<S>> getMedianParams() {
            List<S> l = new ArrayList<>(2);
            l.add(sumVal);
            l.add(sumWeights);
            Pair<NavigableMap<byte[], List<S>>, List<S>> p = new Pair<>(map, l);
            return p;
        }

        @Override
        public synchronized void update(byte[] region, byte[] row, List<S> result) {
            map.put(row, result);
            sumVal = ci.add(sumVal, result.get(0));
            sumWeights = ci.add(sumWeights, result.get(1));
        }
    }
    StdCallback stdCallback = new StdCallback();
    table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, List<S>>() {

        @Override
        public List<S> call(AggregateService instance) throws IOException {
            RpcController controller = new AggregationClientRpcController();
            CoprocessorRpcUtils.BlockingRpcCallback<AggregateResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
            instance.getMedian(controller, requestArg, rpcCallback);
            AggregateResponse response = rpcCallback.get();
            if (controller.failed()) {
                throw new IOException(controller.errorText());
            }
            List<S> list = new ArrayList<>();
            for (int i = 0; i < response.getFirstPartCount(); i++) {
                ByteString b = response.getFirstPart(i);
                T t = getParsedGenericInstance(ci.getClass(), 4, b);
                S s = ci.getPromotedValueFromProto(t);
                list.add(s);
            }
            return list;
        }
    }, stdCallback);
    return stdCallback.getMedianParams();
}
Also used : AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) NavigableMap(java.util.NavigableMap) ByteString(org.apache.hbase.thirdparty.com.google.protobuf.ByteString) ArrayList(java.util.ArrayList) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) ArrayList(java.util.ArrayList) List(java.util.List) Pair(org.apache.hadoop.hbase.util.Pair) IOException(java.io.IOException) TreeMap(java.util.TreeMap) RpcController(org.apache.hbase.thirdparty.com.google.protobuf.RpcController) RpcCallback(org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback) CoprocessorRpcUtils(org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService)

Example 13 with AggregateRequest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest in project hbase by apache.

the class AsyncAggregationClient method std.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<Double> std(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<Double> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<Double> callback = new AbstractAggregationCallback<Double>(future) {

        private S sum;

        private S sumSq;

        private long count;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                sum = ci.add(sum, getPromotedValueFromProto(ci, resp, 0));
                sumSq = ci.add(sumSq, getPromotedValueFromProto(ci, resp, 1));
                count += resp.getSecondPart().asReadOnlyByteBuffer().getLong();
            }
        }

        @Override
        protected Double getFinalResult() {
            double avg = ci.divideForAvg(sum, count);
            double avgSq = ci.divideForAvg(sumSq, count);
            return Math.sqrt(avgSq - avg * avg);
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getStd(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Example 14 with AggregateRequest

use of org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest in project hbase by apache.

the class AsyncAggregationClient method rowCount.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<Long> rowCount(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<Long> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, true);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<Long> callback = new AbstractAggregationCallback<Long>(future) {

        private long count;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            count += resp.getFirstPart(0).asReadOnlyByteBuffer().getLong();
        }

        @Override
        protected Long getFinalResult() {
            return count;
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getRowNum(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Aggregations

IOException (java.io.IOException)14 AggregateRequest (org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest)14 AggregateResponse (org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse)14 AggregateService (org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService)14 NavigableMap (java.util.NavigableMap)8 TreeMap (java.util.TreeMap)8 Map (java.util.Map)7 NavigableSet (java.util.NavigableSet)7 NoSuchElementException (java.util.NoSuchElementException)7 CompletableFuture (java.util.concurrent.CompletableFuture)7 Cell (org.apache.hadoop.hbase.Cell)7 HConstants (org.apache.hadoop.hbase.HConstants)7 AdvancedScanResultConsumer (org.apache.hadoop.hbase.client.AdvancedScanResultConsumer)7 AsyncTable (org.apache.hadoop.hbase.client.AsyncTable)7 CoprocessorCallback (org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback)7 RegionInfo (org.apache.hadoop.hbase.client.RegionInfo)7 Result (org.apache.hadoop.hbase.client.Result)7 Scan (org.apache.hadoop.hbase.client.Scan)7 AggregationHelper.getParsedGenericInstance (org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance)7 AggregationHelper.validateArgAndGetPB (org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB)7