Examples with AggregateResponse org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse used on opensource projects

Example 1 with AggregateResponse

use of org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse in project hbase by apache.

the class AggregationClient method getAvgArgs.

/**
   * It computes average while fetching sum and row count from all the
   * corresponding regions. Approach is to compute a global sum of region level
   * sum and rowcount and then compute the average.
   * @param table
   * @param scan
   * @throws Throwable
   */
private <R, S, P extends Message, Q extends Message, T extends Message> Pair<S, Long> getAvgArgs(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, final Scan scan) throws Throwable {
    final AggregateRequest requestArg = validateArgAndGetPB(scan, ci, false);
    class AvgCallBack implements Batch.Callback<Pair<S, Long>> {

        S sum = null;

        Long rowCount = 0l;

        public synchronized Pair<S, Long> getAvgArgs() {
            return new Pair<>(sum, rowCount);
        }

        @Override
        public synchronized void update(byte[] region, byte[] row, Pair<S, Long> result) {
            sum = ci.add(sum, result.getFirst());
            rowCount += result.getSecond();
        }
    }
    AvgCallBack avgCallBack = new AvgCallBack();
    table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, Pair<S, Long>>() {

        @Override
        public Pair<S, Long> call(AggregateService instance) throws IOException {
            RpcController controller = new AggregationClientRpcController();
            CoprocessorRpcUtils.BlockingRpcCallback<AggregateResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
            instance.getAvg(controller, requestArg, rpcCallback);
            AggregateResponse response = rpcCallback.get();
            if (controller.failed()) {
                throw new IOException(controller.errorText());
            }
            Pair<S, Long> pair = new Pair<>(null, 0L);
            if (response.getFirstPartCount() == 0) {
                return pair;
            }
            ByteString b = response.getFirstPart(0);
            T t = getParsedGenericInstance(ci.getClass(), 4, b);
            S s = ci.getPromotedValueFromProto(t);
            pair.setFirst(s);
            ByteBuffer bb = ByteBuffer.allocate(8).put(getBytesFromResponse(response.getSecondPart()));
            bb.rewind();
            pair.setSecond(bb.getLong());
            return pair;
        }
    }, avgCallBack);
    return avgCallBack.getAvgArgs();
}

Example 2 with AggregateResponse

use of org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse in project hbase by apache.

the class AggregationClient method rowCount.

/**
   * It gives the row count, by summing up the individual results obtained from
   * regions. In case the qualifier is null, FirstKeyValueFilter is used to
   * optimised the operation. In case qualifier is provided, I can't use the
   * filter as it may set the flag to skip to next row, but the value read is
   * not of the given filter: in this case, this particular row will not be
   * counted ==> an error.
   * @param table
   * @param ci
   * @param scan
   * @return <R, S>
   * @throws Throwable
   */
public <R, S, P extends Message, Q extends Message, T extends Message> long rowCount(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, final Scan scan) throws Throwable {
    final AggregateRequest requestArg = validateArgAndGetPB(scan, ci, true);
    class RowNumCallback implements Batch.Callback<Long> {

        private final AtomicLong rowCountL = new AtomicLong(0);

        public long getRowNumCount() {
            return rowCountL.get();
        }

        @Override
        public void update(byte[] region, byte[] row, Long result) {
            rowCountL.addAndGet(result.longValue());
        }
    }
    RowNumCallback rowNum = new RowNumCallback();
    table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, Long>() {

        @Override
        public Long call(AggregateService instance) throws IOException {
            RpcController controller = new AggregationClientRpcController();
            CoprocessorRpcUtils.BlockingRpcCallback<AggregateResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
            instance.getRowNum(controller, requestArg, rpcCallback);
            AggregateResponse response = rpcCallback.get();
            if (controller.failed()) {
                throw new IOException(controller.errorText());
            }
            byte[] bytes = getBytesFromResponse(response.getFirstPart(0));
            ByteBuffer bb = ByteBuffer.allocate(8).put(bytes);
            bb.rewind();
            return bb.getLong();
        }
    }, rowNum);
    return rowNum.getRowNumCount();
}

Example 3 with AggregateResponse

use of org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse in project hbase by apache.

the class AsyncAggregationClient method std.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<Double> std(RawAsyncTable table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<Double> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<Double> callback = new AbstractAggregationCallback<Double>(future) {

        private S sum;

        private S sumSq;

        private long count;

        @Override
        protected void aggregate(HRegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                sum = ci.add(sum, getPromotedValueFromProto(ci, resp, 0));
                sumSq = ci.add(sumSq, getPromotedValueFromProto(ci, resp, 1));
                count += resp.getSecondPart().asReadOnlyByteBuffer().getLong();
            }
        }

        @Override
        protected Double getFinalResult() {
            double avg = ci.divideForAvg(sum, count);
            double avgSq = ci.divideForAvg(sumSq, count);
            return Math.sqrt(avgSq - avg * avg);
        }
    };
    table.coprocessorService(channel -> AggregateService.newStub(channel), (stub, controller, rpcCallback) -> stub.getStd(controller, req, rpcCallback), scan.getStartRow(), scan.includeStartRow(), scan.getStopRow(), scan.includeStopRow(), callback);
    return future;
}

Example 4 with AggregateResponse

use of org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse in project hbase by apache.

the class AsyncAggregationClient method avg.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<Double> avg(RawAsyncTable table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<Double> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<Double> callback = new AbstractAggregationCallback<Double>(future) {

        private S sum;

        long count = 0L;

        @Override
        protected void aggregate(HRegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                sum = ci.add(sum, getPromotedValueFromProto(ci, resp, 0));
                count += resp.getSecondPart().asReadOnlyByteBuffer().getLong();
            }
        }

        @Override
        protected Double getFinalResult() {
            return ci.divideForAvg(sum, count);
        }
    };
    table.coprocessorService(channel -> AggregateService.newStub(channel), (stub, controller, rpcCallback) -> stub.getAvg(controller, req, rpcCallback), scan.getStartRow(), scan.includeStartRow(), scan.getStopRow(), scan.includeStopRow(), callback);
    return future;
}

Example 5 with AggregateResponse

use of org.apache.hadoop.hbase.protobuf.generated.AggregateProtos.AggregateResponse in project hbase by apache.

the class AggregateImplementation method getRowNum.

/**
   * Gives the row count for the given column family and column qualifier, in
   * the given row range as defined in the Scan object.
   */
@Override
public void getRowNum(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) {
    AggregateResponse response = null;
    long counter = 0l;
    List<Cell> results = new ArrayList<>();
    InternalScanner scanner = null;
    try {
        Scan scan = ProtobufUtil.toScan(request.getScan());
        byte[][] colFamilies = scan.getFamilies();
        byte[] colFamily = colFamilies != null ? colFamilies[0] : null;
        NavigableSet<byte[]> qualifiers = colFamilies != null ? scan.getFamilyMap().get(colFamily) : null;
        byte[] qualifier = null;
        if (qualifiers != null && !qualifiers.isEmpty()) {
            qualifier = qualifiers.pollFirst();
        }
        if (scan.getFilter() == null && qualifier == null)
            scan.setFilter(new FirstKeyOnlyFilter());
        scanner = env.getRegion().getScanner(scan);
        boolean hasMoreRows = false;
        do {
            hasMoreRows = scanner.next(results);
            if (results.size() > 0) {
                counter++;
            }
            results.clear();
        } while (hasMoreRows);
        ByteBuffer bb = ByteBuffer.allocate(8).putLong(counter);
        bb.rewind();
        response = AggregateResponse.newBuilder().addFirstPart(ByteString.copyFrom(bb)).build();
    } catch (IOException e) {
        CoprocessorRpcUtils.setControllerException(controller, e);
    } finally {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (IOException ignored) {
            }
        }
    }
    log.info("Row counter from this region is " + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + counter);
    done.run(response);
}