Search in sources :

Example 46 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class AsyncAggregationClient method min.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<R> min(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<R> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<R> callback = new AbstractAggregationCallback<R>(future) {

        private R min;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                R result = getCellValueFromProto(ci, resp, 0);
                if (min == null || (result != null && ci.compare(min, result) > 0)) {
                    min = result;
                }
            }
        }

        @Override
        protected R getFinalResult() {
            return min;
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getMin(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Example 47 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class AsyncAggregationClient method sumByRegion.

// the map key is the startRow of the region
private static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<NavigableMap<byte[], S>> sumByRegion(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<NavigableMap<byte[], S>> future = new CompletableFuture<NavigableMap<byte[], S>>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    int firstPartIndex = scan.getFamilyMap().get(scan.getFamilies()[0]).size() - 1;
    AbstractAggregationCallback<NavigableMap<byte[], S>> callback = new AbstractAggregationCallback<NavigableMap<byte[], S>>(future) {

        private final NavigableMap<byte[], S> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                map.put(region.getStartKey(), getPromotedValueFromProto(ci, resp, firstPartIndex));
            }
        }

        @Override
        protected NavigableMap<byte[], S> getFinalResult() {
            return map;
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getMedian(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) NavigableMap(java.util.NavigableMap) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException) CompletableFuture(java.util.concurrent.CompletableFuture)

Example 48 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class AsyncAggregationClient method max.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<R> max(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<R> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<R> callback = new AbstractAggregationCallback<R>(future) {

        private R max;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                R result = getCellValueFromProto(ci, resp, 0);
                if (max == null || (result != null && ci.compare(max, result) < 0)) {
                    max = result;
                }
            }
        }

        @Override
        protected R getFinalResult() {
            return max;
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getMax(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Example 49 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class AsyncAggregationClient method avg.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<Double> avg(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<Double> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<Double> callback = new AbstractAggregationCallback<Double>(future) {

        private S sum;

        long count = 0L;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                sum = ci.add(sum, getPromotedValueFromProto(ci, resp, 0));
                count += resp.getSecondPart().asReadOnlyByteBuffer().getLong();
            }
        }

        @Override
        protected Double getFinalResult() {
            return ci.divideForAvg(sum, count);
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getAvg(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Example 50 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class AsyncAggregationClient method sum.

public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<S> sum(AsyncTable<?> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<S> future = new CompletableFuture<>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    AbstractAggregationCallback<S> callback = new AbstractAggregationCallback<S>(future) {

        private S sum;

        @Override
        protected void aggregate(RegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                S s = getPromotedValueFromProto(ci, resp, 0);
                sum = ci.add(sum, s);
            }
        }

        @Override
        protected S getFinalResult() {
            return sum;
        }
    };
    table.<AggregateService, AggregateResponse>coprocessorService(AggregateService::newStub, (stub, controller, rpcCallback) -> stub.getSum(controller, req, rpcCallback), callback).fromRow(nullToEmpty(scan.getStartRow()), scan.includeStartRow()).toRow(nullToEmpty(scan.getStopRow()), scan.includeStopRow()).execute();
    return future;
}
Also used : AdvancedScanResultConsumer(org.apache.hadoop.hbase.client.AdvancedScanResultConsumer) FutureUtils.addListener(org.apache.hadoop.hbase.util.FutureUtils.addListener) CoprocessorCallback(org.apache.hadoop.hbase.client.AsyncTable.CoprocessorCallback) ColumnInterpreter(org.apache.hadoop.hbase.coprocessor.ColumnInterpreter) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) Result(org.apache.hadoop.hbase.client.Result) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) Message(org.apache.hbase.thirdparty.com.google.protobuf.Message) HConstants(org.apache.hadoop.hbase.HConstants) Map(java.util.Map) AggregationHelper.validateArgAndGetPB(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.validateArgAndGetPB) NoSuchElementException(java.util.NoSuchElementException) Cell(org.apache.hadoop.hbase.Cell) Bytes(org.apache.hadoop.hbase.util.Bytes) ReflectionUtils(org.apache.hadoop.hbase.util.ReflectionUtils) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) NavigableMap(java.util.NavigableMap) AggregateService(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateService) Scan(org.apache.hadoop.hbase.client.Scan) InterfaceAudience(org.apache.yetus.audience.InterfaceAudience) TreeMap(java.util.TreeMap) AggregationHelper.getParsedGenericInstance(org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance) AsyncTable(org.apache.hadoop.hbase.client.AsyncTable) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) CompletableFuture(java.util.concurrent.CompletableFuture) AggregateRequest(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateRequest) AggregateResponse(org.apache.hadoop.hbase.shaded.protobuf.generated.AggregateProtos.AggregateResponse) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) IOException(java.io.IOException)

Aggregations

RegionInfo (org.apache.hadoop.hbase.client.RegionInfo)824 Test (org.junit.Test)416 TableName (org.apache.hadoop.hbase.TableName)311 ServerName (org.apache.hadoop.hbase.ServerName)191 ArrayList (java.util.ArrayList)175 IOException (java.io.IOException)174 TableDescriptor (org.apache.hadoop.hbase.client.TableDescriptor)174 Path (org.apache.hadoop.fs.Path)141 List (java.util.List)118 HashMap (java.util.HashMap)90 Table (org.apache.hadoop.hbase.client.Table)90 Map (java.util.Map)81 Put (org.apache.hadoop.hbase.client.Put)81 Configuration (org.apache.hadoop.conf.Configuration)80 HRegion (org.apache.hadoop.hbase.regionserver.HRegion)67 TreeMap (java.util.TreeMap)66 Result (org.apache.hadoop.hbase.client.Result)59 FileSystem (org.apache.hadoop.fs.FileSystem)58 Cell (org.apache.hadoop.hbase.Cell)50 Scan (org.apache.hadoop.hbase.client.Scan)46