Examples with HBaseNumericIndexStrategyFilter org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter used on opensource projects

Example 1 with HBaseNumericIndexStrategyFilter

use of org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter in project geowave by locationtech.

the class HBaseOperations method bulkDelete.

public void bulkDelete(final ReaderParams readerParams) {
    final String tableName = readerParams.getIndex().getName();
    final short[] adapterIds = readerParams.getAdapterIds();
    Long total = 0L;
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.HBaseBulkDeleteEndpoint", options.getCoprocessorJar());
        }
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest.Builder requestBuilder = HBaseBulkDeleteProtosClient.BulkDeleteRequest.newBuilder();
        requestBuilder.setDeleteType(DELETE_TYPE);
        requestBuilder.setRowBatchSize(DELETE_BATCH_SIZE);
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = PersistenceUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if ((coords != null) && !coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(PersistenceUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(readerParams.getQueryRanges().getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if ((adapterIds != null) && (adapterIds.length > 0)) {
            final ByteBuffer buf = ByteBuffer.allocate(2 * adapterIds.length);
            for (final Short a : adapterIds) {
                buf.putShort(a);
            }
            requestBuilder.setAdapterIds(ByteString.copyFrom(buf.array()));
        }
        final Table table = getTable(tableName);
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        final Map<byte[], Long> results = table.coprocessorService(HBaseBulkDeleteProtosClient.BulkDeleteService.class, startRow, endRow, new Batch.Call<HBaseBulkDeleteProtosClient.BulkDeleteService, Long>() {

            @Override
            public Long call(final HBaseBulkDeleteProtosClient.BulkDeleteService counter) throws IOException {
                final GeoWaveBlockingRpcCallback<HBaseBulkDeleteProtosClient.BulkDeleteResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                counter.delete(null, request, rpcCallback);
                final BulkDeleteResponse response = rpcCallback.get();
                return response.hasRowsDeleted() ? response.getRowsDeleted() : null;
            }
        });
        int regionCount = 0;
        for (final Map.Entry<byte[], Long> entry : results.entrySet()) {
            regionCount++;
            final Long value = entry.getValue();
            if (value != null) {
                LOGGER.debug("Value from region " + regionCount + " is " + value);
                total += value;
            } else {
                LOGGER.debug("Empty response for region " + regionCount);
            }
        }
    } catch (final Exception e) {
        LOGGER.error("Error during bulk delete.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during bulkdelete.", e);
    }
}

Example 2 with HBaseNumericIndexStrategyFilter

use of org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter in project geowave by locationtech.

the class AggregationEndpoint method aggregate.

@Override
public void aggregate(final RpcController controller, final AggregationProtosServer.AggregationRequest request, final RpcCallback<AggregationProtosServer.AggregationResponse> done) {
    FilterList filterList = null;
    InternalDataAdapter<?> dataAdapter = null;
    AdapterToIndexMapping indexMapping = null;
    Short internalAdapterId = null;
    AggregationProtosServer.AggregationResponse response = null;
    ByteString value = ByteString.EMPTY;
    // Get the aggregation type
    final Aggregation aggregation = (Aggregation) URLClassloaderUtils.fromClassId(request.getAggregation().getClassId().toByteArray());
    // Handle aggregation params
    if (request.getAggregation().hasParams()) {
        final byte[] parameterBytes = request.getAggregation().getParams().toByteArray();
        final Persistable aggregationParams = URLClassloaderUtils.fromBinary(parameterBytes);
        aggregation.setParameters(aggregationParams);
    }
    HBaseDistributableFilter hdFilter = null;
    if (aggregation != null) {
        if (request.hasRangeFilter()) {
            final byte[] rfilterBytes = request.getRangeFilter().toByteArray();
            try {
                final MultiRowRangeFilter rangeFilter = MultiRowRangeFilter.parseFrom(rfilterBytes);
                filterList = new FilterList(rangeFilter);
            } catch (final Exception e) {
                LOGGER.error("Error creating range filter.", e);
            }
        } else {
            LOGGER.error("Input range filter is undefined.");
        }
        if (request.hasNumericIndexStrategyFilter()) {
            final byte[] nisFilterBytes = request.getNumericIndexStrategyFilter().toByteArray();
            try {
                final HBaseNumericIndexStrategyFilter numericIndexStrategyFilter = HBaseNumericIndexStrategyFilter.parseFrom(nisFilterBytes);
                if (filterList == null) {
                    filterList = new FilterList(numericIndexStrategyFilter);
                } else {
                    filterList.addFilter(numericIndexStrategyFilter);
                }
            } catch (final Exception e) {
                LOGGER.error("Error creating index strategy filter.", e);
            }
        }
        try {
            // in the filter list for the dedupe filter to work correctly
            if (request.hasModel()) {
                hdFilter = new HBaseDistributableFilter();
                if (request.hasWholeRowFilter()) {
                    hdFilter.setWholeRowFilter(request.getWholeRowFilter());
                }
                if (request.hasPartitionKeyLength()) {
                    hdFilter.setPartitionKeyLength(request.getPartitionKeyLength());
                }
                final byte[] filterBytes;
                if (request.hasFilter()) {
                    filterBytes = request.getFilter().toByteArray();
                } else {
                    filterBytes = null;
                }
                final byte[] modelBytes = request.getModel().toByteArray();
                if (hdFilter.init(filterBytes, modelBytes)) {
                    if (filterList == null) {
                        filterList = new FilterList(hdFilter);
                    } else {
                        filterList.addFilter(hdFilter);
                    }
                } else {
                    LOGGER.error("Error creating distributable filter.");
                }
            } else {
                LOGGER.error("Input distributable filter is undefined.");
            }
        } catch (final Exception e) {
            LOGGER.error("Error creating distributable filter.", e);
        }
        if (request.hasAdapter()) {
            final byte[] adapterBytes = request.getAdapter().toByteArray();
            dataAdapter = (InternalDataAdapter<?>) URLClassloaderUtils.fromBinary(adapterBytes);
        }
        if (request.hasInternalAdapterId()) {
            final byte[] adapterIdBytes = request.getInternalAdapterId().toByteArray();
            internalAdapterId = ByteArrayUtils.byteArrayToShort(adapterIdBytes);
        }
        if (request.hasIndexMapping()) {
            final byte[] mappingBytes = request.getIndexMapping().toByteArray();
            indexMapping = (AdapterToIndexMapping) URLClassloaderUtils.fromBinary(mappingBytes);
        }
        final String[] authorizations;
        if (request.hasVisLabels()) {
            final byte[] visBytes = request.getVisLabels().toByteArray();
            if (visBytes.length > 0) {
                authorizations = StringUtils.stringsFromBinary(visBytes);
            } else {
                authorizations = null;
            }
        } else {
            authorizations = null;
        }
        try {
            final Object result = getValue(aggregation, filterList, dataAdapter, indexMapping, internalAdapterId, hdFilter, request.getBlockCaching(), request.getCacheSize(), authorizations);
            URLClassloaderUtils.initClassLoader();
            final byte[] bvalue = aggregation.resultToBinary(result);
            value = ByteString.copyFrom(bvalue);
        } catch (final IOException ioe) {
            LOGGER.error("Error during aggregation.", ioe);
        /*
         * ResponseConverter.setControllerException( controller, ioe);
         */
        } catch (final Exception e) {
            LOGGER.error("Error during aggregation.", e);
        }
    }
    response = AggregationProtosServer.AggregationResponse.newBuilder().setValue(value).build();
    done.run(response);
}

Example 3 with HBaseNumericIndexStrategyFilter

use of org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter in project geowave by locationtech.

the class HBaseBulkDeleteEndpoint method delete.

@Override
public void delete(final RpcController controller, final BulkDeleteRequest request, final RpcCallback<BulkDeleteResponse> done) {
    long totalRowsDeleted = 0L;
    long totalVersionsDeleted = 0L;
    FilterList filterList = null;
    final List<byte[]> adapterIds = new ArrayList<>();
    Long timestamp = null;
    if (request.hasTimestamp()) {
        timestamp = request.getTimestamp();
    }
    final BulkDeleteType deleteType = request.getDeleteType();
    /**
     * Extract the filter from the bulkDeleteRequest
     */
    HBaseDistributableFilter hdFilter = null;
    if (request.hasRangeFilter()) {
        final byte[] rfilterBytes = request.getRangeFilter().toByteArray();
        try {
            final MultiRowRangeFilter rangeFilter = MultiRowRangeFilter.parseFrom(rfilterBytes);
            filterList = new FilterList(rangeFilter);
        } catch (final Exception e) {
            LOGGER.error("Error creating range filter.", e);
        }
    } else {
        LOGGER.error("Input range filter is undefined.");
    }
    if (request.hasNumericIndexStrategyFilter()) {
        final byte[] nisFilterBytes = request.getNumericIndexStrategyFilter().toByteArray();
        try {
            final HBaseNumericIndexStrategyFilter numericIndexStrategyFilter = HBaseNumericIndexStrategyFilter.parseFrom(nisFilterBytes);
            if (filterList == null) {
                filterList = new FilterList(numericIndexStrategyFilter);
            } else {
                filterList.addFilter(numericIndexStrategyFilter);
            }
        } catch (final Exception e) {
            LOGGER.error("Error creating index strategy filter.", e);
        }
    }
    try {
        // in the filter list for the dedupe filter to work correctly
        if (request.hasModel()) {
            hdFilter = new HBaseDistributableFilter();
            final byte[] filterBytes;
            if (request.hasFilter()) {
                filterBytes = request.getFilter().toByteArray();
            } else {
                filterBytes = null;
            }
            final byte[] modelBytes = request.getModel().toByteArray();
            if (hdFilter.init(filterBytes, modelBytes)) {
                if (filterList == null) {
                    filterList = new FilterList(hdFilter);
                } else {
                    filterList.addFilter(hdFilter);
                }
            } else {
                LOGGER.error("Error creating distributable filter.");
            }
        } else {
            LOGGER.error("Input distributable filter is undefined.");
        }
    } catch (final Exception e) {
        LOGGER.error("Error creating distributable filter.", e);
    }
    if (request.hasAdapterIds()) {
        final ByteBuffer buf = ByteBuffer.wrap(request.getAdapterIds().toByteArray());
        adapterIds.clear();
        while (buf.hasRemaining()) {
            final short adapterId = buf.getShort();
            adapterIds.add(StringUtils.stringToBinary(ByteArrayUtils.shortToString(adapterId)));
        }
    }
    /**
     * Start the actual delete process
     */
    RegionScanner scanner = null;
    try {
        scanner = null;
        final Scan scan = new Scan();
        scan.setFilter(filterList);
        if (!adapterIds.isEmpty()) {
            for (final byte[] adapterId : adapterIds) {
                scan.addFamily(adapterId);
            }
        }
        final Region region = env.getRegion();
        scanner = region.getScanner(scan);
        boolean hasMore = true;
        final int rowBatchSize = request.getRowBatchSize();
        while (hasMore) {
            final List<List<Cell>> deleteRows = new ArrayList<>(rowBatchSize);
            for (int i = 0; i < rowBatchSize; i++) {
                final List<Cell> results = new ArrayList<>();
                hasMore = scanner.next(results);
                if (results.size() > 0) {
                    deleteRows.add(results);
                }
                if (!hasMore) {
                    // There are no more rows.
                    break;
                }
            }
            if (deleteRows.size() > 0) {
                final Mutation[] deleteArr = new Mutation[deleteRows.size()];
                int i = 0;
                for (final List<Cell> deleteRow : deleteRows) {
                    deleteArr[i++] = createDeleteMutation(deleteRow, deleteType, timestamp);
                }
                final OperationStatus[] opStatus = batchMutate(region, deleteArr);
                for (i = 0; i < opStatus.length; i++) {
                    if (opStatus[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {
                        break;
                    }
                    totalRowsDeleted++;
                    if (deleteType == BulkDeleteType.VERSION) {
                        final byte[] versionsDeleted = deleteArr[i].getAttribute(NO_OF_VERSIONS_TO_DELETE);
                        if (versionsDeleted != null) {
                            totalVersionsDeleted += Bytes.toInt(versionsDeleted);
                        }
                    }
                }
            }
        }
    } catch (final IOException e) {
        LOGGER.error("Unable to delete rows", e);
    } finally {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (final IOException ioe) {
                LOGGER.error("Error during bulk delete in HBase.", ioe);
                ;
            }
        }
    }
    final Builder responseBuilder = BulkDeleteResponse.newBuilder();
    responseBuilder.setRowsDeleted(totalRowsDeleted);
    if (deleteType == BulkDeleteType.VERSION) {
        responseBuilder.setVersionsDeleted(totalVersionsDeleted);
    }
    // Send the response back
    final BulkDeleteResponse response = responseBuilder.build();
    done.run(response);
}

Example 4 with HBaseNumericIndexStrategyFilter

use of org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter in project geowave by locationtech.

the class HBaseOperations method aggregateServerSide.

public <T> Iterator<GeoWaveRow> aggregateServerSide(final ReaderParams<T> readerParams) {
    final String tableName = readerParams.getIndex().getName();
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.AggregationEndpoint", options.getCoprocessorJar());
        }
        final Aggregation aggregation = readerParams.getAggregation().getRight();
        final AggregationProtosClient.AggregationType.Builder aggregationBuilder = AggregationProtosClient.AggregationType.newBuilder();
        aggregationBuilder.setClassId(ByteString.copyFrom(URLClassloaderUtils.toClassId(aggregation)));
        if (aggregation.getParameters() != null) {
            final byte[] paramBytes = URLClassloaderUtils.toBinary(aggregation.getParameters());
            aggregationBuilder.setParams(ByteString.copyFrom(paramBytes));
        }
        final AggregationProtosClient.AggregationRequest.Builder requestBuilder = AggregationProtosClient.AggregationRequest.newBuilder();
        requestBuilder.setAggregation(aggregationBuilder.build());
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList<>();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = URLClassloaderUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if (!coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(URLClassloaderUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final int maxRangeDecomposition = readerParams.getMaxRangeDecomposition() == null ? options.getAggregationMaxRangeDecomposition() : readerParams.getMaxRangeDecomposition();
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(DataStoreUtils.constraintsToQueryRanges(readerParams.getConstraints(), readerParams.getIndex(), null, maxRangeDecomposition).getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if (readerParams.getAggregation().getLeft() != null) {
            if (!(readerParams.getAggregation().getRight() instanceof CommonIndexAggregation)) {
                final byte[] adapterBytes = URLClassloaderUtils.toBinary(readerParams.getAggregation().getLeft());
                requestBuilder.setAdapter(ByteString.copyFrom(adapterBytes));
                final byte[] mappingBytes = URLClassloaderUtils.toBinary(readerParams.getAdapterIndexMappingStore().getMapping(readerParams.getAggregation().getLeft().getAdapterId(), readerParams.getIndex().getName()));
                requestBuilder.setIndexMapping(ByteString.copyFrom(mappingBytes));
            }
            requestBuilder.setInternalAdapterId(ByteString.copyFrom(ByteArrayUtils.shortToByteArray(readerParams.getAggregation().getLeft().getAdapterId())));
        }
        if ((readerParams.getAdditionalAuthorizations() != null) && (readerParams.getAdditionalAuthorizations().length > 0)) {
            requestBuilder.setVisLabels(ByteString.copyFrom(StringUtils.stringsToBinary(readerParams.getAdditionalAuthorizations())));
        }
        if (readerParams.isMixedVisibility()) {
            requestBuilder.setWholeRowFilter(true);
        }
        requestBuilder.setPartitionKeyLength(readerParams.getIndex().getIndexStrategy().getPartitionKeyLength());
        final AggregationProtosClient.AggregationRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        Map<byte[], ByteString> results = null;
        boolean shouldRetry;
        int retries = 0;
        do {
            shouldRetry = false;
            try (final Table table = getTable(tableName)) {
                results = table.coprocessorService(AggregationProtosClient.AggregationService.class, startRow, endRow, new Batch.Call<AggregationProtosClient.AggregationService, ByteString>() {

                    @Override
                    public ByteString call(final AggregationProtosClient.AggregationService counter) throws IOException {
                        final GeoWaveBlockingRpcCallback<AggregationProtosClient.AggregationResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                        counter.aggregate(null, request, rpcCallback);
                        final AggregationProtosClient.AggregationResponse response = rpcCallback.get();
                        if (response == null) {
                            // Region returned no response
                            throw new RegionException();
                        }
                        return response.hasValue() ? response.getValue() : null;
                    }
                });
                break;
            } catch (final RegionException e) {
                retries++;
                if (retries <= MAX_AGGREGATE_RETRIES) {
                    LOGGER.warn("Aggregate timed out due to unavailable region. Retrying (" + retries + " of " + MAX_AGGREGATE_RETRIES + ")");
                    shouldRetry = true;
                }
            }
        } while (shouldRetry);
        if (results == null) {
            LOGGER.error("Aggregate timed out and exceeded max retries.");
            return null;
        }
        return Iterators.transform(results.values().iterator(), b -> new GeoWaveRowImpl(null, new GeoWaveValue[] { new GeoWaveValueImpl(null, null, b.toByteArray()) }));
    } catch (final Exception e) {
        LOGGER.error("Error during aggregation.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during aggregation.", e);
    }
    return null;
}

Example 5 with HBaseNumericIndexStrategyFilter

use of org.locationtech.geowave.datastore.hbase.filters.HBaseNumericIndexStrategyFilter in project geowave by locationtech.

the class HBaseReader method addIndexFilter.

private void addIndexFilter(final ReaderParams<T> params, final FilterList filterList) {
    final List<MultiDimensionalCoordinateRangesArray> coords = params.getCoordinateRanges();
    if ((coords != null) && !coords.isEmpty()) {
        final HBaseNumericIndexStrategyFilter numericIndexFilter = new HBaseNumericIndexStrategyFilter(params.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {}));
        filterList.addFilter(numericIndexFilter);
    }
}