Examples with MultiDimensionalCoordinateRangesArray org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray used on opensource projects

Example 1 with MultiDimensionalCoordinateRangesArray

use of org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray in project geowave by locationtech.

the class AccumuloOperations method addIndexFilterToIterator.

protected <T> void addIndexFilterToIterator(final ReaderParams<T> params, final ScannerBase scanner) {
    final List<MultiDimensionalCoordinateRangesArray> coords = params.getCoordinateRanges();
    if ((coords != null) && !coords.isEmpty()) {
        final IteratorSetting iteratorSetting = new IteratorSetting(NumericIndexStrategyFilterIterator.IDX_FILTER_ITERATOR_PRIORITY, NumericIndexStrategyFilterIterator.IDX_FILTER_ITERATOR_NAME, NumericIndexStrategyFilterIterator.class);
        iteratorSetting.addOption(NumericIndexStrategyFilterIterator.INDEX_STRATEGY_KEY, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(params.getIndex().getIndexStrategy())));
        iteratorSetting.addOption(NumericIndexStrategyFilterIterator.COORDINATE_RANGE_KEY, ByteArrayUtils.byteArrayToString(new ArrayOfArrays(coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toBinary()));
        scanner.addScanIterator(iteratorSetting);
    }
}

Example 2 with MultiDimensionalCoordinateRangesArray

use of org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray in project geowave by locationtech.

the class BaseConstraintsQuery method getCoordinateRanges.

@Override
public List<MultiDimensionalCoordinateRangesArray> getCoordinateRanges() {
    if ((constraints == null) || constraints.isEmpty()) {
        return new ArrayList<>();
    } else {
        final NumericIndexStrategy indexStrategy = index.getIndexStrategy();
        final List<MultiDimensionalCoordinateRangesArray> ranges = new ArrayList<>();
        for (final MultiDimensionalNumericData nd : constraints) {
            final MultiDimensionalCoordinateRanges[] indexStrategyCoordRanges = indexStrategy.getCoordinateRangesPerDimension(nd, indexMetaData);
            if (indexStrategyCoordRanges != null) {
                ranges.add(new MultiDimensionalCoordinateRangesArray(indexStrategyCoordRanges));
            }
        }
        return ranges;
    }
}

Example 3 with MultiDimensionalCoordinateRangesArray

use of org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray in project geowave by locationtech.

the class HBaseOperations method bulkDelete.

public void bulkDelete(final ReaderParams readerParams) {
    final String tableName = readerParams.getIndex().getName();
    final short[] adapterIds = readerParams.getAdapterIds();
    Long total = 0L;
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.HBaseBulkDeleteEndpoint", options.getCoprocessorJar());
        }
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest.Builder requestBuilder = HBaseBulkDeleteProtosClient.BulkDeleteRequest.newBuilder();
        requestBuilder.setDeleteType(DELETE_TYPE);
        requestBuilder.setRowBatchSize(DELETE_BATCH_SIZE);
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = PersistenceUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if ((coords != null) && !coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(PersistenceUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(readerParams.getQueryRanges().getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if ((adapterIds != null) && (adapterIds.length > 0)) {
            final ByteBuffer buf = ByteBuffer.allocate(2 * adapterIds.length);
            for (final Short a : adapterIds) {
                buf.putShort(a);
            }
            requestBuilder.setAdapterIds(ByteString.copyFrom(buf.array()));
        }
        final Table table = getTable(tableName);
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        final Map<byte[], Long> results = table.coprocessorService(HBaseBulkDeleteProtosClient.BulkDeleteService.class, startRow, endRow, new Batch.Call<HBaseBulkDeleteProtosClient.BulkDeleteService, Long>() {

            @Override
            public Long call(final HBaseBulkDeleteProtosClient.BulkDeleteService counter) throws IOException {
                final GeoWaveBlockingRpcCallback<HBaseBulkDeleteProtosClient.BulkDeleteResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                counter.delete(null, request, rpcCallback);
                final BulkDeleteResponse response = rpcCallback.get();
                return response.hasRowsDeleted() ? response.getRowsDeleted() : null;
            }
        });
        int regionCount = 0;
        for (final Map.Entry<byte[], Long> entry : results.entrySet()) {
            regionCount++;
            final Long value = entry.getValue();
            if (value != null) {
                LOGGER.debug("Value from region " + regionCount + " is " + value);
                total += value;
            } else {
                LOGGER.debug("Empty response for region " + regionCount);
            }
        }
    } catch (final Exception e) {
        LOGGER.error("Error during bulk delete.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during bulkdelete.", e);
    }
}

Example 4 with MultiDimensionalCoordinateRangesArray

use of org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray in project geowave by locationtech.

the class HBaseNumericIndexStrategyFilter method parseFrom.

public static HBaseNumericIndexStrategyFilter parseFrom(final byte[] pbBytes) throws DeserializationException {
    final ByteBuffer buf = ByteBuffer.wrap(pbBytes);
    NumericIndexStrategy indexStrategy;
    MultiDimensionalCoordinateRangesArray[] coordinateRanges;
    try {
        final int indexStrategyLength = VarintUtils.readUnsignedInt(buf);
        final byte[] indexStrategyBytes = new byte[indexStrategyLength];
        buf.get(indexStrategyBytes);
        indexStrategy = (NumericIndexStrategy) URLClassloaderUtils.fromBinary(indexStrategyBytes);
        final byte[] coordRangeBytes = new byte[buf.remaining()];
        buf.get(coordRangeBytes);
        final ArrayOfArrays arrays = new ArrayOfArrays();
        arrays.fromBinary(coordRangeBytes);
        coordinateRanges = arrays.getCoordinateArrays();
    } catch (final Exception e) {
        throw new DeserializationException("Unable to read parameters", e);
    }
    return new HBaseNumericIndexStrategyFilter(indexStrategy, coordinateRanges);
}

Example 5 with MultiDimensionalCoordinateRangesArray

use of org.locationtech.geowave.core.index.MultiDimensionalCoordinateRangesArray in project geowave by locationtech.

the class HBaseOperations method aggregateServerSide.

public <T> Iterator<GeoWaveRow> aggregateServerSide(final ReaderParams<T> readerParams) {
    final String tableName = readerParams.getIndex().getName();
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.AggregationEndpoint", options.getCoprocessorJar());
        }
        final Aggregation aggregation = readerParams.getAggregation().getRight();
        final AggregationProtosClient.AggregationType.Builder aggregationBuilder = AggregationProtosClient.AggregationType.newBuilder();
        aggregationBuilder.setClassId(ByteString.copyFrom(URLClassloaderUtils.toClassId(aggregation)));
        if (aggregation.getParameters() != null) {
            final byte[] paramBytes = URLClassloaderUtils.toBinary(aggregation.getParameters());
            aggregationBuilder.setParams(ByteString.copyFrom(paramBytes));
        }
        final AggregationProtosClient.AggregationRequest.Builder requestBuilder = AggregationProtosClient.AggregationRequest.newBuilder();
        requestBuilder.setAggregation(aggregationBuilder.build());
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList<>();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = URLClassloaderUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if (!coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(URLClassloaderUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final int maxRangeDecomposition = readerParams.getMaxRangeDecomposition() == null ? options.getAggregationMaxRangeDecomposition() : readerParams.getMaxRangeDecomposition();
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(DataStoreUtils.constraintsToQueryRanges(readerParams.getConstraints(), readerParams.getIndex(), null, maxRangeDecomposition).getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if (readerParams.getAggregation().getLeft() != null) {
            if (!(readerParams.getAggregation().getRight() instanceof CommonIndexAggregation)) {
                final byte[] adapterBytes = URLClassloaderUtils.toBinary(readerParams.getAggregation().getLeft());
                requestBuilder.setAdapter(ByteString.copyFrom(adapterBytes));
                final byte[] mappingBytes = URLClassloaderUtils.toBinary(readerParams.getAdapterIndexMappingStore().getMapping(readerParams.getAggregation().getLeft().getAdapterId(), readerParams.getIndex().getName()));
                requestBuilder.setIndexMapping(ByteString.copyFrom(mappingBytes));
            }
            requestBuilder.setInternalAdapterId(ByteString.copyFrom(ByteArrayUtils.shortToByteArray(readerParams.getAggregation().getLeft().getAdapterId())));
        }
        if ((readerParams.getAdditionalAuthorizations() != null) && (readerParams.getAdditionalAuthorizations().length > 0)) {
            requestBuilder.setVisLabels(ByteString.copyFrom(StringUtils.stringsToBinary(readerParams.getAdditionalAuthorizations())));
        }
        if (readerParams.isMixedVisibility()) {
            requestBuilder.setWholeRowFilter(true);
        }
        requestBuilder.setPartitionKeyLength(readerParams.getIndex().getIndexStrategy().getPartitionKeyLength());
        final AggregationProtosClient.AggregationRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        Map<byte[], ByteString> results = null;
        boolean shouldRetry;
        int retries = 0;
        do {
            shouldRetry = false;
            try (final Table table = getTable(tableName)) {
                results = table.coprocessorService(AggregationProtosClient.AggregationService.class, startRow, endRow, new Batch.Call<AggregationProtosClient.AggregationService, ByteString>() {

                    @Override
                    public ByteString call(final AggregationProtosClient.AggregationService counter) throws IOException {
                        final GeoWaveBlockingRpcCallback<AggregationProtosClient.AggregationResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                        counter.aggregate(null, request, rpcCallback);
                        final AggregationProtosClient.AggregationResponse response = rpcCallback.get();
                        if (response == null) {
                            // Region returned no response
                            throw new RegionException();
                        }
                        return response.hasValue() ? response.getValue() : null;
                    }
                });
                break;
            } catch (final RegionException e) {
                retries++;
                if (retries <= MAX_AGGREGATE_RETRIES) {
                    LOGGER.warn("Aggregate timed out due to unavailable region. Retrying (" + retries + " of " + MAX_AGGREGATE_RETRIES + ")");
                    shouldRetry = true;
                }
            }
        } while (shouldRetry);
        if (results == null) {
            LOGGER.error("Aggregate timed out and exceeded max retries.");
            return null;
        }
        return Iterators.transform(results.values().iterator(), b -> new GeoWaveRowImpl(null, new GeoWaveValue[] { new GeoWaveValueImpl(null, null, b.toByteArray()) }));
    } catch (final Exception e) {
        LOGGER.error("Error during aggregation.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during aggregation.", e);
    }
    return null;
}