Examples with CommonIndexAggregation org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation used on opensource projects

Example 1 with CommonIndexAggregation

use of org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation in project geowave by locationtech.

the class AccumuloOperations method addConstraintsScanIteratorSettings.

protected <T> void addConstraintsScanIteratorSettings(final ReaderParams<T> params, final ScannerBase scanner, final DataStoreOptions options) {
    addFieldSubsettingToIterator(params, scanner);
    IteratorSetting iteratorSettings = null;
    if (params.isServersideAggregation()) {
        if (params.isMixedVisibility()) {
            iteratorSettings = new IteratorSetting(QueryFilterIterator.QUERY_ITERATOR_PRIORITY, QueryFilterIterator.QUERY_ITERATOR_NAME, WholeRowAggregationIterator.class);
        } else {
            iteratorSettings = new IteratorSetting(QueryFilterIterator.QUERY_ITERATOR_PRIORITY, QueryFilterIterator.QUERY_ITERATOR_NAME, AggregationIterator.class);
        }
        if ((params.getIndex() != null) && (params.getIndex().getIndexModel() != null)) {
            iteratorSettings.addOption(QueryFilterIterator.MODEL, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(params.getIndex().getIndexModel())));
        }
        if ((params.getIndex() != null) && (params.getIndex().getIndexStrategy() != null)) {
            iteratorSettings.addOption(QueryFilterIterator.PARTITION_KEY_LENGTH, Integer.toString(params.getIndex().getIndexStrategy().getPartitionKeyLength()));
        }
        if (!(params.getAggregation().getRight() instanceof CommonIndexAggregation) && (params.getAggregation().getLeft() != null)) {
            iteratorSettings.addOption(AggregationIterator.ADAPTER_OPTION_NAME, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(params.getAggregation().getLeft())));
            final AdapterToIndexMapping mapping = params.getAdapterIndexMappingStore().getMapping(params.getAggregation().getLeft().getAdapterId(), params.getIndex().getName());
            iteratorSettings.addOption(AggregationIterator.ADAPTER_INDEX_MAPPING_OPTION_NAME, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(mapping)));
        }
        final Aggregation aggr = params.getAggregation().getRight();
        iteratorSettings.addOption(AggregationIterator.AGGREGATION_OPTION_NAME, ByteArrayUtils.byteArrayToString(PersistenceUtils.toClassId(aggr)));
        if (aggr.getParameters() != null) {
            // sets the parameters
            iteratorSettings.addOption(AggregationIterator.PARAMETER_OPTION_NAME, ByteArrayUtils.byteArrayToString((PersistenceUtils.toBinary(aggr.getParameters()))));
        }
    }
    boolean usingDistributableFilter = false;
    if ((params.getFilter() != null) && !options.isSecondaryIndexing()) {
        usingDistributableFilter = true;
        if (iteratorSettings == null) {
            if (params.isMixedVisibility()) {
                iteratorSettings = new IteratorSetting(QueryFilterIterator.QUERY_ITERATOR_PRIORITY, QueryFilterIterator.QUERY_ITERATOR_NAME, WholeRowQueryFilterIterator.class);
            } else {
                iteratorSettings = new IteratorSetting(QueryFilterIterator.QUERY_ITERATOR_PRIORITY, QueryFilterIterator.QUERY_ITERATOR_NAME, QueryFilterIterator.class);
            }
        }
        iteratorSettings.addOption(QueryFilterIterator.FILTER, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(params.getFilter())));
        if (!iteratorSettings.getOptions().containsKey(QueryFilterIterator.MODEL)) {
            // it may already be added as an option if its an aggregation
            iteratorSettings.addOption(QueryFilterIterator.MODEL, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(params.getIndex().getIndexModel())));
            iteratorSettings.addOption(QueryFilterIterator.PARTITION_KEY_LENGTH, Integer.toString(params.getIndex().getIndexStrategy().getPartitionKeyLength()));
        }
    } else if ((iteratorSettings == null) && params.isMixedVisibility()) {
        // we have to at least use a whole row iterator
        iteratorSettings = new IteratorSetting(QueryFilterIterator.QUERY_ITERATOR_PRIORITY, QueryFilterIterator.QUERY_ITERATOR_NAME, WholeRowIterator.class);
    }
    if (!usingDistributableFilter && (!options.isSecondaryIndexing())) {
        // it ends up being duplicative and slower to add both a
        // distributable query and the index constraints, but one of the two
        // is important to limit client-side filtering
        addIndexFilterToIterator(params, scanner);
    }
    if (iteratorSettings != null) {
        scanner.addScanIterator(iteratorSettings);
    }
}

Example 2 with CommonIndexAggregation

use of org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation in project geowave by locationtech.

the class HBaseOperations method aggregateServerSide.

public <T> Iterator<GeoWaveRow> aggregateServerSide(final ReaderParams<T> readerParams) {
    final String tableName = readerParams.getIndex().getName();
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.AggregationEndpoint", options.getCoprocessorJar());
        }
        final Aggregation aggregation = readerParams.getAggregation().getRight();
        final AggregationProtosClient.AggregationType.Builder aggregationBuilder = AggregationProtosClient.AggregationType.newBuilder();
        aggregationBuilder.setClassId(ByteString.copyFrom(URLClassloaderUtils.toClassId(aggregation)));
        if (aggregation.getParameters() != null) {
            final byte[] paramBytes = URLClassloaderUtils.toBinary(aggregation.getParameters());
            aggregationBuilder.setParams(ByteString.copyFrom(paramBytes));
        }
        final AggregationProtosClient.AggregationRequest.Builder requestBuilder = AggregationProtosClient.AggregationRequest.newBuilder();
        requestBuilder.setAggregation(aggregationBuilder.build());
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList<>();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = URLClassloaderUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if (!coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(URLClassloaderUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final int maxRangeDecomposition = readerParams.getMaxRangeDecomposition() == null ? options.getAggregationMaxRangeDecomposition() : readerParams.getMaxRangeDecomposition();
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(DataStoreUtils.constraintsToQueryRanges(readerParams.getConstraints(), readerParams.getIndex(), null, maxRangeDecomposition).getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if (readerParams.getAggregation().getLeft() != null) {
            if (!(readerParams.getAggregation().getRight() instanceof CommonIndexAggregation)) {
                final byte[] adapterBytes = URLClassloaderUtils.toBinary(readerParams.getAggregation().getLeft());
                requestBuilder.setAdapter(ByteString.copyFrom(adapterBytes));
                final byte[] mappingBytes = URLClassloaderUtils.toBinary(readerParams.getAdapterIndexMappingStore().getMapping(readerParams.getAggregation().getLeft().getAdapterId(), readerParams.getIndex().getName()));
                requestBuilder.setIndexMapping(ByteString.copyFrom(mappingBytes));
            }
            requestBuilder.setInternalAdapterId(ByteString.copyFrom(ByteArrayUtils.shortToByteArray(readerParams.getAggregation().getLeft().getAdapterId())));
        }
        if ((readerParams.getAdditionalAuthorizations() != null) && (readerParams.getAdditionalAuthorizations().length > 0)) {
            requestBuilder.setVisLabels(ByteString.copyFrom(StringUtils.stringsToBinary(readerParams.getAdditionalAuthorizations())));
        }
        if (readerParams.isMixedVisibility()) {
            requestBuilder.setWholeRowFilter(true);
        }
        requestBuilder.setPartitionKeyLength(readerParams.getIndex().getIndexStrategy().getPartitionKeyLength());
        final AggregationProtosClient.AggregationRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        Map<byte[], ByteString> results = null;
        boolean shouldRetry;
        int retries = 0;
        do {
            shouldRetry = false;
            try (final Table table = getTable(tableName)) {
                results = table.coprocessorService(AggregationProtosClient.AggregationService.class, startRow, endRow, new Batch.Call<AggregationProtosClient.AggregationService, ByteString>() {

                    @Override
                    public ByteString call(final AggregationProtosClient.AggregationService counter) throws IOException {
                        final GeoWaveBlockingRpcCallback<AggregationProtosClient.AggregationResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                        counter.aggregate(null, request, rpcCallback);
                        final AggregationProtosClient.AggregationResponse response = rpcCallback.get();
                        if (response == null) {
                            // Region returned no response
                            throw new RegionException();
                        }
                        return response.hasValue() ? response.getValue() : null;
                    }
                });
                break;
            } catch (final RegionException e) {
                retries++;
                if (retries <= MAX_AGGREGATE_RETRIES) {
                    LOGGER.warn("Aggregate timed out due to unavailable region. Retrying (" + retries + " of " + MAX_AGGREGATE_RETRIES + ")");
                    shouldRetry = true;
                }
            }
        } while (shouldRetry);
        if (results == null) {
            LOGGER.error("Aggregate timed out and exceeded max retries.");
            return null;
        }
        return Iterators.transform(results.values().iterator(), b -> new GeoWaveRowImpl(null, new GeoWaveValue[] { new GeoWaveValueImpl(null, null, b.toByteArray()) }));
    } catch (final Exception e) {
        LOGGER.error("Error during aggregation.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during aggregation.", e);
    }
    return null;
}