Examples with AdapterAndIndexBasedQueryConstraints org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints used on opensource projects

Example 1 with AdapterAndIndexBasedQueryConstraints

use of org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints in project geowave by locationtech.

the class SplitsProvider method getSplits.

/**
 * Read the metadata table to get tablets and match up ranges to them.
 */
public List<InputSplit> getSplits(final DataStoreOperations operations, final CommonQueryOptions commonOptions, final DataTypeQueryOptions<?> typeOptions, final IndexQueryOptions indexOptions, final QueryConstraints constraints, final TransientAdapterStore adapterStore, final DataStatisticsStore statsStore, final InternalAdapterStore internalAdapterStore, final IndexStore indexStore, final AdapterIndexMappingStore adapterIndexMappingStore, final JobContext context, final Integer minSplits, final Integer maxSplits) throws IOException, InterruptedException {
    final Map<Pair<Index, ByteArray>, RowRangeHistogramValue> statsCache = new HashMap<>();
    final List<InputSplit> retVal = new ArrayList<>();
    final TreeSet<IntermediateSplitInfo> splits = new TreeSet<>();
    final Map<String, List<Short>> indexIdToAdaptersMap = new HashMap<>();
    for (final Pair<Index, List<Short>> indexAdapterIdPair : BaseDataStoreUtils.getAdaptersWithMinimalSetOfIndices(typeOptions.getTypeNames(), indexOptions.getIndexName(), adapterStore, internalAdapterStore, adapterIndexMappingStore, indexStore, constraints)) {
        QueryConstraints indexAdapterConstraints;
        if (constraints instanceof AdapterAndIndexBasedQueryConstraints) {
            final List<Short> adapters = indexAdapterIdPair.getRight();
            DataTypeAdapter<?> adapter = null;
            // types/adapters
            if (adapters.size() == 1) {
                final String typeName = internalAdapterStore.getTypeName(adapters.get(0));
                if (typeName != null) {
                    adapter = adapterStore.getAdapter(typeName);
                }
            }
            if (adapter == null) {
                indexAdapterConstraints = constraints;
                LOGGER.info("Unable to find type matching an adapter dependent query");
            } else {
                indexAdapterConstraints = ((AdapterAndIndexBasedQueryConstraints) constraints).createQueryConstraints(adapter.asInternalAdapter(adapters.get(0)), indexAdapterIdPair.getLeft(), adapterIndexMappingStore.getMapping(adapters.get(0), indexAdapterIdPair.getLeft().getName()));
                if (indexAdapterConstraints == null) {
                    continue;
                }
                // make sure we pass along the new constraints to the record
                // reader - for spark on YARN (not localy though), job
                // configuration is immutable so while picking up the
                // appropriate constraint from the configuration is more
                // efficient, also do a check for
                // AdapterAndIndexBasedQueryConstraints within the Record Reader
                // itself
                GeoWaveInputFormat.setQueryConstraints(context.getConfiguration(), indexAdapterConstraints);
            }
        } else {
            indexAdapterConstraints = constraints;
        }
        indexIdToAdaptersMap.put(indexAdapterIdPair.getKey().getName(), indexAdapterIdPair.getValue());
        IndexMetaData[] indexMetadata = null;
        if (indexAdapterConstraints != null) {
            final IndexMetaDataSetValue statValue = InternalStatisticsHelper.getIndexMetadata(indexAdapterIdPair.getLeft(), indexAdapterIdPair.getRight(), new AdapterStoreWrapper(adapterStore, internalAdapterStore), statsStore, commonOptions.getAuthorizations());
            if (statValue != null) {
                indexMetadata = statValue.toArray();
            }
        }
        populateIntermediateSplits(splits, operations, indexAdapterIdPair.getLeft(), indexAdapterIdPair.getValue(), statsCache, adapterStore, internalAdapterStore, statsStore, maxSplits, indexAdapterConstraints, (double[]) commonOptions.getHints().get(DataStoreUtils.TARGET_RESOLUTION_PER_DIMENSION_FOR_HIERARCHICAL_INDEX), indexMetadata, commonOptions.getAuthorizations());
    }
    // this is an incremental algorithm, it may be better use the target
    // split count to drive it (ie. to get 3 splits this will split 1
    // large
    // range into two down the middle and then split one of those ranges
    // down the middle to get 3, rather than splitting one range into
    // thirds)
    final List<IntermediateSplitInfo> unsplittable = new ArrayList<>();
    if (!statsCache.isEmpty() && !splits.isEmpty() && (minSplits != null) && (splits.size() < minSplits)) {
        // set the ranges to at least min splits
        do {
            // remove the highest range, split it into 2 and add both
            // back,
            // increasing the size by 1
            final IntermediateSplitInfo highestSplit = splits.pollLast();
            final IntermediateSplitInfo otherSplit = highestSplit.split(statsCache);
            // working our way up the split set.
            if (otherSplit == null) {
                unsplittable.add(highestSplit);
            } else {
                splits.add(highestSplit);
                splits.add(otherSplit);
            }
        } while ((splits.size() != 0) && ((splits.size() + unsplittable.size()) < minSplits));
        // Add all unsplittable splits back to splits array
        splits.addAll(unsplittable);
        if (splits.size() < minSplits) {
            LOGGER.warn("Truly unable to meet split count. Actual Count: " + splits.size());
        }
    } else if (((maxSplits != null) && (maxSplits > 0)) && (splits.size() > maxSplits)) {
        // merge splits to fit within max splits
        do {
            // this is the naive approach, remove the lowest two ranges
            // and merge them, decreasing the size by 1
            // TODO Ideally merge takes into account locations (as well
            // as possibly the index as a secondary criteria) to limit
            // the number of locations/indices
            final IntermediateSplitInfo lowestSplit = splits.pollFirst();
            final IntermediateSplitInfo nextLowestSplit = splits.pollFirst();
            lowestSplit.merge(nextLowestSplit);
            splits.add(lowestSplit);
        } while (splits.size() > maxSplits);
    }
    for (final IntermediateSplitInfo split : splits) {
        retVal.add(split.toFinalSplit(statsStore, adapterStore, internalAdapterStore, indexIdToAdaptersMap, commonOptions.getAuthorizations()));
    }
    return retVal;
}

Example 2 with AdapterAndIndexBasedQueryConstraints

use of org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints in project geowave by locationtech.

the class GeoWaveRecordReader method initialize.

/**
 * Initialize a scanner over the given input split using this task attempt configuration.
 */
@Override
public void initialize(final InputSplit inSplit, final TaskAttemptContext attempt) throws IOException {
    split = (GeoWaveInputSplit) inSplit;
    numKeysRead = 0;
    final Set<String> indices = split.getIndexNames();
    final BigDecimal sum = BigDecimal.ZERO;
    final Map<RangeLocationPair, BigDecimal> incrementalRangeSums = new LinkedHashMap<>();
    final List<CloseableIterator<Pair<GeoWaveInputKey, T>>> allIterators = new ArrayList<>();
    final NextRangeCallback callback = new InternalCallback();
    final short[] adapters;
    // the splits provider was unable to set it
    if (constraints instanceof AdapterAndIndexBasedQueryConstraints) {
        adapters = sanitizedQueryOptions.getAdapterIds(internalAdapterStore);
    } else {
        adapters = null;
    }
    for (final String i : indices) {
        final SplitInfo splitInfo = split.getInfo(i);
        List<QueryFilter> queryFilters = null;
        if (constraints != null) {
            // AdapterAndIndexBasedQueryConstraints
            if (adapters != null) {
                InternalDataAdapter<?> adapter = null;
                if (adapters.length > 1) {
                    // iterator of results per adapter
                    for (final short adapterId : adapters) {
                        final String typeName = internalAdapterStore.getTypeName(adapterId);
                        if (typeName != null) {
                            final DataTypeAdapter<?> baseAdapter = adapterStore.getAdapter(typeName);
                            if (baseAdapter != null) {
                                adapter = baseAdapter.asInternalAdapter(adapterId);
                            }
                        }
                        if (adapter == null) {
                            LOGGER.warn("Unable to find type matching an adapter dependent query");
                        }
                        queryFilters = ((AdapterAndIndexBasedQueryConstraints) constraints).createQueryConstraints(adapter, splitInfo.getIndex(), aimStore.getMapping(adapterId, splitInfo.getIndex().getName())).createFilters(splitInfo.getIndex());
                        sanitizedQueryOptions.setAdapterId(adapterId);
                        fillIterators(allIterators, splitInfo, queryFilters, sum, incrementalRangeSums, callback);
                    }
                    continue;
                }
                // multiple types/adapters
                if (adapters.length == 1) {
                    final String typeName = internalAdapterStore.getTypeName(adapters[0]);
                    if (typeName != null) {
                        final DataTypeAdapter<?> baseAdapter = adapterStore.getAdapter(typeName);
                        if (baseAdapter != null) {
                            adapter = baseAdapter.asInternalAdapter(adapters[0]);
                        }
                    }
                }
                if (adapter == null) {
                    LOGGER.warn("Unable to find type matching an adapter dependent query");
                }
                final QueryConstraints tempConstraints = ((AdapterAndIndexBasedQueryConstraints) constraints).createQueryConstraints(adapter, splitInfo.getIndex(), adapter != null ? aimStore.getMapping(adapter.getAdapterId(), splitInfo.getIndex().getName()) : null);
                if (tempConstraints == null) {
                    LOGGER.warn("Adapter and Index based constraints not satisfied for adapter '" + adapter.getTypeName() + "'");
                    continue;
                } else {
                    constraints = tempConstraints;
                }
            }
            queryFilters = constraints.createFilters(splitInfo.getIndex());
        }
        fillIterators(allIterators, splitInfo, queryFilters, sum, incrementalRangeSums, callback);
    }
    // finally we can compute percent progress
    progressPerRange = new LinkedHashMap<>();
    RangeLocationPair prevRangeIndex = null;
    float prevProgress = 0f;
    if (sum.compareTo(BigDecimal.ZERO) > 0) {
        try {
            for (final Entry<RangeLocationPair, BigDecimal> entry : incrementalRangeSums.entrySet()) {
                final BigDecimal value = entry.getValue();
                final float progress = value.divide(sum, RoundingMode.HALF_UP).floatValue();
                if (prevRangeIndex != null) {
                    progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, progress));
                }
                prevRangeIndex = entry.getKey();
                prevProgress = progress;
            }
            progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, 1f));
        } catch (final Exception e) {
            LOGGER.warn("Unable to calculate progress", e);
        }
    }
    // concatenate iterators
    iterator = new CloseableIteratorWrapper<>(new Closeable() {

        @Override
        public void close() throws IOException {
            for (final CloseableIterator<Pair<GeoWaveInputKey, T>> reader : allIterators) {
                reader.close();
            }
        }
    }, Iterators.concat(allIterators.iterator()));
}

Example 3 with AdapterAndIndexBasedQueryConstraints

use of org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints in project geowave by locationtech.

the class BaseDataStore method internalQuery.

protected <T> CloseableIterator<T> internalQuery(final QueryConstraints constraints, final BaseQueryOptions queryOptions, final DeletionMode deleteMode) {
    // Note: The DeletionMode option is provided to avoid recursively
    // adding DuplicateDeletionCallbacks when actual duplicates are removed
    // via the DuplicateDeletionCallback. The callback should only be added
    // during the initial deletion query.
    final boolean delete = ((deleteMode == DeletionMode.DELETE) || (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES));
    final List<CloseableIterator<Object>> results = new ArrayList<>();
    // If CQL filter is set
    if (constraints instanceof TypeConstraintQuery) {
        final String constraintTypeName = ((TypeConstraintQuery) constraints).getTypeName();
        if ((queryOptions.getAdapterIds() == null) || (queryOptions.getAdapterIds().length == 0)) {
            queryOptions.setAdapterId(internalAdapterStore.getAdapterId(constraintTypeName));
        } else if (queryOptions.getAdapterIds().length == 1) {
            final Short adapterId = internalAdapterStore.getAdapterId(constraintTypeName);
            if ((adapterId == null) || (queryOptions.getAdapterIds()[0] != adapterId.shortValue())) {
                LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
                throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
            }
        } else {
            // Throw exception when QueryOptions has more than one adapter
            // and CQL Adapter is set.
            LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
            throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
        }
    }
    final QueryConstraints sanitizedConstraints = (constraints == null) ? new EverythingQuery() : constraints;
    final List<DataStoreCallbackManager> deleteCallbacks = new ArrayList<>();
    final Map<Short, Set<ByteArray>> dataIdsToDelete;
    if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && (baseOptions.isSecondaryIndexing())) {
        dataIdsToDelete = new ConcurrentHashMap<>();
    } else {
        dataIdsToDelete = null;
    }
    final boolean dataIdIndexIsBest = baseOptions.isSecondaryIndexing() && ((sanitizedConstraints instanceof DataIdQuery) || (sanitizedConstraints instanceof DataIdRangeQuery) || (sanitizedConstraints instanceof EverythingQuery));
    if (!delete && dataIdIndexIsBest) {
        try {
            // just grab the values directly from the Data Index
            InternalDataAdapter<?>[] adapters = queryOptions.getAdaptersArray(adapterStore);
            if (!queryOptions.isAllIndices()) {
                final Set<Short> adapterIds = new HashSet<>(Arrays.asList(ArrayUtils.toObject(queryOptions.getValidAdapterIds(internalAdapterStore, indexMappingStore))));
                adapters = Arrays.stream(adapters).filter(a -> adapterIds.contains(a.getAdapterId())).toArray(i -> new InternalDataAdapter<?>[i]);
            }
            // TODO test whether aggregations work in this case
            for (final InternalDataAdapter<?> adapter : adapters) {
                RowReader<GeoWaveRow> rowReader;
                if (sanitizedConstraints instanceof DataIdQuery) {
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
                } else if (sanitizedConstraints instanceof DataIdRangeQuery) {
                    if (((DataIdRangeQuery) sanitizedConstraints).isReverse() && !isReverseIterationSupported()) {
                        throw new UnsupportedOperationException("Currently the underlying datastore does not support reverse iteration");
                    }
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).isReverse());
                } else {
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId());
                }
                results.add(new CloseableIteratorWrapper(rowReader, new NativeEntryIteratorWrapper(adapterStore, indexMappingStore, DataIndexUtils.DATA_ID_INDEX, rowReader, null, queryOptions.getScanCallback(), BaseDataStoreUtils.getFieldBitmask(queryOptions.getFieldIdsAdapterPair(), DataIndexUtils.DATA_ID_INDEX), queryOptions.getMaxResolutionSubsamplingPerDimension(), !BaseDataStoreUtils.isCommonIndexAggregation(queryOptions.getAggregation()), null)));
            }
            if (BaseDataStoreUtils.isAggregation(queryOptions.getAggregation())) {
                return BaseDataStoreUtils.aggregate(new CloseableIteratorWrapper(new Closeable() {

                    @Override
                    public void close() throws IOException {
                        for (final CloseableIterator<Object> result : results) {
                            result.close();
                        }
                    }
                }, Iterators.concat(results.iterator())), (Aggregation) queryOptions.getAggregation().getRight(), (DataTypeAdapter) queryOptions.getAggregation().getLeft());
            }
        } catch (final IOException e1) {
            LOGGER.error("Failed to resolve adapter or index for query", e1);
        }
    } else {
        final boolean isConstraintsAdapterIndexSpecific = sanitizedConstraints instanceof AdapterAndIndexBasedQueryConstraints;
        final boolean isAggregationAdapterIndexSpecific = (queryOptions.getAggregation() != null) && (queryOptions.getAggregation().getRight() instanceof AdapterAndIndexBasedAggregation);
        // all queries will use the same instance of the dedupe filter for
        // client side filtering because the filter needs to be applied across
        // indices
        DedupeFilter dedupeFilter = new DedupeFilter();
        MemoryPersistentAdapterStore tempAdapterStore = new MemoryPersistentAdapterStore(queryOptions.getAdaptersArray(adapterStore));
        MemoryAdapterIndexMappingStore memoryMappingStore = new MemoryAdapterIndexMappingStore();
        // keep a list of adapters that have been queried, to only load an
        // adapter to be queried once
        final Set<Short> queriedAdapters = new HashSet<>();
        // if its an ordered constraints then it is dependent on the index selected, if its
        // secondary indexing its inefficient to delete by constraints
        final boolean deleteAllIndicesByConstraints = ((delete && ((constraints == null) || !constraints.indexMustBeSpecified()) && !baseOptions.isSecondaryIndexing()));
        final List<Pair<Index, List<InternalDataAdapter<?>>>> indexAdapterPairList = (deleteAllIndicesByConstraints) ? queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore) : queryOptions.getBestQueryIndices(tempAdapterStore, indexMappingStore, indexStore, statisticsStore, sanitizedConstraints);
        Map<Short, List<Index>> additionalIndicesToDelete = null;
        if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && !deleteAllIndicesByConstraints) {
            additionalIndicesToDelete = new HashMap<>();
            // we have to make sure to delete from the other indices if they exist
            final List<Pair<Index, List<InternalDataAdapter<?>>>> allIndices = queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore);
            for (final Pair<Index, List<InternalDataAdapter<?>>> allPair : allIndices) {
                for (final Pair<Index, List<InternalDataAdapter<?>>> constraintPair : indexAdapterPairList) {
                    if (((constraintPair.getKey() == null) && (allPair.getKey() == null)) || constraintPair.getKey().equals(allPair.getKey())) {
                        allPair.getRight().removeAll(constraintPair.getRight());
                        break;
                    }
                }
                for (final InternalDataAdapter<?> adapter : allPair.getRight()) {
                    List<Index> indices = additionalIndicesToDelete.get(adapter.getAdapterId());
                    if (indices == null) {
                        indices = new ArrayList<>();
                        additionalIndicesToDelete.put(adapter.getAdapterId(), indices);
                    }
                    indices.add(allPair.getLeft());
                }
            }
        }
        final Pair<InternalDataAdapter<?>, Aggregation<?, ?, ?>> aggregation = queryOptions.getAggregation();
        final ScanCallback callback = queryOptions.getScanCallback();
        for (final Pair<Index, List<InternalDataAdapter<?>>> indexAdapterPair : indexAdapterPairList) {
            if (indexAdapterPair.getKey() == null) {
                // queries
                if (dataIdIndexIsBest) {
                    // prior logic for !delete
                    for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
                        // this must be a data index only adapter, just worry about updating statistics and
                        // not other indices or duplicates
                        ScanCallback scanCallback = callback;
                        if (baseOptions.isPersistDataStatistics()) {
                            final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
                            deleteCallbacks.add(callbackCache);
                            scanCallback = new ScanCallback<Object, GeoWaveRow>() {

                                @Override
                                public void entryScanned(final Object entry, final GeoWaveRow row) {
                                    if (callback != null) {
                                        callback.entryScanned(entry, row);
                                    }
                                    callbackCache.getDeleteCallback(adapter, null, null).entryDeleted(entry, row);
                                }
                            };
                        }
                        if (sanitizedConstraints instanceof DataIdQuery) {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
                        } else if (sanitizedConstraints instanceof DataIdRangeQuery) {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive());
                        } else {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId());
                        }
                    }
                } else {
                    final String[] typeNames = indexAdapterPair.getRight().stream().map(a -> a.getAdapter().getTypeName()).toArray(k -> new String[k]);
                    LOGGER.warn("Data types '" + ArrayUtils.toString(typeNames) + "' do not have an index that satisfies the query");
                }
                continue;
            }
            final List<Short> adapterIdsToQuery = new ArrayList<>();
            // this only needs to be done once per index, not once per
            // adapter
            boolean queriedAllAdaptersByPrefix = false;
            // maintain a set of data IDs if deleting using secondary indexing
            for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
                final Index index = indexAdapterPair.getLeft();
                final AdapterToIndexMapping indexMapping = indexMappingStore.getMapping(adapter.getAdapterId(), index.getName());
                memoryMappingStore.addAdapterIndexMapping(indexMapping);
                if (delete) {
                    final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
                    // want the stats to change
                    if (!(constraints instanceof InsertionIdQuery)) {
                        callbackCache.setPersistStats(baseOptions.isPersistDataStatistics());
                    } else {
                        callbackCache.setPersistStats(false);
                    }
                    deleteCallbacks.add(callbackCache);
                    if (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES) {
                        final DeleteCallbackList<T, GeoWaveRow> delList = (DeleteCallbackList<T, GeoWaveRow>) callbackCache.getDeleteCallback(adapter, indexMapping, index);
                        final DuplicateDeletionCallback<T> dupDeletionCallback = new DuplicateDeletionCallback<>(this, adapter, indexMapping, index);
                        delList.addCallback(dupDeletionCallback);
                        if ((additionalIndicesToDelete != null) && (additionalIndicesToDelete.get(adapter.getAdapterId()) != null)) {
                            delList.addCallback(new DeleteOtherIndicesCallback<>(baseOperations, adapter, additionalIndicesToDelete.get(adapter.getAdapterId()), adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getAuthorizations()));
                        }
                    }
                    final Map<Short, Set<ByteArray>> internalDataIdsToDelete = dataIdsToDelete;
                    queryOptions.setScanCallback(new ScanCallback<Object, GeoWaveRow>() {

                        @Override
                        public void entryScanned(final Object entry, final GeoWaveRow row) {
                            if (callback != null) {
                                callback.entryScanned(entry, row);
                            }
                            if (internalDataIdsToDelete != null) {
                                final ByteArray dataId = new ByteArray(row.getDataId());
                                Set<ByteArray> currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
                                if (currentDataIdsToDelete == null) {
                                    synchronized (internalDataIdsToDelete) {
                                        currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
                                        if (currentDataIdsToDelete == null) {
                                            currentDataIdsToDelete = Sets.newConcurrentHashSet();
                                            internalDataIdsToDelete.put(row.getAdapterId(), currentDataIdsToDelete);
                                        }
                                    }
                                }
                                currentDataIdsToDelete.add(dataId);
                            }
                            callbackCache.getDeleteCallback(adapter, indexMapping, index).entryDeleted(entry, row);
                        }
                    });
                }
                QueryConstraints adapterIndexConstraints;
                if (isConstraintsAdapterIndexSpecific) {
                    adapterIndexConstraints = ((AdapterAndIndexBasedQueryConstraints) sanitizedConstraints).createQueryConstraints(adapter, indexAdapterPair.getLeft(), indexMapping);
                    if (adapterIndexConstraints == null) {
                        continue;
                    }
                } else {
                    adapterIndexConstraints = sanitizedConstraints;
                }
                if (isAggregationAdapterIndexSpecific) {
                    queryOptions.setAggregation(((AdapterAndIndexBasedAggregation) aggregation.getRight()).createAggregation(adapter, indexMapping, index), aggregation.getLeft());
                }
                if (adapterIndexConstraints instanceof InsertionIdQuery) {
                    queryOptions.setLimit(-1);
                    results.add(queryInsertionId(adapter, index, (InsertionIdQuery) adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, delete));
                    continue;
                } else if (adapterIndexConstraints instanceof PrefixIdQuery) {
                    if (!queriedAllAdaptersByPrefix) {
                        final PrefixIdQuery prefixIdQuery = (PrefixIdQuery) adapterIndexConstraints;
                        results.add(queryRowPrefix(index, prefixIdQuery.getPartitionKey(), prefixIdQuery.getSortKeyPrefix(), queryOptions, indexAdapterPair.getRight(), tempAdapterStore, delete));
                        queriedAllAdaptersByPrefix = true;
                    }
                    continue;
                } else if (isConstraintsAdapterIndexSpecific || isAggregationAdapterIndexSpecific) {
                    // can't query multiple adapters in the same scan
                    results.add(queryConstraints(Collections.singletonList(adapter.getAdapterId()), index, adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
                    continue;
                }
                // finally just add it to a list to query multiple adapters
                // in on scan
                adapterIdsToQuery.add(adapter.getAdapterId());
            }
            // in one query instance (one scanner) for efficiency
            if (adapterIdsToQuery.size() > 0) {
                results.add(queryConstraints(adapterIdsToQuery, indexAdapterPair.getLeft(), sanitizedConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
            }
            if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode)) {
                // Make sure each index query has a clean dedupe filter so that entries from other indices
                // get deleted
                dedupeFilter = new DedupeFilter();
            }
        }
    }
    return new CloseableIteratorWrapper<>(new Closeable() {

        @Override
        public void close() throws IOException {
            for (final CloseableIterator<Object> result : results) {
                result.close();
            }
            for (final DataStoreCallbackManager c : deleteCallbacks) {
                c.close();
            }
            if ((dataIdsToDelete != null) && !dataIdsToDelete.isEmpty()) {
                if (baseOptions.isSecondaryIndexing()) {
                    deleteFromDataIndex(dataIdsToDelete, queryOptions.getAuthorizations());
                }
            }
        }
    }, Iterators.concat(new CastIterator<T>(results.iterator())));
}

Example 4 with AdapterAndIndexBasedQueryConstraints

use of org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints in project geowave by locationtech.

the class BaseDataStoreUtils method chooseBestIndex.

public static Index chooseBestIndex(final Index[] indices, final QueryConstraints query, final InternalDataAdapter<?> adapter, final AdapterIndexMappingStore mappingStore) {
    final boolean isConstraintsAdapterIndexSpecific = query instanceof AdapterAndIndexBasedQueryConstraints;
    Index nextIdx = null;
    int i = 0;
    double bestIndexBitsUsed = -1;
    int bestIndexDimensionCount = -1;
    Index bestIdx = null;
    while (i < indices.length) {
        nextIdx = indices[i++];
        if ((nextIdx == null) || (nextIdx.getIndexStrategy() == null) || (nextIdx.getIndexStrategy().getOrderedDimensionDefinitions() == null) || (nextIdx.getIndexStrategy().getOrderedDimensionDefinitions().length == 0)) {
            continue;
        }
        QueryConstraints adapterIndexConstraints;
        if (isConstraintsAdapterIndexSpecific) {
            adapterIndexConstraints = ((AdapterAndIndexBasedQueryConstraints) query).createQueryConstraints(adapter, nextIdx, mappingStore.getMapping(adapter.getAdapterId(), nextIdx.getName()));
            if (adapterIndexConstraints == null) {
                continue;
            }
        } else {
            adapterIndexConstraints = query;
        }
        final List<MultiDimensionalNumericData> queryRanges = adapterIndexConstraints.getIndexConstraints(nextIdx);
        final int currentDimensionCount = nextIdx.getIndexStrategy().getOrderedDimensionDefinitions().length;
        if (IndexUtils.isFullTableScan(queryRanges) || !queryRangeDimensionsMatch(currentDimensionCount, queryRanges)) {
            // result in a full table scan
            if (bestIdx == null) {
                bestIdx = nextIdx;
            }
        } else {
            double currentBitsUsed = 0;
            if (currentDimensionCount >= bestIndexDimensionCount) {
                for (final MultiDimensionalNumericData qr : queryRanges) {
                    final double[] dataRangePerDimension = new double[qr.getDimensionCount()];
                    for (int d = 0; d < dataRangePerDimension.length; d++) {
                        dataRangePerDimension[d] = qr.getMaxValuesPerDimension()[d] - qr.getMinValuesPerDimension()[d];
                    }
                    currentBitsUsed += IndexUtils.getDimensionalBitsUsed(nextIdx.getIndexStrategy(), dataRangePerDimension);
                }
                if ((currentDimensionCount > bestIndexDimensionCount) || (currentBitsUsed > bestIndexBitsUsed)) {
                    bestIndexBitsUsed = currentBitsUsed;
                    bestIndexDimensionCount = currentDimensionCount;
                    bestIdx = nextIdx;
                }
            }
        }
    }
    if ((bestIdx == null) && (indices.length > 0)) {
        bestIdx = indices[0];
    }
    return bestIdx;
}