use of org.locationtech.geowave.core.store.callback.DeleteCallbackList in project geowave by locationtech.
the class BaseDataStore method internalQuery.
protected <T> CloseableIterator<T> internalQuery(final QueryConstraints constraints, final BaseQueryOptions queryOptions, final DeletionMode deleteMode) {
// Note: The DeletionMode option is provided to avoid recursively
// adding DuplicateDeletionCallbacks when actual duplicates are removed
// via the DuplicateDeletionCallback. The callback should only be added
// during the initial deletion query.
final boolean delete = ((deleteMode == DeletionMode.DELETE) || (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES));
final List<CloseableIterator<Object>> results = new ArrayList<>();
// If CQL filter is set
if (constraints instanceof TypeConstraintQuery) {
final String constraintTypeName = ((TypeConstraintQuery) constraints).getTypeName();
if ((queryOptions.getAdapterIds() == null) || (queryOptions.getAdapterIds().length == 0)) {
queryOptions.setAdapterId(internalAdapterStore.getAdapterId(constraintTypeName));
} else if (queryOptions.getAdapterIds().length == 1) {
final Short adapterId = internalAdapterStore.getAdapterId(constraintTypeName);
if ((adapterId == null) || (queryOptions.getAdapterIds()[0] != adapterId.shortValue())) {
LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
}
} else {
// Throw exception when QueryOptions has more than one adapter
// and CQL Adapter is set.
LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
}
}
final QueryConstraints sanitizedConstraints = (constraints == null) ? new EverythingQuery() : constraints;
final List<DataStoreCallbackManager> deleteCallbacks = new ArrayList<>();
final Map<Short, Set<ByteArray>> dataIdsToDelete;
if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && (baseOptions.isSecondaryIndexing())) {
dataIdsToDelete = new ConcurrentHashMap<>();
} else {
dataIdsToDelete = null;
}
final boolean dataIdIndexIsBest = baseOptions.isSecondaryIndexing() && ((sanitizedConstraints instanceof DataIdQuery) || (sanitizedConstraints instanceof DataIdRangeQuery) || (sanitizedConstraints instanceof EverythingQuery));
if (!delete && dataIdIndexIsBest) {
try {
// just grab the values directly from the Data Index
InternalDataAdapter<?>[] adapters = queryOptions.getAdaptersArray(adapterStore);
if (!queryOptions.isAllIndices()) {
final Set<Short> adapterIds = new HashSet<>(Arrays.asList(ArrayUtils.toObject(queryOptions.getValidAdapterIds(internalAdapterStore, indexMappingStore))));
adapters = Arrays.stream(adapters).filter(a -> adapterIds.contains(a.getAdapterId())).toArray(i -> new InternalDataAdapter<?>[i]);
}
// TODO test whether aggregations work in this case
for (final InternalDataAdapter<?> adapter : adapters) {
RowReader<GeoWaveRow> rowReader;
if (sanitizedConstraints instanceof DataIdQuery) {
rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
} else if (sanitizedConstraints instanceof DataIdRangeQuery) {
if (((DataIdRangeQuery) sanitizedConstraints).isReverse() && !isReverseIterationSupported()) {
throw new UnsupportedOperationException("Currently the underlying datastore does not support reverse iteration");
}
rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).isReverse());
} else {
rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId());
}
results.add(new CloseableIteratorWrapper(rowReader, new NativeEntryIteratorWrapper(adapterStore, indexMappingStore, DataIndexUtils.DATA_ID_INDEX, rowReader, null, queryOptions.getScanCallback(), BaseDataStoreUtils.getFieldBitmask(queryOptions.getFieldIdsAdapterPair(), DataIndexUtils.DATA_ID_INDEX), queryOptions.getMaxResolutionSubsamplingPerDimension(), !BaseDataStoreUtils.isCommonIndexAggregation(queryOptions.getAggregation()), null)));
}
if (BaseDataStoreUtils.isAggregation(queryOptions.getAggregation())) {
return BaseDataStoreUtils.aggregate(new CloseableIteratorWrapper(new Closeable() {
@Override
public void close() throws IOException {
for (final CloseableIterator<Object> result : results) {
result.close();
}
}
}, Iterators.concat(results.iterator())), (Aggregation) queryOptions.getAggregation().getRight(), (DataTypeAdapter) queryOptions.getAggregation().getLeft());
}
} catch (final IOException e1) {
LOGGER.error("Failed to resolve adapter or index for query", e1);
}
} else {
final boolean isConstraintsAdapterIndexSpecific = sanitizedConstraints instanceof AdapterAndIndexBasedQueryConstraints;
final boolean isAggregationAdapterIndexSpecific = (queryOptions.getAggregation() != null) && (queryOptions.getAggregation().getRight() instanceof AdapterAndIndexBasedAggregation);
// all queries will use the same instance of the dedupe filter for
// client side filtering because the filter needs to be applied across
// indices
DedupeFilter dedupeFilter = new DedupeFilter();
MemoryPersistentAdapterStore tempAdapterStore = new MemoryPersistentAdapterStore(queryOptions.getAdaptersArray(adapterStore));
MemoryAdapterIndexMappingStore memoryMappingStore = new MemoryAdapterIndexMappingStore();
// keep a list of adapters that have been queried, to only load an
// adapter to be queried once
final Set<Short> queriedAdapters = new HashSet<>();
// if its an ordered constraints then it is dependent on the index selected, if its
// secondary indexing its inefficient to delete by constraints
final boolean deleteAllIndicesByConstraints = ((delete && ((constraints == null) || !constraints.indexMustBeSpecified()) && !baseOptions.isSecondaryIndexing()));
final List<Pair<Index, List<InternalDataAdapter<?>>>> indexAdapterPairList = (deleteAllIndicesByConstraints) ? queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore) : queryOptions.getBestQueryIndices(tempAdapterStore, indexMappingStore, indexStore, statisticsStore, sanitizedConstraints);
Map<Short, List<Index>> additionalIndicesToDelete = null;
if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && !deleteAllIndicesByConstraints) {
additionalIndicesToDelete = new HashMap<>();
// we have to make sure to delete from the other indices if they exist
final List<Pair<Index, List<InternalDataAdapter<?>>>> allIndices = queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore);
for (final Pair<Index, List<InternalDataAdapter<?>>> allPair : allIndices) {
for (final Pair<Index, List<InternalDataAdapter<?>>> constraintPair : indexAdapterPairList) {
if (((constraintPair.getKey() == null) && (allPair.getKey() == null)) || constraintPair.getKey().equals(allPair.getKey())) {
allPair.getRight().removeAll(constraintPair.getRight());
break;
}
}
for (final InternalDataAdapter<?> adapter : allPair.getRight()) {
List<Index> indices = additionalIndicesToDelete.get(adapter.getAdapterId());
if (indices == null) {
indices = new ArrayList<>();
additionalIndicesToDelete.put(adapter.getAdapterId(), indices);
}
indices.add(allPair.getLeft());
}
}
}
final Pair<InternalDataAdapter<?>, Aggregation<?, ?, ?>> aggregation = queryOptions.getAggregation();
final ScanCallback callback = queryOptions.getScanCallback();
for (final Pair<Index, List<InternalDataAdapter<?>>> indexAdapterPair : indexAdapterPairList) {
if (indexAdapterPair.getKey() == null) {
// queries
if (dataIdIndexIsBest) {
// prior logic for !delete
for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
// this must be a data index only adapter, just worry about updating statistics and
// not other indices or duplicates
ScanCallback scanCallback = callback;
if (baseOptions.isPersistDataStatistics()) {
final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
deleteCallbacks.add(callbackCache);
scanCallback = new ScanCallback<Object, GeoWaveRow>() {
@Override
public void entryScanned(final Object entry, final GeoWaveRow row) {
if (callback != null) {
callback.entryScanned(entry, row);
}
callbackCache.getDeleteCallback(adapter, null, null).entryDeleted(entry, row);
}
};
}
if (sanitizedConstraints instanceof DataIdQuery) {
DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
} else if (sanitizedConstraints instanceof DataIdRangeQuery) {
DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive());
} else {
DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId());
}
}
} else {
final String[] typeNames = indexAdapterPair.getRight().stream().map(a -> a.getAdapter().getTypeName()).toArray(k -> new String[k]);
LOGGER.warn("Data types '" + ArrayUtils.toString(typeNames) + "' do not have an index that satisfies the query");
}
continue;
}
final List<Short> adapterIdsToQuery = new ArrayList<>();
// this only needs to be done once per index, not once per
// adapter
boolean queriedAllAdaptersByPrefix = false;
// maintain a set of data IDs if deleting using secondary indexing
for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
final Index index = indexAdapterPair.getLeft();
final AdapterToIndexMapping indexMapping = indexMappingStore.getMapping(adapter.getAdapterId(), index.getName());
memoryMappingStore.addAdapterIndexMapping(indexMapping);
if (delete) {
final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
// want the stats to change
if (!(constraints instanceof InsertionIdQuery)) {
callbackCache.setPersistStats(baseOptions.isPersistDataStatistics());
} else {
callbackCache.setPersistStats(false);
}
deleteCallbacks.add(callbackCache);
if (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES) {
final DeleteCallbackList<T, GeoWaveRow> delList = (DeleteCallbackList<T, GeoWaveRow>) callbackCache.getDeleteCallback(adapter, indexMapping, index);
final DuplicateDeletionCallback<T> dupDeletionCallback = new DuplicateDeletionCallback<>(this, adapter, indexMapping, index);
delList.addCallback(dupDeletionCallback);
if ((additionalIndicesToDelete != null) && (additionalIndicesToDelete.get(adapter.getAdapterId()) != null)) {
delList.addCallback(new DeleteOtherIndicesCallback<>(baseOperations, adapter, additionalIndicesToDelete.get(adapter.getAdapterId()), adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getAuthorizations()));
}
}
final Map<Short, Set<ByteArray>> internalDataIdsToDelete = dataIdsToDelete;
queryOptions.setScanCallback(new ScanCallback<Object, GeoWaveRow>() {
@Override
public void entryScanned(final Object entry, final GeoWaveRow row) {
if (callback != null) {
callback.entryScanned(entry, row);
}
if (internalDataIdsToDelete != null) {
final ByteArray dataId = new ByteArray(row.getDataId());
Set<ByteArray> currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
if (currentDataIdsToDelete == null) {
synchronized (internalDataIdsToDelete) {
currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
if (currentDataIdsToDelete == null) {
currentDataIdsToDelete = Sets.newConcurrentHashSet();
internalDataIdsToDelete.put(row.getAdapterId(), currentDataIdsToDelete);
}
}
}
currentDataIdsToDelete.add(dataId);
}
callbackCache.getDeleteCallback(adapter, indexMapping, index).entryDeleted(entry, row);
}
});
}
QueryConstraints adapterIndexConstraints;
if (isConstraintsAdapterIndexSpecific) {
adapterIndexConstraints = ((AdapterAndIndexBasedQueryConstraints) sanitizedConstraints).createQueryConstraints(adapter, indexAdapterPair.getLeft(), indexMapping);
if (adapterIndexConstraints == null) {
continue;
}
} else {
adapterIndexConstraints = sanitizedConstraints;
}
if (isAggregationAdapterIndexSpecific) {
queryOptions.setAggregation(((AdapterAndIndexBasedAggregation) aggregation.getRight()).createAggregation(adapter, indexMapping, index), aggregation.getLeft());
}
if (adapterIndexConstraints instanceof InsertionIdQuery) {
queryOptions.setLimit(-1);
results.add(queryInsertionId(adapter, index, (InsertionIdQuery) adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, delete));
continue;
} else if (adapterIndexConstraints instanceof PrefixIdQuery) {
if (!queriedAllAdaptersByPrefix) {
final PrefixIdQuery prefixIdQuery = (PrefixIdQuery) adapterIndexConstraints;
results.add(queryRowPrefix(index, prefixIdQuery.getPartitionKey(), prefixIdQuery.getSortKeyPrefix(), queryOptions, indexAdapterPair.getRight(), tempAdapterStore, delete));
queriedAllAdaptersByPrefix = true;
}
continue;
} else if (isConstraintsAdapterIndexSpecific || isAggregationAdapterIndexSpecific) {
// can't query multiple adapters in the same scan
results.add(queryConstraints(Collections.singletonList(adapter.getAdapterId()), index, adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
continue;
}
// finally just add it to a list to query multiple adapters
// in on scan
adapterIdsToQuery.add(adapter.getAdapterId());
}
// in one query instance (one scanner) for efficiency
if (adapterIdsToQuery.size() > 0) {
results.add(queryConstraints(adapterIdsToQuery, indexAdapterPair.getLeft(), sanitizedConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
}
if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode)) {
// Make sure each index query has a clean dedupe filter so that entries from other indices
// get deleted
dedupeFilter = new DedupeFilter();
}
}
}
return new CloseableIteratorWrapper<>(new Closeable() {
@Override
public void close() throws IOException {
for (final CloseableIterator<Object> result : results) {
result.close();
}
for (final DataStoreCallbackManager c : deleteCallbacks) {
c.close();
}
if ((dataIdsToDelete != null) && !dataIdsToDelete.isEmpty()) {
if (baseOptions.isSecondaryIndexing()) {
deleteFromDataIndex(dataIdsToDelete, queryOptions.getAuthorizations());
}
}
}
}, Iterators.concat(new CastIterator<T>(results.iterator())));
}
Aggregations