Examples with Query org.locationtech.geowave.core.store.api.Query used on opensource projects

Example 1 with Query

use of org.locationtech.geowave.core.store.api.Query in project geowave by locationtech.

the class BaseDataStore method internalQuery.

protected <T> CloseableIterator<T> internalQuery(final QueryConstraints constraints, final BaseQueryOptions queryOptions, final DeletionMode deleteMode) {
    // Note: The DeletionMode option is provided to avoid recursively
    // adding DuplicateDeletionCallbacks when actual duplicates are removed
    // via the DuplicateDeletionCallback. The callback should only be added
    // during the initial deletion query.
    final boolean delete = ((deleteMode == DeletionMode.DELETE) || (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES));
    final List<CloseableIterator<Object>> results = new ArrayList<>();
    // If CQL filter is set
    if (constraints instanceof TypeConstraintQuery) {
        final String constraintTypeName = ((TypeConstraintQuery) constraints).getTypeName();
        if ((queryOptions.getAdapterIds() == null) || (queryOptions.getAdapterIds().length == 0)) {
            queryOptions.setAdapterId(internalAdapterStore.getAdapterId(constraintTypeName));
        } else if (queryOptions.getAdapterIds().length == 1) {
            final Short adapterId = internalAdapterStore.getAdapterId(constraintTypeName);
            if ((adapterId == null) || (queryOptions.getAdapterIds()[0] != adapterId.shortValue())) {
                LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
                throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
            }
        } else {
            // Throw exception when QueryOptions has more than one adapter
            // and CQL Adapter is set.
            LOGGER.error("Constraint Query Type name does not match Query Options Type Name");
            throw new RuntimeException("Constraint Query Type name does not match Query Options Type Name");
        }
    }
    final QueryConstraints sanitizedConstraints = (constraints == null) ? new EverythingQuery() : constraints;
    final List<DataStoreCallbackManager> deleteCallbacks = new ArrayList<>();
    final Map<Short, Set<ByteArray>> dataIdsToDelete;
    if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && (baseOptions.isSecondaryIndexing())) {
        dataIdsToDelete = new ConcurrentHashMap<>();
    } else {
        dataIdsToDelete = null;
    }
    final boolean dataIdIndexIsBest = baseOptions.isSecondaryIndexing() && ((sanitizedConstraints instanceof DataIdQuery) || (sanitizedConstraints instanceof DataIdRangeQuery) || (sanitizedConstraints instanceof EverythingQuery));
    if (!delete && dataIdIndexIsBest) {
        try {
            // just grab the values directly from the Data Index
            InternalDataAdapter<?>[] adapters = queryOptions.getAdaptersArray(adapterStore);
            if (!queryOptions.isAllIndices()) {
                final Set<Short> adapterIds = new HashSet<>(Arrays.asList(ArrayUtils.toObject(queryOptions.getValidAdapterIds(internalAdapterStore, indexMappingStore))));
                adapters = Arrays.stream(adapters).filter(a -> adapterIds.contains(a.getAdapterId())).toArray(i -> new InternalDataAdapter<?>[i]);
            }
            // TODO test whether aggregations work in this case
            for (final InternalDataAdapter<?> adapter : adapters) {
                RowReader<GeoWaveRow> rowReader;
                if (sanitizedConstraints instanceof DataIdQuery) {
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
                } else if (sanitizedConstraints instanceof DataIdRangeQuery) {
                    if (((DataIdRangeQuery) sanitizedConstraints).isReverse() && !isReverseIterationSupported()) {
                        throw new UnsupportedOperationException("Currently the underlying datastore does not support reverse iteration");
                    }
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).isReverse());
                } else {
                    rowReader = DataIndexUtils.getRowReader(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), adapter.getAdapterId());
                }
                results.add(new CloseableIteratorWrapper(rowReader, new NativeEntryIteratorWrapper(adapterStore, indexMappingStore, DataIndexUtils.DATA_ID_INDEX, rowReader, null, queryOptions.getScanCallback(), BaseDataStoreUtils.getFieldBitmask(queryOptions.getFieldIdsAdapterPair(), DataIndexUtils.DATA_ID_INDEX), queryOptions.getMaxResolutionSubsamplingPerDimension(), !BaseDataStoreUtils.isCommonIndexAggregation(queryOptions.getAggregation()), null)));
            }
            if (BaseDataStoreUtils.isAggregation(queryOptions.getAggregation())) {
                return BaseDataStoreUtils.aggregate(new CloseableIteratorWrapper(new Closeable() {

                    @Override
                    public void close() throws IOException {
                        for (final CloseableIterator<Object> result : results) {
                            result.close();
                        }
                    }
                }, Iterators.concat(results.iterator())), (Aggregation) queryOptions.getAggregation().getRight(), (DataTypeAdapter) queryOptions.getAggregation().getLeft());
            }
        } catch (final IOException e1) {
            LOGGER.error("Failed to resolve adapter or index for query", e1);
        }
    } else {
        final boolean isConstraintsAdapterIndexSpecific = sanitizedConstraints instanceof AdapterAndIndexBasedQueryConstraints;
        final boolean isAggregationAdapterIndexSpecific = (queryOptions.getAggregation() != null) && (queryOptions.getAggregation().getRight() instanceof AdapterAndIndexBasedAggregation);
        // all queries will use the same instance of the dedupe filter for
        // client side filtering because the filter needs to be applied across
        // indices
        DedupeFilter dedupeFilter = new DedupeFilter();
        MemoryPersistentAdapterStore tempAdapterStore = new MemoryPersistentAdapterStore(queryOptions.getAdaptersArray(adapterStore));
        MemoryAdapterIndexMappingStore memoryMappingStore = new MemoryAdapterIndexMappingStore();
        // keep a list of adapters that have been queried, to only load an
        // adapter to be queried once
        final Set<Short> queriedAdapters = new HashSet<>();
        // if its an ordered constraints then it is dependent on the index selected, if its
        // secondary indexing its inefficient to delete by constraints
        final boolean deleteAllIndicesByConstraints = ((delete && ((constraints == null) || !constraints.indexMustBeSpecified()) && !baseOptions.isSecondaryIndexing()));
        final List<Pair<Index, List<InternalDataAdapter<?>>>> indexAdapterPairList = (deleteAllIndicesByConstraints) ? queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore) : queryOptions.getBestQueryIndices(tempAdapterStore, indexMappingStore, indexStore, statisticsStore, sanitizedConstraints);
        Map<Short, List<Index>> additionalIndicesToDelete = null;
        if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode) && !deleteAllIndicesByConstraints) {
            additionalIndicesToDelete = new HashMap<>();
            // we have to make sure to delete from the other indices if they exist
            final List<Pair<Index, List<InternalDataAdapter<?>>>> allIndices = queryOptions.getIndicesForAdapters(tempAdapterStore, indexMappingStore, indexStore);
            for (final Pair<Index, List<InternalDataAdapter<?>>> allPair : allIndices) {
                for (final Pair<Index, List<InternalDataAdapter<?>>> constraintPair : indexAdapterPairList) {
                    if (((constraintPair.getKey() == null) && (allPair.getKey() == null)) || constraintPair.getKey().equals(allPair.getKey())) {
                        allPair.getRight().removeAll(constraintPair.getRight());
                        break;
                    }
                }
                for (final InternalDataAdapter<?> adapter : allPair.getRight()) {
                    List<Index> indices = additionalIndicesToDelete.get(adapter.getAdapterId());
                    if (indices == null) {
                        indices = new ArrayList<>();
                        additionalIndicesToDelete.put(adapter.getAdapterId(), indices);
                    }
                    indices.add(allPair.getLeft());
                }
            }
        }
        final Pair<InternalDataAdapter<?>, Aggregation<?, ?, ?>> aggregation = queryOptions.getAggregation();
        final ScanCallback callback = queryOptions.getScanCallback();
        for (final Pair<Index, List<InternalDataAdapter<?>>> indexAdapterPair : indexAdapterPairList) {
            if (indexAdapterPair.getKey() == null) {
                // queries
                if (dataIdIndexIsBest) {
                    // prior logic for !delete
                    for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
                        // this must be a data index only adapter, just worry about updating statistics and
                        // not other indices or duplicates
                        ScanCallback scanCallback = callback;
                        if (baseOptions.isPersistDataStatistics()) {
                            final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
                            deleteCallbacks.add(callbackCache);
                            scanCallback = new ScanCallback<Object, GeoWaveRow>() {

                                @Override
                                public void entryScanned(final Object entry, final GeoWaveRow row) {
                                    if (callback != null) {
                                        callback.entryScanned(entry, row);
                                    }
                                    callbackCache.getDeleteCallback(adapter, null, null).entryDeleted(entry, row);
                                }
                            };
                        }
                        if (sanitizedConstraints instanceof DataIdQuery) {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdQuery) sanitizedConstraints).getDataIds());
                        } else if (sanitizedConstraints instanceof DataIdRangeQuery) {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId(), ((DataIdRangeQuery) sanitizedConstraints).getStartDataIdInclusive(), ((DataIdRangeQuery) sanitizedConstraints).getEndDataIdInclusive());
                        } else {
                            DataIndexUtils.delete(baseOperations, adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getFieldIdsAdapterPair(), queryOptions.getAggregation(), queryOptions.getAuthorizations(), scanCallback, adapter.getAdapterId());
                        }
                    }
                } else {
                    final String[] typeNames = indexAdapterPair.getRight().stream().map(a -> a.getAdapter().getTypeName()).toArray(k -> new String[k]);
                    LOGGER.warn("Data types '" + ArrayUtils.toString(typeNames) + "' do not have an index that satisfies the query");
                }
                continue;
            }
            final List<Short> adapterIdsToQuery = new ArrayList<>();
            // this only needs to be done once per index, not once per
            // adapter
            boolean queriedAllAdaptersByPrefix = false;
            // maintain a set of data IDs if deleting using secondary indexing
            for (final InternalDataAdapter adapter : indexAdapterPair.getRight()) {
                final Index index = indexAdapterPair.getLeft();
                final AdapterToIndexMapping indexMapping = indexMappingStore.getMapping(adapter.getAdapterId(), index.getName());
                memoryMappingStore.addAdapterIndexMapping(indexMapping);
                if (delete) {
                    final DataStoreCallbackManager callbackCache = new DataStoreCallbackManager(statisticsStore, queriedAdapters.add(adapter.getAdapterId()));
                    // want the stats to change
                    if (!(constraints instanceof InsertionIdQuery)) {
                        callbackCache.setPersistStats(baseOptions.isPersistDataStatistics());
                    } else {
                        callbackCache.setPersistStats(false);
                    }
                    deleteCallbacks.add(callbackCache);
                    if (deleteMode == DeletionMode.DELETE_WITH_DUPLICATES) {
                        final DeleteCallbackList<T, GeoWaveRow> delList = (DeleteCallbackList<T, GeoWaveRow>) callbackCache.getDeleteCallback(adapter, indexMapping, index);
                        final DuplicateDeletionCallback<T> dupDeletionCallback = new DuplicateDeletionCallback<>(this, adapter, indexMapping, index);
                        delList.addCallback(dupDeletionCallback);
                        if ((additionalIndicesToDelete != null) && (additionalIndicesToDelete.get(adapter.getAdapterId()) != null)) {
                            delList.addCallback(new DeleteOtherIndicesCallback<>(baseOperations, adapter, additionalIndicesToDelete.get(adapter.getAdapterId()), adapterStore, indexMappingStore, internalAdapterStore, queryOptions.getAuthorizations()));
                        }
                    }
                    final Map<Short, Set<ByteArray>> internalDataIdsToDelete = dataIdsToDelete;
                    queryOptions.setScanCallback(new ScanCallback<Object, GeoWaveRow>() {

                        @Override
                        public void entryScanned(final Object entry, final GeoWaveRow row) {
                            if (callback != null) {
                                callback.entryScanned(entry, row);
                            }
                            if (internalDataIdsToDelete != null) {
                                final ByteArray dataId = new ByteArray(row.getDataId());
                                Set<ByteArray> currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
                                if (currentDataIdsToDelete == null) {
                                    synchronized (internalDataIdsToDelete) {
                                        currentDataIdsToDelete = internalDataIdsToDelete.get(row.getAdapterId());
                                        if (currentDataIdsToDelete == null) {
                                            currentDataIdsToDelete = Sets.newConcurrentHashSet();
                                            internalDataIdsToDelete.put(row.getAdapterId(), currentDataIdsToDelete);
                                        }
                                    }
                                }
                                currentDataIdsToDelete.add(dataId);
                            }
                            callbackCache.getDeleteCallback(adapter, indexMapping, index).entryDeleted(entry, row);
                        }
                    });
                }
                QueryConstraints adapterIndexConstraints;
                if (isConstraintsAdapterIndexSpecific) {
                    adapterIndexConstraints = ((AdapterAndIndexBasedQueryConstraints) sanitizedConstraints).createQueryConstraints(adapter, indexAdapterPair.getLeft(), indexMapping);
                    if (adapterIndexConstraints == null) {
                        continue;
                    }
                } else {
                    adapterIndexConstraints = sanitizedConstraints;
                }
                if (isAggregationAdapterIndexSpecific) {
                    queryOptions.setAggregation(((AdapterAndIndexBasedAggregation) aggregation.getRight()).createAggregation(adapter, indexMapping, index), aggregation.getLeft());
                }
                if (adapterIndexConstraints instanceof InsertionIdQuery) {
                    queryOptions.setLimit(-1);
                    results.add(queryInsertionId(adapter, index, (InsertionIdQuery) adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, delete));
                    continue;
                } else if (adapterIndexConstraints instanceof PrefixIdQuery) {
                    if (!queriedAllAdaptersByPrefix) {
                        final PrefixIdQuery prefixIdQuery = (PrefixIdQuery) adapterIndexConstraints;
                        results.add(queryRowPrefix(index, prefixIdQuery.getPartitionKey(), prefixIdQuery.getSortKeyPrefix(), queryOptions, indexAdapterPair.getRight(), tempAdapterStore, delete));
                        queriedAllAdaptersByPrefix = true;
                    }
                    continue;
                } else if (isConstraintsAdapterIndexSpecific || isAggregationAdapterIndexSpecific) {
                    // can't query multiple adapters in the same scan
                    results.add(queryConstraints(Collections.singletonList(adapter.getAdapterId()), index, adapterIndexConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
                    continue;
                }
                // finally just add it to a list to query multiple adapters
                // in on scan
                adapterIdsToQuery.add(adapter.getAdapterId());
            }
            // in one query instance (one scanner) for efficiency
            if (adapterIdsToQuery.size() > 0) {
                results.add(queryConstraints(adapterIdsToQuery, indexAdapterPair.getLeft(), sanitizedConstraints, dedupeFilter, queryOptions, tempAdapterStore, memoryMappingStore, delete));
            }
            if (DeletionMode.DELETE_WITH_DUPLICATES.equals(deleteMode)) {
                // Make sure each index query has a clean dedupe filter so that entries from other indices
                // get deleted
                dedupeFilter = new DedupeFilter();
            }
        }
    }
    return new CloseableIteratorWrapper<>(new Closeable() {

        @Override
        public void close() throws IOException {
            for (final CloseableIterator<Object> result : results) {
                result.close();
            }
            for (final DataStoreCallbackManager c : deleteCallbacks) {
                c.close();
            }
            if ((dataIdsToDelete != null) && !dataIdsToDelete.isEmpty()) {
                if (baseOptions.isSecondaryIndexing()) {
                    deleteFromDataIndex(dataIdsToDelete, queryOptions.getAuthorizations());
                }
            }
        }
    }, Iterators.concat(new CastIterator<T>(results.iterator())));
}

Example 2 with Query

use of org.locationtech.geowave.core.store.api.Query in project geowave by locationtech.

the class BaseDataStore method copyTo.

@Override
public void copyTo(final DataStore other, final Query<?> query) {
    // check for 'everything' query
    if (query == null) {
        copyTo(other);
        return;
    }
    final String[] typeNames = query.getDataTypeQueryOptions().getTypeNames();
    final String indexName = query.getIndexQueryOptions().getIndexName();
    final boolean isAllIndices = query.getIndexQueryOptions().isAllIndices();
    final List<DataTypeAdapter<?>> typesToCopy;
    // if typeNames are not specified, then it means 'everything' as well
    if (((typeNames == null) || (typeNames.length == 0))) {
        if ((query.getQueryConstraints() == null) || (query.getQueryConstraints() instanceof EverythingQuery)) {
            copyTo(other);
            return;
        } else {
            typesToCopy = Arrays.asList(getTypes());
        }
    } else {
        // make sure the types requested exist in the source store (this)
        // before trying to copy!
        final DataTypeAdapter<?>[] sourceTypes = getTypes();
        typesToCopy = new ArrayList<>();
        for (int i = 0; i < typeNames.length; i++) {
            boolean found = false;
            for (int k = 0; k < sourceTypes.length; k++) {
                if (sourceTypes[k].getTypeName().compareTo(typeNames[i]) == 0) {
                    found = true;
                    typesToCopy.add(sourceTypes[k]);
                    break;
                }
            }
            if (!found) {
                throw new IllegalArgumentException("Some type names specified in the query do not exist in the source database and thus cannot be copied.");
            }
        }
    }
    // if there is an index requested in the query, make sure it exists in
    // the source store before trying to copy as well!
    final Index[] sourceIndices = getIndices();
    Index indexToCopy = null;
    if (!isAllIndices) {
        // just add the one index specified by the query
        // first make sure source index exists though
        boolean found = false;
        for (int i = 0; i < sourceIndices.length; i++) {
            if (sourceIndices[i].getName().compareTo(indexName) == 0) {
                found = true;
                indexToCopy = sourceIndices[i];
                break;
            }
        }
        if (!found) {
            throw new IllegalArgumentException("The index specified in the query does not exist in the source database and thus cannot be copied.");
        }
        // also make sure the types/index mapping for the query are legit
        for (int i = 0; i < typeNames.length; i++) {
            final short adapterId = internalAdapterStore.getAdapterId(typeNames[i]);
            final AdapterToIndexMapping[] indexMappings = indexMappingStore.getIndicesForAdapter(adapterId);
            found = false;
            for (int k = 0; k < indexMappings.length; k++) {
                if (indexMappings[k].getIndexName().compareTo(indexName) == 0) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                throw new IllegalArgumentException("The index " + indexName + " and the type " + typeNames[i] + " specified by the query are not associated in the source database");
            }
        }
    }
    // add all the types that the destination store doesn't have yet
    final DataTypeAdapter<?>[] destTypes = other.getTypes();
    for (int i = 0; i < typesToCopy.size(); i++) {
        boolean found = false;
        for (int k = 0; k < destTypes.length; k++) {
            if (destTypes[k].getTypeName().compareTo(typesToCopy.get(i).getTypeName()) == 0) {
                found = true;
                break;
            }
        }
        if (!found) {
            other.addType(typesToCopy.get(i));
        }
    }
    // add all the indices that the destination store doesn't have yet
    if (isAllIndices) {
        // in this case, all indices from the types requested by the query
        for (int i = 0; i < typesToCopy.size(); i++) {
            final String typeName = typesToCopy.get(i).getTypeName();
            final short adapterId = internalAdapterStore.getAdapterId(typeName);
            final AdapterToIndexMapping[] indexMappings = indexMappingStore.getIndicesForAdapter(adapterId);
            final Index[] indices = Arrays.stream(indexMappings).map(mapping -> mapping.getIndex(indexStore)).toArray(Index[]::new);
            other.addIndex(typeName, indices);
            final QueryBuilder<?, ?> qb = QueryBuilder.newBuilder().addTypeName(typeName).constraints(query.getQueryConstraints());
            try (CloseableIterator<?> it = query(qb.build())) {
                try (Writer<Object> writer = other.createWriter(typeName)) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            }
        }
    } else {
        // query
        for (int i = 0; i < typesToCopy.size(); i++) {
            other.addIndex(typesToCopy.get(i).getTypeName(), indexToCopy);
        }
        // we can write appropriately
        for (int k = 0; k < typesToCopy.size(); k++) {
            final InternalDataAdapter<?> adapter = adapterStore.getAdapter(internalAdapterStore.getAdapterId(typesToCopy.get(k).getTypeName()));
            final QueryBuilder<?, ?> qb = QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(indexToCopy.getName()).constraints(query.getQueryConstraints());
            try (CloseableIterator<?> it = query(qb.build())) {
                try (Writer<Object> writer = other.createWriter(adapter.getTypeName())) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            }
        }
    }
}

Example 3 with Query

use of org.locationtech.geowave.core.store.api.Query in project geowave by locationtech.

the class PropertyManagementTest method testQuery.

@Test
public void testQuery() throws Exception {
    final Geometry testGeoFilter = factory.createPolygon(new Coordinate[] { new Coordinate(24, 33), new Coordinate(28, 33), new Coordinate(28, 31), new Coordinate(24, 31), new Coordinate(24, 33) });
    final ExplicitSpatialQuery sq = new ExplicitSpatialQuery(testGeoFilter);
    final PropertyManagement pm = new PropertyManagement();
    pm.store(ExtractParameters.Extract.QUERY, QueryBuilder.newBuilder().constraints(sq).build());
    final Query q = pm.getPropertyAsQuery(ExtractParameters.Extract.QUERY);
    assertNotNull(q);
    final QueryConstraints c = q.getQueryConstraints();
    assertNotNull(c);
    assertNotNull(((ExplicitSpatialQuery) c).getQueryGeometry());
    assertEquals("POLYGON ((24 33, 28 33, 28 31, 24 31, 24 33))", ((ExplicitSpatialQuery) c).getQueryGeometry().toText());
    pm.store(ExtractParameters.Extract.QUERY, q);
    final Query q1 = (Query) pm.getPropertyAsPersistable(ExtractParameters.Extract.QUERY);
    assertNotNull(q1);
    final QueryConstraints c1 = q1.getQueryConstraints();
    assertNotNull(c1);
    assertNotNull(((ExplicitSpatialQuery) c1).getQueryGeometry());
    assertEquals("POLYGON ((24 33, 28 33, 28 31, 24 31, 24 33))", ((ExplicitSpatialQuery) c1).getQueryGeometry().toText());
}

Example 4 with Query

use of org.locationtech.geowave.core.store.api.Query in project geowave by locationtech.

the class GeoWaveAnalyticExtractJobRunner method run.

@Override
public int run(final Configuration config, final PropertyManagement runTimeProperties) throws Exception {
    runTimeProperties.storeIfEmpty(ExtractParameters.Extract.OUTPUT_DATA_TYPE_ID, "centroid");
    runTimeProperties.setConfig(new ParameterEnum[] { MapReduceParameters.MRConfig.HDFS_BASE_DIR, ExtractParameters.Extract.REDUCER_COUNT, ExtractParameters.Extract.DATA_NAMESPACE_URI, ExtractParameters.Extract.OUTPUT_DATA_TYPE_ID }, config, SimpleFeatureOutputReducer.class);
    config.set(GeoWaveConfiguratorBase.enumToConfKey(SimpleFeatureOutputReducer.class, ExtractParameters.Extract.GROUP_ID), runTimeProperties.getPropertyAsString(ExtractParameters.Extract.GROUP_ID, UUID.randomUUID().toString()));
    config.set(GeoWaveConfiguratorBase.enumToConfKey(SimpleFeatureOutputReducer.class, GlobalParameters.Global.BATCH_ID), runTimeProperties.getPropertyAsString(GlobalParameters.Global.BATCH_ID, UUID.randomUUID().toString()));
    final Query query = runTimeProperties.getPropertyAsQuery(ExtractParameters.Extract.QUERY);
    setMinInputSplits(runTimeProperties.getPropertyAsInt(ExtractParameters.Extract.MIN_INPUT_SPLIT, 1));
    setMaxInputSplits(runTimeProperties.getPropertyAsInt(ExtractParameters.Extract.MAX_INPUT_SPLIT, 10000));
    if (query != null) {
        if (query.getQueryConstraints() != null) {
            GeoWaveInputFormat.setQueryConstraints(config, query.getQueryConstraints());
            setQueryConstraints(query.getQueryConstraints());
        }
        if (query.getCommonQueryOptions() != null) {
            GeoWaveInputFormat.setCommonQueryOptions(config, query.getCommonQueryOptions());
            setCommonQueryOptions(query.getCommonQueryOptions());
        }
        if (query.getDataTypeQueryOptions() != null) {
            GeoWaveInputFormat.setDataTypeQueryOptions(config, query.getDataTypeQueryOptions(), dataStoreOptions.createAdapterStore(), dataStoreOptions.createInternalAdapterStore());
            setDataTypeQueryOptions(query.getDataTypeQueryOptions());
        }
        if (query.getIndexQueryOptions() != null) {
            GeoWaveInputFormat.setIndexQueryOptions(config, query.getIndexQueryOptions(), dataStoreOptions.createIndexStore());
            setIndexQueryOptions(query.getIndexQueryOptions());
        }
    }
    if (minInputSplits != null) {
        GeoWaveInputFormat.setMinimumSplitCount(config, minInputSplits);
    }
    if (maxInputSplits != null) {
        GeoWaveInputFormat.setMaximumSplitCount(config, maxInputSplits);
    }
    setConf(config);
    config.setClass(GeoWaveConfiguratorBase.enumToConfKey(SimpleFeatureOutputReducer.class, ExtractParameters.Extract.DIMENSION_EXTRACT_CLASS), runTimeProperties.getPropertyAsClass(ExtractParameters.Extract.DIMENSION_EXTRACT_CLASS, DimensionExtractor.class, SimpleFeatureGeometryExtractor.class), DimensionExtractor.class);
    final PersistableStore store = ((PersistableStore) runTimeProperties.getProperty(StoreParam.INPUT_STORE));
    dataStoreOptions = store.getDataStoreOptions();
    GeoWaveInputFormat.setStoreOptions(config, dataStoreOptions);
    GeoWaveOutputFormat.setStoreOptions(config, dataStoreOptions);
    try (final FileSystem fs = FileSystem.get(config)) {
        if (fs.exists(getHdfsOutputPath())) {
            fs.delete(// False positive - path is internally managed
            getHdfsOutputPath(), true);
        }
        return ToolRunner.run(config, this, new String[] {});
    }
}

Example 5 with Query

use of org.locationtech.geowave.core.store.api.Query in project geowave by locationtech.

the class BaseDataStore method copyTo.

@Override
public void copyTo(final DataStore other) {
    if (other instanceof BaseDataStore) {
        // efficiently copy underlying GeoWaveRow and GeoWaveMetadata
        for (final MetadataType metadataType : MetadataType.values()) {
            try (MetadataWriter writer = ((BaseDataStore) other).baseOperations.createMetadataWriter(metadataType)) {
                final MetadataReader reader = baseOperations.createMetadataReader(metadataType);
                try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery())) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            } catch (final Exception e) {
                LOGGER.error("Unable to write metadata on copy", e);
            }
        }
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            final AdapterToIndexMapping[] mappings = indexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
            for (final AdapterToIndexMapping mapping : mappings) {
                final Index index = mapping.getIndex(indexStore);
                final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
                final ReaderParamsBuilder<GeoWaveRow> bldr = new ReaderParamsBuilder<>(index, adapterStore, indexMappingStore, internalAdapterStore, rowMerging ? new GeoWaveRowMergingTransform(BaseDataStoreUtils.getRowMergingAdapter(adapter), adapter.getAdapterId()) : GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
                bldr.adapterIds(new short[] { adapter.getAdapterId() });
                bldr.isClientsideRowMerging(rowMerging);
                try (RowReader<GeoWaveRow> reader = baseOperations.createReader(bldr.build())) {
                    try (RowWriter writer = ((BaseDataStore) other).baseOperations.createWriter(index, adapter)) {
                        while (reader.hasNext()) {
                            writer.write(reader.next());
                        }
                    }
                } catch (final Exception e) {
                    LOGGER.error("Unable to write metadata on copy", e);
                }
            }
        }
    } else {
        final DataTypeAdapter<?>[] sourceTypes = getTypes();
        // add all the types that the destination store doesn't have yet
        final DataTypeAdapter<?>[] destTypes = other.getTypes();
        for (int i = 0; i < sourceTypes.length; i++) {
            boolean found = false;
            for (int k = 0; k < destTypes.length; k++) {
                if (destTypes[k].getTypeName().compareTo(sourceTypes[i].getTypeName()) == 0) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                other.addType(sourceTypes[i]);
            }
        }
        // add the indices for each type
        for (int i = 0; i < sourceTypes.length; i++) {
            final String typeName = sourceTypes[i].getTypeName();
            final short adapterId = internalAdapterStore.getAdapterId(typeName);
            final AdapterToIndexMapping[] indexMappings = indexMappingStore.getIndicesForAdapter(adapterId);
            final Index[] indices = Arrays.stream(indexMappings).map(mapping -> mapping.getIndex(indexStore)).toArray(Index[]::new);
            other.addIndex(typeName, indices);
            final QueryBuilder<?, ?> qb = QueryBuilder.newBuilder().addTypeName(typeName);
            try (CloseableIterator<?> it = query(qb.build())) {
                try (final Writer<Object> writer = other.createWriter(typeName)) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            }
        }
    }
}