Examples with PersistentAdapterStore org.locationtech.geowave.core.store.adapter.PersistentAdapterStore used on opensource projects

Example 1 with PersistentAdapterStore

use of org.locationtech.geowave.core.store.adapter.PersistentAdapterStore in project geowave by locationtech.

the class KMeansRunner method run.

public void run() throws IOException {
    initContext();
    // Validate inputs
    if (inputDataStore == null) {
        LOGGER.error("You must supply an input datastore!");
        throw new IOException("You must supply an input datastore!");
    }
    if (isUseTime()) {
        scaledRange = KMeansUtils.setRunnerTimeParams(this, inputDataStore, typeName);
        if (scaledRange == null) {
            LOGGER.error("Failed to set time params for kmeans. Please specify a valid feature type.");
            throw new ParameterException("--useTime option: Failed to set time params");
        }
    }
    // Retrieve the feature adapters
    final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
    List<String> featureTypeNames;
    // If provided, just use the one
    if (typeName != null) {
        featureTypeNames = new ArrayList<>();
        featureTypeNames.add(typeName);
    } else {
        // otherwise, grab all the feature adapters
        featureTypeNames = FeatureDataUtils.getFeatureTypeNames(inputDataStore);
    }
    bldr.setTypeNames(featureTypeNames.toArray(new String[0]));
    // This is required due to some funkiness in GeoWaveInputFormat
    final PersistentAdapterStore adapterStore = inputDataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = inputDataStore.createInternalAdapterStore();
    // Add a spatial filter if requested
    try {
        if (cqlFilter != null) {
            Geometry bbox = null;
            String cqlTypeName;
            if (typeName == null) {
                cqlTypeName = featureTypeNames.get(0);
            } else {
                cqlTypeName = typeName;
            }
            final short adapterId = internalAdapterStore.getAdapterId(cqlTypeName);
            final DataTypeAdapter<?> adapter = adapterStore.getAdapter(adapterId).getAdapter();
            if (adapter instanceof GeotoolsFeatureDataAdapter) {
                final String geometryAttribute = ((GeotoolsFeatureDataAdapter) adapter).getFeatureType().getGeometryDescriptor().getLocalName();
                Filter filter;
                filter = ECQL.toFilter(cqlFilter);
                final ExtractGeometryFilterVisitorResult geoAndCompareOpData = (ExtractGeometryFilterVisitorResult) filter.accept(new ExtractGeometryFilterVisitor(GeometryUtils.getDefaultCRS(), geometryAttribute), null);
                bbox = geoAndCompareOpData.getGeometry();
            }
            if ((bbox != null) && !bbox.equals(GeometryUtils.infinity())) {
                bldr.constraints(bldr.constraintsFactory().spatialTemporalConstraints().spatialConstraints(bbox).build());
            }
        }
    } catch (final CQLException e) {
        LOGGER.error("Unable to parse CQL: " + cqlFilter);
    }
    // Load RDD from datastore
    final RDDOptions kmeansOpts = new RDDOptions();
    kmeansOpts.setMinSplits(minSplits);
    kmeansOpts.setMaxSplits(maxSplits);
    kmeansOpts.setQuery(bldr.build());
    final GeoWaveRDD kmeansRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kmeansOpts);
    // Retrieve the input centroids
    LOGGER.debug("Retrieving input centroids from RDD...");
    centroidVectors = RDDUtils.rddFeatureVectors(kmeansRDD, timeField, scaledTimeRange);
    centroidVectors.cache();
    // Init the algorithm
    final KMeans kmeans = new KMeans();
    kmeans.setInitializationMode("kmeans||");
    kmeans.setK(numClusters);
    kmeans.setMaxIterations(numIterations);
    if (epsilon > -1.0) {
        kmeans.setEpsilon(epsilon);
    }
    // Run KMeans
    LOGGER.debug("Running KMeans algorithm...");
    outputModel = kmeans.run(centroidVectors.rdd());
    LOGGER.debug("Writing results to output store...");
    writeToOutputStore();
    LOGGER.debug("Results successfully written!");
}

Example 2 with PersistentAdapterStore

use of org.locationtech.geowave.core.store.adapter.PersistentAdapterStore in project geowave by locationtech.

the class DataStoreUtils method mergeData.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
    final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
    try {
        final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
                mergingAdapters.put(adapter.getAdapterId(), adapter);
            }
        }
        final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
        final short[] adapterIds = new short[1];
        for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
            adapterIds[0] = adapter.getKey();
            paramsBuilder.adapterIds(adapterIds);
            try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
                final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
                final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
                while (iterator.hasNext()) {
                    iterator.next();
                }
            } catch (final Exception e) {
                LOGGER.error("Exception occurred while merging data.", e);
                throw new RuntimeException(e);
            }
        }
    } finally {
        try {
            deleter.close();
        } catch (final Exception e) {
            LOGGER.warn("Exception occurred when closing deleter.", e);
        }
    }
    return true;
}

Example 3 with PersistentAdapterStore

use of org.locationtech.geowave.core.store.adapter.PersistentAdapterStore in project geowave by locationtech.

the class AccumuloUtils method getIterator.

private static CloseableIterator<Entry<Key, Value>> getIterator(final Connector connector, final String namespace, final Index index) throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
    CloseableIterator<Entry<Key, Value>> iterator = null;
    final AccumuloOptions options = new AccumuloOptions();
    final AccumuloOperations operations = new AccumuloOperations(connector, namespace, new AccumuloOptions());
    final IndexStore indexStore = new IndexStoreImpl(operations, options);
    final PersistentAdapterStore adapterStore = new AdapterStoreImpl(operations, options);
    final AdapterIndexMappingStore mappingStore = new AdapterIndexMappingStoreImpl(operations, options);
    if (indexStore.indexExists(index.getName())) {
        final ScannerBase scanner = operations.createBatchScanner(index.getName());
        ((BatchScanner) scanner).setRanges(AccumuloUtils.byteArrayRangesToAccumuloRanges(null));
        final IteratorSetting iteratorSettings = new IteratorSetting(10, "GEOWAVE_WHOLE_ROW_ITERATOR", WholeRowIterator.class);
        scanner.addScanIterator(iteratorSettings);
        final Iterator<Entry<Key, Value>> it = new IteratorWrapper(adapterStore, mappingStore, index, scanner.iterator(), new QueryFilter[] { new DedupeFilter() });
        iterator = new CloseableIteratorWrapper<>(new ScannerClosableWrapper(scanner), it);
    }
    return iterator;
}

Example 4 with PersistentAdapterStore

use of org.locationtech.geowave.core.store.adapter.PersistentAdapterStore in project geowave by locationtech.

the class GeoWaveSparkIngestIT method testBasicSparkIngest.

@Test
public void testBasicSparkIngest() throws Exception {
    // ingest test points
    TestUtils.testSparkIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILES, "gdelt");
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        // query by the full bounding box, make sure there is more than
        // 0 count and make sure the count matches the number of results
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_COUNT), new Integer(resultCount));
    }
    // Clean up
    TestUtils.deleteAll(dataStore);
}

Example 5 with PersistentAdapterStore

use of org.locationtech.geowave.core.store.adapter.PersistentAdapterStore in project geowave by locationtech.

the class BasicKafkaIT method testBasicIngestGpx.

@Test
public void testBasicIngestGpx() throws Exception {
    KafkaTestUtils.testKafkaStage(OSM_GPX_INPUT_DIR);
    KafkaTestUtils.testKafkaIngest(dataStorePluginOptions, false, OSM_GPX_INPUT_DIR);
    final DataStatisticsStore statsStore = dataStorePluginOptions.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStorePluginOptions.createAdapterStore();
    int adapterCount = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", EXPECTED_COUNT_PER_ADAPTER_ID.get(adapter.getTypeName()), new Integer(resultCount));
        adapterCount++;
    }
    assertTrue("There should be exactly two adapters", (adapterCount == 2));
}