use of org.locationtech.geowave.core.store.adapter.InternalAdapterStore in project geowave by locationtech.
the class KMeansRunner method run.
public void run() throws IOException {
initContext();
// Validate inputs
if (inputDataStore == null) {
LOGGER.error("You must supply an input datastore!");
throw new IOException("You must supply an input datastore!");
}
if (isUseTime()) {
scaledRange = KMeansUtils.setRunnerTimeParams(this, inputDataStore, typeName);
if (scaledRange == null) {
LOGGER.error("Failed to set time params for kmeans. Please specify a valid feature type.");
throw new ParameterException("--useTime option: Failed to set time params");
}
}
// Retrieve the feature adapters
final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
List<String> featureTypeNames;
// If provided, just use the one
if (typeName != null) {
featureTypeNames = new ArrayList<>();
featureTypeNames.add(typeName);
} else {
// otherwise, grab all the feature adapters
featureTypeNames = FeatureDataUtils.getFeatureTypeNames(inputDataStore);
}
bldr.setTypeNames(featureTypeNames.toArray(new String[0]));
// This is required due to some funkiness in GeoWaveInputFormat
final PersistentAdapterStore adapterStore = inputDataStore.createAdapterStore();
final InternalAdapterStore internalAdapterStore = inputDataStore.createInternalAdapterStore();
// Add a spatial filter if requested
try {
if (cqlFilter != null) {
Geometry bbox = null;
String cqlTypeName;
if (typeName == null) {
cqlTypeName = featureTypeNames.get(0);
} else {
cqlTypeName = typeName;
}
final short adapterId = internalAdapterStore.getAdapterId(cqlTypeName);
final DataTypeAdapter<?> adapter = adapterStore.getAdapter(adapterId).getAdapter();
if (adapter instanceof GeotoolsFeatureDataAdapter) {
final String geometryAttribute = ((GeotoolsFeatureDataAdapter) adapter).getFeatureType().getGeometryDescriptor().getLocalName();
Filter filter;
filter = ECQL.toFilter(cqlFilter);
final ExtractGeometryFilterVisitorResult geoAndCompareOpData = (ExtractGeometryFilterVisitorResult) filter.accept(new ExtractGeometryFilterVisitor(GeometryUtils.getDefaultCRS(), geometryAttribute), null);
bbox = geoAndCompareOpData.getGeometry();
}
if ((bbox != null) && !bbox.equals(GeometryUtils.infinity())) {
bldr.constraints(bldr.constraintsFactory().spatialTemporalConstraints().spatialConstraints(bbox).build());
}
}
} catch (final CQLException e) {
LOGGER.error("Unable to parse CQL: " + cqlFilter);
}
// Load RDD from datastore
final RDDOptions kmeansOpts = new RDDOptions();
kmeansOpts.setMinSplits(minSplits);
kmeansOpts.setMaxSplits(maxSplits);
kmeansOpts.setQuery(bldr.build());
final GeoWaveRDD kmeansRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kmeansOpts);
// Retrieve the input centroids
LOGGER.debug("Retrieving input centroids from RDD...");
centroidVectors = RDDUtils.rddFeatureVectors(kmeansRDD, timeField, scaledTimeRange);
centroidVectors.cache();
// Init the algorithm
final KMeans kmeans = new KMeans();
kmeans.setInitializationMode("kmeans||");
kmeans.setK(numClusters);
kmeans.setMaxIterations(numIterations);
if (epsilon > -1.0) {
kmeans.setEpsilon(epsilon);
}
// Run KMeans
LOGGER.debug("Running KMeans algorithm...");
outputModel = kmeans.run(centroidVectors.rdd());
LOGGER.debug("Writing results to output store...");
writeToOutputStore();
LOGGER.debug("Results successfully written!");
}
use of org.locationtech.geowave.core.store.adapter.InternalAdapterStore in project geowave by locationtech.
the class RasterTileResizeSparkRunner method run.
public void run() throws IOException {
initContext();
// Validate inputs
if (inputStoreOptions == null) {
LOGGER.error("You must supply an input datastore!");
throw new IOException("You must supply an input datastore!");
}
final InternalAdapterStore internalAdapterStore = inputStoreOptions.createInternalAdapterStore();
final short internalAdapterId = internalAdapterStore.getAdapterId(rasterResizeOptions.getInputCoverageName());
final DataTypeAdapter adapter = inputStoreOptions.createAdapterStore().getAdapter(internalAdapterId).getAdapter();
if (adapter == null) {
throw new IllegalArgumentException("Adapter for coverage '" + rasterResizeOptions.getInputCoverageName() + "' does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
}
Index index = null;
final IndexStore indexStore = inputStoreOptions.createIndexStore();
if (rasterResizeOptions.getIndexName() != null) {
index = indexStore.getIndex(rasterResizeOptions.getIndexName());
}
if (index == null) {
try (CloseableIterator<Index> indices = indexStore.getIndices()) {
index = indices.next();
}
if (index == null) {
throw new IllegalArgumentException("Index does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
}
}
final RasterDataAdapter newAdapter = new RasterDataAdapter((RasterDataAdapter) adapter, rasterResizeOptions.getOutputCoverageName(), rasterResizeOptions.getOutputTileSize());
final DataStore store = outputStoreOptions.createDataStore();
store.addType(newAdapter, index);
final short newInternalAdapterId = outputStoreOptions.createInternalAdapterStore().addTypeName(newAdapter.getTypeName());
final RDDOptions options = new RDDOptions();
if (rasterResizeOptions.getMinSplits() != null) {
options.setMinSplits(rasterResizeOptions.getMinSplits());
}
if (rasterResizeOptions.getMaxSplits() != null) {
options.setMaxSplits(rasterResizeOptions.getMaxSplits());
}
final JavaPairRDD<GeoWaveInputKey, GridCoverage> inputRDD = GeoWaveRDDLoader.loadRawRasterRDD(jsc.sc(), inputStoreOptions, index.getName(), rasterResizeOptions.getMinSplits(), rasterResizeOptions.getMaxSplits());
LOGGER.debug("Writing results to output store...");
RDDUtils.writeRasterToGeoWave(jsc.sc(), index, outputStoreOptions, newAdapter, inputRDD.flatMapToPair(new RasterResizeMappingFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)).groupByKey().map(new MergeRasterFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)));
LOGGER.debug("Results successfully written!");
}
use of org.locationtech.geowave.core.store.adapter.InternalAdapterStore in project geowave by locationtech.
the class KSamplerMapReduceTest method setUp.
@Before
public void setUp() throws IOException {
final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
mapDriver = MapDriver.newMapDriver(mapper);
reduceDriver = ReduceDriver.newReduceDriver(reducer);
final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
final PropertyManagement propManagement = new PropertyManagement();
final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
pluginOptions.selectPlugin("memory");
final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
opts.setGeoWaveNamespace(namespace);
final PersistableStore store = new PersistableStore(pluginOptions);
propManagement.store(StoreParam.INPUT_STORE, store);
propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
// TODO it seems the centroid adapter is required to have been written,
// should this initialization be handled by the runner class rather than
// externally such as in the test?
final DataStore dataStore = store.getDataStoreOptions().createDataStore();
final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
serializations();
}
use of org.locationtech.geowave.core.store.adapter.InternalAdapterStore in project geowave by locationtech.
the class SpatialJoinRunner method createDefaultAdapterTypeName.
private String createDefaultAdapterTypeName(final String typeName, final DataStorePluginOptions storeOptions) {
final StringBuffer defaultAdapterName = new StringBuffer(typeName + "_joined");
final InternalAdapterStore adapterStore = storeOptions.createInternalAdapterStore();
if (adapterStore.getAdapterId(defaultAdapterName.toString()) == null) {
return defaultAdapterName.toString();
}
Integer iSuffix = 0;
final StringBuffer uniNum = new StringBuffer("_" + String.format("%02d", iSuffix));
defaultAdapterName.append(uniNum);
while (adapterStore.getAdapterId(defaultAdapterName.toString()) != null) {
// Should be _00 _01 etc
iSuffix += 1;
uniNum.append("_").append(String.format("%02d", iSuffix));
defaultAdapterName.append(uniNum);
}
return defaultAdapterName.toString();
}
use of org.locationtech.geowave.core.store.adapter.InternalAdapterStore in project geowave by locationtech.
the class DataStoreUtils method mergeData.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
try {
final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
mergingAdapters.put(adapter.getAdapterId(), adapter);
}
}
final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
final short[] adapterIds = new short[1];
for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
adapterIds[0] = adapter.getKey();
paramsBuilder.adapterIds(adapterIds);
try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
while (iterator.hasNext()) {
iterator.next();
}
} catch (final Exception e) {
LOGGER.error("Exception occurred while merging data.", e);
throw new RuntimeException(e);
}
}
} finally {
try {
deleter.close();
} catch (final Exception e) {
LOGGER.warn("Exception occurred when closing deleter.", e);
}
}
return true;
}
Aggregations