Search in sources :

Example 1 with MapReduceIntegration

use of org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration in project geowave by locationtech.

the class NNJobRunnerTest method init.

@Before
public void init() {
    jjJobRunner.setMapReduceIntegrater(new MapReduceIntegration() {

        @Override
        public int submit(final Configuration configuration, final PropertyManagement runTimeProperties, final GeoWaveAnalyticJobRunner tool) throws Exception {
            tool.setConf(configuration);
            return ToolRunner.run(configuration, tool, new String[] {});
        }

        @Override
        public Counters waitForCompletion(final Job job) throws ClassNotFoundException, IOException, InterruptedException {
            Assert.assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());
            Assert.assertEquals(10, job.getNumReduceTasks());
            final ScopedJobConfiguration configWrapper = new ScopedJobConfiguration(job.getConfiguration(), NNMapReduce.class);
            Assert.assertEquals("file://foo/bin", job.getConfiguration().get("mapred.input.dir"));
            Assert.assertEquals(0.4, configWrapper.getDouble(Partition.MAX_DISTANCE, 0.0), 0.001);
            Assert.assertEquals(100, configWrapper.getInt(Partition.MAX_MEMBER_SELECTION, 1));
            try {
                final Partitioner<?> wrapper = configWrapper.getInstance(Partition.PARTITIONER_CLASS, Partitioner.class, null);
                Assert.assertEquals(OrthodromicDistancePartitioner.class, wrapper.getClass());
                final Partitioner<?> secondary = configWrapper.getInstance(Partition.SECONDARY_PARTITIONER_CLASS, Partitioner.class, null);
                Assert.assertEquals(OrthodromicDistancePartitioner.class, secondary.getClass());
                final DistanceFn<?> distancFn = configWrapper.getInstance(CommonParameters.Common.DISTANCE_FUNCTION_CLASS, DistanceFn.class, GeometryCentroidDistanceFn.class);
                Assert.assertEquals(FeatureCentroidDistanceFn.class, distancFn.getClass());
            } catch (final InstantiationException e) {
                throw new IOException("Unable to configure system", e);
            } catch (final IllegalAccessException e) {
                throw new IOException("Unable to configure system", e);
            }
            Assert.assertEquals(10, job.getNumReduceTasks());
            return new Counters();
        }

        @Override
        public Job getJob(final Tool tool) throws IOException {
            return new Job(tool.getConf());
        }

        @Override
        public Configuration getConfiguration(final PropertyManagement runTimeProperties) throws IOException {
            return new Configuration();
        }
    });
    jjJobRunner.setInputFormatConfiguration(new SequenceFileInputFormatConfiguration(new Path("file://foo/bin")));
    jjJobRunner.setReducerCount(10);
    runTimeProperties.store(MRConfig.HDFS_BASE_DIR, "/");
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    runTimeProperties.store(StoreParam.INPUT_STORE, store);
    runTimeProperties.store(CommonParameters.Common.DISTANCE_FUNCTION_CLASS, FeatureCentroidDistanceFn.class);
    runTimeProperties.store(Partition.PARTITIONER_CLASS, OrthodromicDistancePartitioner.class);
    runTimeProperties.store(Partition.SECONDARY_PARTITIONER_CLASS, OrthodromicDistancePartitioner.class);
    runTimeProperties.store(Partition.MAX_DISTANCE, Double.valueOf(0.4));
    runTimeProperties.store(Partition.MAX_MEMBER_SELECTION, Integer.valueOf(100));
}
Also used : GeometryCentroidDistanceFn(org.locationtech.geowave.analytic.distance.GeometryCentroidDistanceFn) Configuration(org.apache.hadoop.conf.Configuration) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) OrthodromicDistancePartitioner(org.locationtech.geowave.analytic.partitioner.OrthodromicDistancePartitioner) MemoryStoreFactoryFamily(org.locationtech.geowave.core.store.memory.MemoryStoreFactoryFamily) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) Job(org.apache.hadoop.mapreduce.Job) MapReduceIntegration(org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) MemoryRequiredOptions(org.locationtech.geowave.core.store.memory.MemoryRequiredOptions) Partitioner(org.locationtech.geowave.analytic.partitioner.Partitioner) OrthodromicDistancePartitioner(org.locationtech.geowave.analytic.partitioner.OrthodromicDistancePartitioner) Tool(org.apache.hadoop.util.Tool) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) Path(org.apache.hadoop.fs.Path) SequenceFileInputFormat(org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat) PropertyManagement(org.locationtech.geowave.analytic.PropertyManagement) IOException(java.io.IOException) FeatureCentroidDistanceFn(org.locationtech.geowave.analytic.distance.FeatureCentroidDistanceFn) GeometryCentroidDistanceFn(org.locationtech.geowave.analytic.distance.GeometryCentroidDistanceFn) DistanceFn(org.locationtech.geowave.analytic.distance.DistanceFn) IOException(java.io.IOException) GeoWaveAnalyticJobRunner(org.locationtech.geowave.analytic.mapreduce.GeoWaveAnalyticJobRunner) FeatureCentroidDistanceFn(org.locationtech.geowave.analytic.distance.FeatureCentroidDistanceFn) PersistableStore(org.locationtech.geowave.analytic.store.PersistableStore) Counters(org.apache.hadoop.mapreduce.Counters) Before(org.junit.Before)

Example 2 with MapReduceIntegration

use of org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration in project geowave by locationtech.

the class ConvexHullJobRunnerTest method init.

@Before
public void init() {
    final SimpleFeatureType ftype = AnalyticFeature.createGeometryFeatureAdapter("centroidtest", new String[] { "extra1" }, BasicFeatureTypes.DEFAULT_NAMESPACE, ClusteringUtils.CLUSTERING_CRS).getFeatureType();
    hullRunner.setMapReduceIntegrater(new MapReduceIntegration() {

        @Override
        public int submit(final Configuration configuration, final PropertyManagement runTimeProperties, final GeoWaveAnalyticJobRunner tool) throws Exception {
            tool.setConf(configuration);
            ((ParameterHelper<Object>) StoreParam.INPUT_STORE.getHelper()).setValue(configuration, ConvexHullMapReduce.class, StoreParam.INPUT_STORE.getHelper().getValue(runTimeProperties));
            return tool.run(new String[] {});
        }

        @Override
        public Counters waitForCompletion(final Job job) throws ClassNotFoundException, IOException, InterruptedException {
            Assert.assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());
            Assert.assertEquals(10, job.getNumReduceTasks());
            final ScopedJobConfiguration configWrapper = new ScopedJobConfiguration(job.getConfiguration(), ConvexHullMapReduce.class);
            Assert.assertEquals("file://foo/bin", job.getConfiguration().get("mapred.input.dir"));
            final PersistableStore persistableStore = (PersistableStore) StoreParam.INPUT_STORE.getHelper().getValue(job, ConvexHullMapReduce.class, null);
            final IndexStore indexStore = persistableStore.getDataStoreOptions().createIndexStore();
            try {
                Assert.assertTrue(indexStore.indexExists("spatial"));
                final PersistableStore persistableAdapterStore = (PersistableStore) StoreParam.INPUT_STORE.getHelper().getValue(job, ConvexHullMapReduce.class, null);
                final PersistentAdapterStore adapterStore = persistableAdapterStore.getDataStoreOptions().createAdapterStore();
                Assert.assertTrue(adapterStore.adapterExists(persistableAdapterStore.getDataStoreOptions().createInternalAdapterStore().getAdapterId("centroidtest")));
                final Projection<?> projection = configWrapper.getInstance(HullParameters.Hull.PROJECTION_CLASS, Projection.class, SimpleFeatureProjection.class);
                Assert.assertEquals(SimpleFeatureProjection.class, projection.getClass());
            } catch (final InstantiationException e) {
                throw new IOException("Unable to configure system", e);
            } catch (final IllegalAccessException e) {
                throw new IOException("Unable to configure system", e);
            }
            Assert.assertEquals(10, job.getNumReduceTasks());
            Assert.assertEquals(2, configWrapper.getInt(CentroidParameters.Centroid.ZOOM_LEVEL, -1));
            return new Counters();
        }

        @Override
        public Job getJob(final Tool tool) throws IOException {
            return new Job(tool.getConf());
        }

        @Override
        public Configuration getConfiguration(final PropertyManagement runTimeProperties) throws IOException {
            return new Configuration();
        }
    });
    hullRunner.setInputFormatConfiguration(new SequenceFileInputFormatConfiguration());
    runTimeProperties.store(MRConfig.HDFS_BASE_DIR, "/");
    runTimeProperties.store(InputParameters.Input.HDFS_INPUT_PATH, new Path("file://foo/bin"));
    runTimeProperties.store(GlobalParameters.Global.BATCH_ID, "b1234");
    runTimeProperties.store(HullParameters.Hull.DATA_TYPE_ID, "hullType");
    runTimeProperties.store(HullParameters.Hull.REDUCER_COUNT, 10);
    runTimeProperties.store(HullParameters.Hull.INDEX_NAME, "spatial");
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    runTimeProperties.store(StoreParam.INPUT_STORE, store);
    final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
    pluginOptions.createAdapterStore().addAdapter(adapter.asInternalAdapter(pluginOptions.createInternalAdapterStore().addTypeName(adapter.getTypeName())));
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) SimpleFeatureProjection(org.locationtech.geowave.analytic.SimpleFeatureProjection) Projection(org.locationtech.geowave.analytic.Projection) MemoryStoreFactoryFamily(org.locationtech.geowave.core.store.memory.MemoryStoreFactoryFamily) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) Job(org.apache.hadoop.mapreduce.Job) MapReduceIntegration(org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) MemoryRequiredOptions(org.locationtech.geowave.core.store.memory.MemoryRequiredOptions) Tool(org.apache.hadoop.util.Tool) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) Path(org.apache.hadoop.fs.Path) ConvexHullMapReduce(org.locationtech.geowave.analytic.mapreduce.clustering.ConvexHullMapReduce) SequenceFileInputFormat(org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat) PropertyManagement(org.locationtech.geowave.analytic.PropertyManagement) IOException(java.io.IOException) IOException(java.io.IOException) GeoWaveAnalyticJobRunner(org.locationtech.geowave.analytic.mapreduce.GeoWaveAnalyticJobRunner) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) SimpleFeatureType(org.opengis.feature.simple.SimpleFeatureType) PersistableStore(org.locationtech.geowave.analytic.store.PersistableStore) Counters(org.apache.hadoop.mapreduce.Counters) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) SimpleFeatureProjection(org.locationtech.geowave.analytic.SimpleFeatureProjection) Before(org.junit.Before)

Example 3 with MapReduceIntegration

use of org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration in project geowave by locationtech.

the class GroupAssigmentJobRunnerTest method init.

@Before
public void init() {
    final SimpleFeatureType ftype = AnalyticFeature.createGeometryFeatureAdapter("centroidtest", new String[] { "extra1" }, BasicFeatureTypes.DEFAULT_NAMESPACE, ClusteringUtils.CLUSTERING_CRS).getFeatureType();
    runner.setMapReduceIntegrater(new MapReduceIntegration() {

        @Override
        public int submit(final Configuration configuration, final PropertyManagement runTimeProperties, final GeoWaveAnalyticJobRunner tool) throws Exception {
            tool.setConf(configuration);
            ((ParameterHelper<Object>) StoreParam.INPUT_STORE.getHelper()).setValue(configuration, GroupAssignmentMapReduce.class, StoreParam.INPUT_STORE.getHelper().getValue(runTimeProperties));
            return tool.run(new String[] {});
        }

        @Override
        public Counters waitForCompletion(final Job job) throws ClassNotFoundException, IOException, InterruptedException {
            Assert.assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());
            Assert.assertEquals(10, job.getNumReduceTasks());
            final ScopedJobConfiguration configWrapper = new ScopedJobConfiguration(job.getConfiguration(), GroupAssignmentMapReduce.class);
            Assert.assertEquals("file://foo/bin", job.getConfiguration().get("mapred.input.dir"));
            Assert.assertEquals(3, configWrapper.getInt(CentroidParameters.Centroid.ZOOM_LEVEL, -1));
            Assert.assertEquals("b1234", configWrapper.getString(GlobalParameters.Global.PARENT_BATCH_ID, ""));
            Assert.assertEquals("b12345", configWrapper.getString(GlobalParameters.Global.BATCH_ID, ""));
            try {
                final AnalyticItemWrapperFactory<?> wrapper = configWrapper.getInstance(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, AnalyticItemWrapperFactory.class, SimpleFeatureItemWrapperFactory.class);
                Assert.assertEquals(SimpleFeatureItemWrapperFactory.class, wrapper.getClass());
                final DistanceFn<?> distancFn = configWrapper.getInstance(CommonParameters.Common.DISTANCE_FUNCTION_CLASS, DistanceFn.class, GeometryCentroidDistanceFn.class);
                Assert.assertEquals(FeatureCentroidDistanceFn.class, distancFn.getClass());
            } catch (final InstantiationException e) {
                throw new IOException("Unable to configure system", e);
            } catch (final IllegalAccessException e) {
                throw new IOException("Unable to configure system", e);
            }
            return new Counters();
        }

        @Override
        public Job getJob(final Tool tool) throws IOException {
            return new Job(tool.getConf());
        }

        @Override
        public Configuration getConfiguration(final PropertyManagement runTimeProperties) throws IOException {
            return new Configuration();
        }
    });
    runner.setInputFormatConfiguration(new SequenceFileInputFormatConfiguration(new Path("file://foo/bin")));
    runner.setZoomLevel(3);
    runner.setReducerCount(10);
    runTimeProperties.store(MRConfig.HDFS_BASE_DIR, "/");
    runTimeProperties.store(GlobalParameters.Global.BATCH_ID, "b12345");
    runTimeProperties.store(GlobalParameters.Global.PARENT_BATCH_ID, "b1234");
    runTimeProperties.store(CommonParameters.Common.DISTANCE_FUNCTION_CLASS, FeatureCentroidDistanceFn.class);
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    runTimeProperties.store(StoreParam.INPUT_STORE, store);
    final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
    pluginOptions.createAdapterStore().addAdapter(adapter.asInternalAdapter(pluginOptions.createInternalAdapterStore().addTypeName(adapter.getTypeName())));
}
Also used : GeometryCentroidDistanceFn(org.locationtech.geowave.analytic.distance.GeometryCentroidDistanceFn) Configuration(org.apache.hadoop.conf.Configuration) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) GroupAssignmentMapReduce(org.locationtech.geowave.analytic.mapreduce.clustering.GroupAssignmentMapReduce) SimpleFeatureItemWrapperFactory(org.locationtech.geowave.analytic.SimpleFeatureItemWrapperFactory) MemoryStoreFactoryFamily(org.locationtech.geowave.core.store.memory.MemoryStoreFactoryFamily) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) Job(org.apache.hadoop.mapreduce.Job) MapReduceIntegration(org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration) SequenceFileInputFormatConfiguration(org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration) MemoryRequiredOptions(org.locationtech.geowave.core.store.memory.MemoryRequiredOptions) AnalyticItemWrapperFactory(org.locationtech.geowave.analytic.AnalyticItemWrapperFactory) Tool(org.apache.hadoop.util.Tool) ScopedJobConfiguration(org.locationtech.geowave.analytic.ScopedJobConfiguration) Path(org.apache.hadoop.fs.Path) SequenceFileInputFormat(org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat) PropertyManagement(org.locationtech.geowave.analytic.PropertyManagement) IOException(java.io.IOException) FeatureCentroidDistanceFn(org.locationtech.geowave.analytic.distance.FeatureCentroidDistanceFn) GeometryCentroidDistanceFn(org.locationtech.geowave.analytic.distance.GeometryCentroidDistanceFn) DistanceFn(org.locationtech.geowave.analytic.distance.DistanceFn) IOException(java.io.IOException) GeoWaveAnalyticJobRunner(org.locationtech.geowave.analytic.mapreduce.GeoWaveAnalyticJobRunner) SimpleFeatureType(org.opengis.feature.simple.SimpleFeatureType) FeatureCentroidDistanceFn(org.locationtech.geowave.analytic.distance.FeatureCentroidDistanceFn) PersistableStore(org.locationtech.geowave.analytic.store.PersistableStore) Counters(org.apache.hadoop.mapreduce.Counters) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Before(org.junit.Before)

Aggregations

IOException (java.io.IOException)3 Configuration (org.apache.hadoop.conf.Configuration)3 Path (org.apache.hadoop.fs.Path)3 Counters (org.apache.hadoop.mapreduce.Counters)3 Job (org.apache.hadoop.mapreduce.Job)3 SequenceFileInputFormat (org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat)3 Tool (org.apache.hadoop.util.Tool)3 Before (org.junit.Before)3 PropertyManagement (org.locationtech.geowave.analytic.PropertyManagement)3 ScopedJobConfiguration (org.locationtech.geowave.analytic.ScopedJobConfiguration)3 GeoWaveAnalyticJobRunner (org.locationtech.geowave.analytic.mapreduce.GeoWaveAnalyticJobRunner)3 MapReduceIntegration (org.locationtech.geowave.analytic.mapreduce.MapReduceIntegration)3 SequenceFileInputFormatConfiguration (org.locationtech.geowave.analytic.mapreduce.SequenceFileInputFormatConfiguration)3 PersistableStore (org.locationtech.geowave.analytic.store.PersistableStore)3 DataStorePluginOptions (org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions)3 MemoryRequiredOptions (org.locationtech.geowave.core.store.memory.MemoryRequiredOptions)3 MemoryStoreFactoryFamily (org.locationtech.geowave.core.store.memory.MemoryStoreFactoryFamily)3 FeatureDataAdapter (org.locationtech.geowave.adapter.vector.FeatureDataAdapter)2 DistanceFn (org.locationtech.geowave.analytic.distance.DistanceFn)2 FeatureCentroidDistanceFn (org.locationtech.geowave.analytic.distance.FeatureCentroidDistanceFn)2