Examples with IndexPluginOptions org.locationtech.geowave.core.store.index.IndexPluginOptions used on opensource projects

Example 1 with IndexPluginOptions

use of org.locationtech.geowave.core.store.index.IndexPluginOptions in project geowave by locationtech.

the class KafkaTestUtils method testKafkaIngest.

public static void testKafkaIngest(final DataStorePluginOptions options, final boolean spatialTemporal, final String ingestFilePath) throws Exception {
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    // // FIXME
    // final String[] args = StringUtils.split("-kafkaingest" +
    // " -f gpx -batchSize 1 -consumerTimeoutMs 5000 -reconnectOnTimeout
    // -groupId testGroup"
    // + " -autoOffsetReset smallest -fetchMessageMaxBytes " +
    // MAX_MESSAGE_BYTES +
    // " -zookeeperConnect " + zookeeper + " -" +
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin("gpx");
    // Indexes
    final IndexPluginOptions indexOption = new IndexPluginOptions();
    indexOption.selectPlugin((spatialTemporal ? "spatial_temporal" : "spatial"));
    // Execute Command
    final KafkaToGeoWaveCommand kafkaToGeowave = new KafkaToGeoWaveCommand();
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(options);
    addStore.execute(params);
    final IndexStore indexStore = options.createIndexStore();
    final DataStore dataStore = options.createDataStore();
    if (indexStore.getIndex("testIndex") == null) {
        indexOption.setName("testIndex");
        dataStore.addIndex(indexOption.createIndex(dataStore));
    }
    kafkaToGeowave.setPluginFormats(ingestFormatOptions);
    kafkaToGeowave.getKafkaOptions().setBootstrapServers(KafkaTestEnvironment.getInstance().getBootstrapServers());
    kafkaToGeowave.getKafkaOptions().setConsumerTimeoutMs("5000");
    kafkaToGeowave.getKafkaOptions().setReconnectOnTimeout(false);
    kafkaToGeowave.getKafkaOptions().setGroupId("testGroup");
    kafkaToGeowave.getKafkaOptions().setAutoOffsetReset("earliest");
    kafkaToGeowave.getKafkaOptions().setMaxPartitionFetchBytes(MAX_MESSAGE_BYTES);
    kafkaToGeowave.setParameters("test-store", "testIndex");
    kafkaToGeowave.execute(params);
    // finish.
    try {
        kafkaToGeowave.getDriver().waitFutures();
    } catch (InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with IndexPluginOptions

use of org.locationtech.geowave.core.store.index.IndexPluginOptions in project geowave by locationtech.

the class MapReduceTestUtils method testMapReduceIngest.

public static void testMapReduceIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String format, final String ingestFilePath) throws Exception {
    // ingest gpx data directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    final Thread progressLogger = startProgressLogger();
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    // Ingest Formats
    final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // create temporary config file and use it for hdfs FS URL config
    final File configFile = File.createTempFile("test_mr", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final ConfigHDFSCommand configHdfs = new ConfigHDFSCommand();
    configHdfs.setHdfsUrlParameter(env.getHdfs());
    configHdfs.execute(operationParams);
    final LocalToMapReduceToGeoWaveCommand mrGw = new LocalToMapReduceToGeoWaveCommand();
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        String indexName = "testIndex" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    mrGw.setPluginFormats(ingestFormatOptions);
    mrGw.setParameters(ingestFilePath, env.getHdfsBaseDirectory(), "test-store", indexParam.toString());
    mrGw.getMapReduceOptions().setJobTrackerHostPort(env.getJobtracker());
    mrGw.execute(operationParams);
    progressLogger.interrupt();
}

Example 3 with IndexPluginOptions

use of org.locationtech.geowave.core.store.index.IndexPluginOptions in project geowave by locationtech.

the class TestUtils method testLocalIngest.

public static void testLocalIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String crsCode, final String ingestFilePath, final String format, final int nthreads, final boolean supportTimeRange) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        if (crsCode != null) {
            if (indexOption.getDimensionalityOptions() instanceof SpatialOptions) {
                ((SpatialOptions) indexOption.getDimensionalityOptions()).setCrs(crsCode);
            } else if (indexOption.getDimensionalityOptions() instanceof SpatialTemporalOptions) {
                ((SpatialTemporalOptions) indexOption.getDimensionalityOptions()).setCrs(crsCode);
            }
        }
        if (indexOption.getDimensionalityOptions() instanceof TemporalOptions) {
            ((TemporalOptions) indexOption.getDimensionalityOptions()).setNoTimeRanges(!supportTimeRange);
        }
        indexOptions.add(indexOption);
    }
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    // Add Store
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(params);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    // Add indices
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        final String indexName = "test-index" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    // Create the command and execute.
    final LocalToGeoWaveCommand localIngester = new LocalToGeoWaveCommand();
    localIngester.setPluginFormats(ingestFormatOptions);
    localIngester.setParameters(ingestFilePath, "test-store", indexParam.toString());
    localIngester.setThreads(nthreads);
    localIngester.execute(params);
    verifyStats(dataStore);
}

Example 4 with IndexPluginOptions

use of org.locationtech.geowave.core.store.index.IndexPluginOptions in project geowave by locationtech.

the class TestUtils method testS3LocalIngest.

public static void testS3LocalIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String s3Url, final String ingestFilePath, final String format, final int nthreads) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    final File configFile = File.createTempFile("test_s3_local_ingest", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        final String indexName = "test-index" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    final ConfigAWSCommand configS3 = new ConfigAWSCommand();
    configS3.setS3UrlParameter(s3Url);
    configS3.execute(operationParams);
    // Create the command and execute.
    final LocalToGeoWaveCommand localIngester = new LocalToGeoWaveCommand();
    localIngester.setPluginFormats(ingestFormatOptions);
    localIngester.setParameters(ingestFilePath, "test-store", indexParam.toString());
    localIngester.setThreads(nthreads);
    localIngester.execute(operationParams);
    verifyStats(dataStore);
}

Example 5 with IndexPluginOptions

use of org.locationtech.geowave.core.store.index.IndexPluginOptions in project geowave by locationtech.

the class CustomCRSKDERasterResizeIT method testKDEAndRasterResize.

@Test
public void testKDEAndRasterResize() throws Exception {
    TestUtils.deleteAll(inputDataStorePluginOptions);
    TestUtils.testLocalIngest(inputDataStorePluginOptions, DimensionalityType.SPATIAL, "EPSG:4901", KDE_SHAPEFILE_FILE, "geotools-vector", 1);
    final File configFile = File.createTempFile("test_export", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-in");
    addStore.setPluginOptions(inputDataStorePluginOptions);
    addStore.execute(params);
    addStore.setParameters("raster-spatial");
    addStore.setPluginOptions(outputDataStorePluginOptions);
    addStore.execute(params);
    final String outputIndexName = "raster-spatial-idx";
    final IndexPluginOptions outputIndexOptions = new IndexPluginOptions();
    outputIndexOptions.selectPlugin("spatial");
    outputIndexOptions.setName(outputIndexName);
    ((SpatialOptions) outputIndexOptions.getDimensionalityOptions()).setCrs("EPSG:4240");
    final DataStore outputDataStore = outputDataStorePluginOptions.createDataStore();
    final Index outputIndex = outputIndexOptions.createIndex(outputDataStore);
    outputDataStore.addIndex(outputIndex);
    // use the min level to define the request boundary because it is the
    // most coarse grain
    final double decimalDegreesPerCellMinLevel = 180.0 / Math.pow(2, BASE_MIN_LEVEL);
    final double cellOriginXMinLevel = Math.round(TARGET_MIN_LON / decimalDegreesPerCellMinLevel);
    final double cellOriginYMinLevel = Math.round(TARGET_MIN_LAT / decimalDegreesPerCellMinLevel);
    final double numCellsMinLevel = Math.round(TARGET_DECIMAL_DEGREES_SIZE / decimalDegreesPerCellMinLevel);
    final GeneralEnvelope queryEnvelope = new GeneralEnvelope(new double[] { // scaling on the tile composition/rendering
    decimalDegreesPerCellMinLevel * cellOriginXMinLevel, decimalDegreesPerCellMinLevel * cellOriginYMinLevel }, new double[] { // scaling
    decimalDegreesPerCellMinLevel * (cellOriginXMinLevel + numCellsMinLevel), decimalDegreesPerCellMinLevel * (cellOriginYMinLevel + numCellsMinLevel) });
    final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
    final String geomField = ((FeatureDataAdapter) inputDataStorePluginOptions.createDataStore().getTypes()[0]).getFeatureType().getGeometryDescriptor().getLocalName();
    final Envelope cqlEnv = JTS.transform(new Envelope(155.12, 155.17, 16.07, 16.12), CRS.findMathTransform(CRS.decode("EPSG:4326"), CRS.decode("EPSG:4901"), true));
    final String cqlStr = String.format("BBOX(%s, %f, %f, %f, %f)", geomField, cqlEnv.getMinX(), cqlEnv.getMinY(), cqlEnv.getMaxX(), cqlEnv.getMaxY());
    for (int i = MIN_TILE_SIZE_POWER_OF_2; i <= MAX_TILE_SIZE_POWER_OF_2; i += INCREMENT) {
        LOGGER.warn("running mapreduce kde: " + i);
        final String tileSizeCoverageName = TEST_COVERAGE_NAME_MR_PREFIX + i;
        final KdeCommand command = new KdeCommand();
        command.setParameters("test-in", "raster-spatial");
        command.getKdeOptions().setCqlFilter(cqlStr);
        command.getKdeOptions().setOutputIndex(outputIndexName);
        command.getKdeOptions().setFeatureType(KDE_FEATURE_TYPE_NAME);
        command.getKdeOptions().setMinLevel(BASE_MIN_LEVEL);
        command.getKdeOptions().setMaxLevel(BASE_MAX_LEVEL);
        command.getKdeOptions().setMinSplits(MapReduceTestUtils.MIN_INPUT_SPLITS);
        command.getKdeOptions().setMaxSplits(MapReduceTestUtils.MAX_INPUT_SPLITS);
        command.getKdeOptions().setCoverageName(tileSizeCoverageName);
        command.getKdeOptions().setHdfsHostPort(env.getHdfs());
        command.getKdeOptions().setJobTrackerOrResourceManHostPort(env.getJobtracker());
        command.getKdeOptions().setTileSize((int) Math.pow(2, i));
        ToolRunner.run(command.createRunner(params), new String[] {});
    }
    final int numLevels = (BASE_MAX_LEVEL - BASE_MIN_LEVEL) + 1;
    final double[][][][] initialSampleValuesPerRequestSize = new double[numLevels][][][];
    LOGGER.warn("testing mapreduce kdes");
    for (int l = 0; l < numLevels; l++) {
        initialSampleValuesPerRequestSize[l] = testSamplesMatch(TEST_COVERAGE_NAME_MR_PREFIX, ((MAX_TILE_SIZE_POWER_OF_2 - MIN_TILE_SIZE_POWER_OF_2) / INCREMENT) + 1, queryEnvelope, new Rectangle((int) (numCellsMinLevel * Math.pow(2, l)), (int) (numCellsMinLevel * Math.pow(2, l))), null);
    }
    for (int i = MIN_TILE_SIZE_POWER_OF_2; i <= MAX_TILE_SIZE_POWER_OF_2; i += INCREMENT) {
        LOGGER.warn("running spark kde: " + i);
        final String tileSizeCoverageName = TEST_COVERAGE_NAME_SPARK_PREFIX + i;
        final KDESparkCommand command = new KDESparkCommand();
        // We're going to override these anyway.
        command.setParameters("test-in", "raster-spatial");
        command.getKDESparkOptions().setOutputIndex(outputIndexName);
        command.getKDESparkOptions().setCqlFilter(cqlStr);
        command.getKDESparkOptions().setTypeName(KDE_FEATURE_TYPE_NAME);
        command.getKDESparkOptions().setMinLevel(BASE_MIN_LEVEL);
        command.getKDESparkOptions().setMaxLevel(BASE_MAX_LEVEL);
        command.getKDESparkOptions().setMinSplits(MapReduceTestUtils.MIN_INPUT_SPLITS);
        command.getKDESparkOptions().setMaxSplits(MapReduceTestUtils.MAX_INPUT_SPLITS);
        command.getKDESparkOptions().setCoverageName(tileSizeCoverageName);
        command.getKDESparkOptions().setMaster("local[*]");
        command.getKDESparkOptions().setTileSize((int) Math.pow(2, i));
        command.execute(params);
    }
    LOGGER.warn("testing spark kdes");
    for (int l = 0; l < numLevels; l++) {
        testSamplesMatch(TEST_COVERAGE_NAME_SPARK_PREFIX, ((MAX_TILE_SIZE_POWER_OF_2 - MIN_TILE_SIZE_POWER_OF_2) / INCREMENT) + 1, queryEnvelope, new Rectangle((int) (numCellsMinLevel * Math.pow(2, l)), (int) (numCellsMinLevel * Math.pow(2, l))), initialSampleValuesPerRequestSize[l]);
    }
    // go from the original mr KDEs to a resized version using the MR command
    for (int i = MIN_TILE_SIZE_POWER_OF_2; i <= MAX_TILE_SIZE_POWER_OF_2; i += INCREMENT) {
        LOGGER.warn("running mapreduce resize: " + i);
        final String originalTileSizeCoverageName = TEST_COVERAGE_NAME_MR_PREFIX + i;
        final String resizeTileSizeCoverageName = TEST_RESIZE_COVERAGE_NAME_MR_PREFIX + i;
        final ResizeMRCommand command = new ResizeMRCommand();
        // We're going to override these anyway.
        command.setParameters("raster-spatial", "raster-spatial");
        command.getOptions().setInputCoverageName(originalTileSizeCoverageName);
        command.getOptions().setMinSplits(MapReduceTestUtils.MIN_INPUT_SPLITS);
        command.getOptions().setMaxSplits(MapReduceTestUtils.MAX_INPUT_SPLITS);
        command.setHdfsHostPort(env.getHdfs());
        command.setJobTrackerOrResourceManHostPort(env.getJobtracker());
        command.getOptions().setOutputCoverageName(resizeTileSizeCoverageName);
        command.getOptions().setIndexName(TestUtils.createWebMercatorSpatialIndex().getName());
        // due to time considerations when running the test, downsample to
        // at most 2 powers of 2 lower
        int targetRes = (MAX_TILE_SIZE_POWER_OF_2 - i);
        if ((i - targetRes) > 2) {
            targetRes = i - 2;
        }
        command.getOptions().setOutputTileSize((int) Math.pow(2, targetRes));
        ToolRunner.run(command.createRunner(params), new String[] {});
    }
    LOGGER.warn("testing mapreduce resize");
    for (int l = 0; l < numLevels; l++) {
        testSamplesMatch(TEST_RESIZE_COVERAGE_NAME_MR_PREFIX, ((MAX_TILE_SIZE_POWER_OF_2 - MIN_TILE_SIZE_POWER_OF_2) / INCREMENT) + 1, queryEnvelope, new Rectangle((int) (numCellsMinLevel * Math.pow(2, l)), (int) (numCellsMinLevel * Math.pow(2, l))), initialSampleValuesPerRequestSize[l]);
    }
    // Spark command
    for (int i = MIN_TILE_SIZE_POWER_OF_2; i <= MAX_TILE_SIZE_POWER_OF_2; i += INCREMENT) {
        LOGGER.warn("running spark resize: " + i);
        final String originalTileSizeCoverageName = TEST_COVERAGE_NAME_SPARK_PREFIX + i;
        final String resizeTileSizeCoverageName = TEST_RESIZE_COVERAGE_NAME_SPARK_PREFIX + i;
        final ResizeSparkCommand command = new ResizeSparkCommand();
        // We're going to override these anyway.
        command.setParameters("raster-spatial", "raster-spatial");
        command.getOptions().setInputCoverageName(originalTileSizeCoverageName);
        command.getOptions().setMinSplits(MapReduceTestUtils.MIN_INPUT_SPLITS);
        command.getOptions().setMaxSplits(MapReduceTestUtils.MAX_INPUT_SPLITS);
        command.getOptions().setOutputCoverageName(resizeTileSizeCoverageName);
        command.getOptions().setIndexName(TestUtils.createWebMercatorSpatialIndex().getName());
        command.setMaster("local[*]");
        // due to time considerations when running the test, downsample to
        // at most 2 powers of 2 lower
        int targetRes = (MAX_TILE_SIZE_POWER_OF_2 - i);
        if ((i - targetRes) > 2) {
            targetRes = i - 2;
        }
        command.getOptions().setOutputTileSize((int) Math.pow(2, targetRes));
        command.execute(params);
    }
    LOGGER.warn("testing spark resize");
    for (int l = 0; l < numLevels; l++) {
        testSamplesMatch(TEST_RESIZE_COVERAGE_NAME_SPARK_PREFIX, ((MAX_TILE_SIZE_POWER_OF_2 - MIN_TILE_SIZE_POWER_OF_2) / INCREMENT) + 1, queryEnvelope, new Rectangle((int) (numCellsMinLevel * Math.pow(2, l)), (int) (numCellsMinLevel * Math.pow(2, l))), initialSampleValuesPerRequestSize[l]);
    }
}