Examples with AddStoreCommand org.locationtech.geowave.core.store.cli.store.AddStoreCommand used on opensource projects

Example 1 with AddStoreCommand

use of org.locationtech.geowave.core.store.cli.store.AddStoreCommand in project geowave by locationtech.

the class ConfigCacheIT method addStore.

@Test
public void addStore() {
    final String storeName = new MemoryStoreFactoryFamily().getType();
    final AddStoreCommand command = new AddStoreCommand();
    command.setParameters("abc");
    command.setMakeDefault(true);
    command.setStoreType(storeName);
    // This will load the params via SPI.
    command.prepare(operationParams);
    final DataStorePluginOptions options = command.getPluginOptions();
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) options.getFactoryOptions();
    opts.setGeoWaveNamespace("namespace");
    command.execute(operationParams);
    final Properties props = ConfigOptions.loadProperties(configFile);
    Assert.assertEquals("namespace", props.getProperty("store.abc.opts." + StoreFactoryOptions.GEOWAVE_NAMESPACE_OPTION));
    Assert.assertEquals("abc", props.getProperty(DataStorePluginOptions.DEFAULT_PROPERTY_NAMESPACE));
}

Example 2 with AddStoreCommand

use of org.locationtech.geowave.core.store.cli.store.AddStoreCommand in project geowave by locationtech.

the class StoreCopyIT method testStoreCopy.

@Test
public void testStoreCopy() throws Exception {
    // Load some test data
    LOGGER.warn("Loading input data...");
    TestUtils.testLocalIngest(inputDataStorePluginOptions, DimensionalityType.SPATIAL, HAIL_SHAPEFILE_FILE, 1);
    LOGGER.warn("Querying input data...");
    // Query the input store
    try {
        testQuery(new File(TEST_BOX_FILTER_FILE).toURI().toURL(), new URL[] { new File(HAIL_EXPECTED_BOX_FILTER_RESULTS_FILE).toURI().toURL() }, TestUtils.DEFAULT_SPATIAL_INDEX, "bounding box constraint only");
    } catch (final Exception e) {
        e.printStackTrace();
        TestUtils.deleteAll(inputDataStorePluginOptions);
        Assert.fail("Error occurred while querying the input store: '" + e.getLocalizedMessage() + "'");
    }
    LOGGER.warn("Execute storecopy job...");
    try {
        final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
        // Set up the copy command
        final CopyCommand command = new CopyCommand();
        final File configFile = File.createTempFile("test_stats", null);
        final ManualOperationParams params = new ManualOperationParams();
        params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
        final AddStoreCommand addStore = new AddStoreCommand();
        addStore.setParameters("test-store-in");
        addStore.setPluginOptions(inputDataStorePluginOptions);
        addStore.execute(params);
        addStore.setParameters("test-store-out");
        addStore.setPluginOptions(outputDataStorePluginOptions);
        addStore.execute(params);
        command.setParameters("test-store-in", "test-store-out");
        command.getOptions().setHdfsHostPort(env.getHdfs());
        command.getOptions().setJobTrackerOrResourceManHostPort(env.getJobtracker());
        command.getOptions().setMinSplits(MapReduceTestUtils.MIN_INPUT_SPLITS);
        command.getOptions().setMaxSplits(MapReduceTestUtils.MAX_INPUT_SPLITS);
        command.getOptions().setNumReducers(8);
        ToolRunner.run(command.createRunner(params), new String[] {});
    } catch (final Exception e) {
        e.printStackTrace();
        TestUtils.deleteAll(inputDataStorePluginOptions);
        Assert.fail("Error occurred while copying the datastore: '" + e.getLocalizedMessage() + "'");
    }
    LOGGER.warn("Querying output data...");
    // Query the copy store
    testOutput = true;
    try {
        testQuery(new File(TEST_BOX_FILTER_FILE).toURI().toURL(), new URL[] { new File(HAIL_EXPECTED_BOX_FILTER_RESULTS_FILE).toURI().toURL() }, TestUtils.DEFAULT_SPATIAL_INDEX, "bounding box constraint only");
    } catch (final Exception e) {
        e.printStackTrace();
        TestUtils.deleteAll(outputDataStorePluginOptions);
        Assert.fail("Error occurred while querying the output store: '" + e.getLocalizedMessage() + "'");
    }
    LOGGER.warn("Copy complete.");
}

Example 3 with AddStoreCommand

use of org.locationtech.geowave.core.store.cli.store.AddStoreCommand in project geowave by locationtech.

the class MapReduceIT method testIngestOSMPBF.

@Test
public void testIngestOSMPBF() throws Exception {
    TestUtils.deleteAll(dataStoreOptions);
    // NOTE: This will probably fail unless you bump up the memory for the
    // tablet
    // servers, for whatever reason, using the
    // miniAccumuloConfig.setMemory() function.
    final MapReduceTestEnvironment mrEnv = MapReduceTestEnvironment.getInstance();
    // TODO: for now this only works with accumulo, generalize the data
    // store usage
    final AccumuloStoreTestEnvironment accumuloEnv = AccumuloStoreTestEnvironment.getInstance();
    final String hdfsPath = mrEnv.getHdfsBaseDirectory() + "/osm_stage/";
    final StageOSMToHDFSCommand stage = new StageOSMToHDFSCommand();
    stage.setParameters(TEST_DATA_BASE_DIR, hdfsPath);
    stage.execute(mrEnv.getOperationParams());
    final Connector conn = new ZooKeeperInstance(accumuloEnv.getAccumuloInstance(), accumuloEnv.getZookeeper()).getConnector(accumuloEnv.getAccumuloUser(), new PasswordToken(accumuloEnv.getAccumuloPassword()));
    final Authorizations auth = new Authorizations(new String[] { "public" });
    conn.securityOperations().changeUserAuthorizations(accumuloEnv.getAccumuloUser(), auth);
    final IngestOSMToGeoWaveCommand ingest = new IngestOSMToGeoWaveCommand();
    ingest.setParameters(hdfsPath, "test-store");
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStoreOptions);
    addStore.execute(mrEnv.getOperationParams());
    ingest.getIngestOptions().setJobName("ConversionTest");
    // Execute for node's ways, and relations.
    ingest.getIngestOptions().setMapperType("NODE");
    ingest.execute(mrEnv.getOperationParams());
    System.out.println("finished accumulo ingest Node");
    ingest.getIngestOptions().setMapperType("WAY");
    ingest.execute(mrEnv.getOperationParams());
    System.out.println("finished accumulo ingest Way");
    ingest.getIngestOptions().setMapperType("RELATION");
    ingest.execute(mrEnv.getOperationParams());
    System.out.println("finished accumulo ingest Relation");
}

Example 4 with AddStoreCommand

use of org.locationtech.geowave.core.store.cli.store.AddStoreCommand in project geowave by locationtech.

the class KafkaTestUtils method testKafkaIngest.

public static void testKafkaIngest(final DataStorePluginOptions options, final boolean spatialTemporal, final String ingestFilePath) throws Exception {
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    // // FIXME
    // final String[] args = StringUtils.split("-kafkaingest" +
    // " -f gpx -batchSize 1 -consumerTimeoutMs 5000 -reconnectOnTimeout
    // -groupId testGroup"
    // + " -autoOffsetReset smallest -fetchMessageMaxBytes " +
    // MAX_MESSAGE_BYTES +
    // " -zookeeperConnect " + zookeeper + " -" +
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin("gpx");
    // Indexes
    final IndexPluginOptions indexOption = new IndexPluginOptions();
    indexOption.selectPlugin((spatialTemporal ? "spatial_temporal" : "spatial"));
    // Execute Command
    final KafkaToGeoWaveCommand kafkaToGeowave = new KafkaToGeoWaveCommand();
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(options);
    addStore.execute(params);
    final IndexStore indexStore = options.createIndexStore();
    final DataStore dataStore = options.createDataStore();
    if (indexStore.getIndex("testIndex") == null) {
        indexOption.setName("testIndex");
        dataStore.addIndex(indexOption.createIndex(dataStore));
    }
    kafkaToGeowave.setPluginFormats(ingestFormatOptions);
    kafkaToGeowave.getKafkaOptions().setBootstrapServers(KafkaTestEnvironment.getInstance().getBootstrapServers());
    kafkaToGeowave.getKafkaOptions().setConsumerTimeoutMs("5000");
    kafkaToGeowave.getKafkaOptions().setReconnectOnTimeout(false);
    kafkaToGeowave.getKafkaOptions().setGroupId("testGroup");
    kafkaToGeowave.getKafkaOptions().setAutoOffsetReset("earliest");
    kafkaToGeowave.getKafkaOptions().setMaxPartitionFetchBytes(MAX_MESSAGE_BYTES);
    kafkaToGeowave.setParameters("test-store", "testIndex");
    kafkaToGeowave.execute(params);
    // finish.
    try {
        kafkaToGeowave.getDriver().waitFutures();
    } catch (InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}

Example 5 with AddStoreCommand

use of org.locationtech.geowave.core.store.cli.store.AddStoreCommand in project geowave by locationtech.

the class MapReduceTestUtils method testMapReduceIngest.

public static void testMapReduceIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String format, final String ingestFilePath) throws Exception {
    // ingest gpx data directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    final Thread progressLogger = startProgressLogger();
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    // Ingest Formats
    final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // create temporary config file and use it for hdfs FS URL config
    final File configFile = File.createTempFile("test_mr", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final ConfigHDFSCommand configHdfs = new ConfigHDFSCommand();
    configHdfs.setHdfsUrlParameter(env.getHdfs());
    configHdfs.execute(operationParams);
    final LocalToMapReduceToGeoWaveCommand mrGw = new LocalToMapReduceToGeoWaveCommand();
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        String indexName = "testIndex" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    mrGw.setPluginFormats(ingestFormatOptions);
    mrGw.setParameters(ingestFilePath, env.getHdfsBaseDirectory(), "test-store", indexParam.toString());
    mrGw.getMapReduceOptions().setJobTrackerHostPort(env.getJobtracker());
    mrGw.execute(operationParams);
    progressLogger.interrupt();
}