Examples with ConfigAWSCommand org.locationtech.geowave.core.ingest.operations.ConfigAWSCommand used on opensource projects

Example 1 with ConfigAWSCommand

use of org.locationtech.geowave.core.ingest.operations.ConfigAWSCommand in project geowave by locationtech.

the class TestUtils method testS3LocalIngest.

public static void testS3LocalIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String s3Url, final String ingestFilePath, final String format, final int nthreads) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    final File configFile = File.createTempFile("test_s3_local_ingest", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        final String indexName = "test-index" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    final ConfigAWSCommand configS3 = new ConfigAWSCommand();
    configS3.setS3UrlParameter(s3Url);
    configS3.execute(operationParams);
    // Create the command and execute.
    final LocalToGeoWaveCommand localIngester = new LocalToGeoWaveCommand();
    localIngester.setPluginFormats(ingestFormatOptions);
    localIngester.setParameters(ingestFilePath, "test-store", indexParam.toString());
    localIngester.setThreads(nthreads);
    localIngester.execute(operationParams);
    verifyStats(dataStore);
}

Example 2 with ConfigAWSCommand

use of org.locationtech.geowave.core.ingest.operations.ConfigAWSCommand in project geowave by locationtech.

the class TestUtils method testSparkIngest.

public static void testSparkIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String s3Url, final String ingestFilePath, final String format) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Indexes
    final String indexes = dimensionalityType.getDimensionalityArg();
    final File configFile = File.createTempFile("test_spark_ingest", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final ConfigAWSCommand configS3 = new ConfigAWSCommand();
    configS3.setS3UrlParameter(s3Url);
    configS3.execute(operationParams);
    final LocalInputCommandLineOptions localOptions = new LocalInputCommandLineOptions();
    localOptions.setFormats(format);
    final SparkCommandLineOptions sparkOptions = new SparkCommandLineOptions();
    sparkOptions.setAppName("SparkIngestTest");
    sparkOptions.setMaster("local");
    sparkOptions.setHost("localhost");
    // Create the command and execute.
    final SparkIngestDriver sparkIngester = new SparkIngestDriver();
    final Properties props = new Properties();
    dataStore.save(props, DataStorePluginOptions.getStoreNamespace("test"));
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    for (final String indexType : indexTypes) {
        if (indexStore.getIndex(indexType) == null) {
            final IndexPluginOptions pluginOptions = new IndexPluginOptions();
            pluginOptions.selectPlugin(indexType);
            pluginOptions.setName(indexType);
            pluginOptions.save(props, IndexPluginOptions.getIndexNamespace(indexType));
            geowaveDataStore.addIndex(pluginOptions.createIndex(geowaveDataStore));
        }
    }
    props.setProperty(ConfigAWSCommand.AWS_S3_ENDPOINT_URL, s3Url);
    sparkIngester.runOperation(configFile, localOptions, "test", indexes, new VisibilityOptions(), sparkOptions, ingestFilePath, new JCommander().getConsole());
    verifyStats(dataStore);
}

Example 3 with ConfigAWSCommand

use of org.locationtech.geowave.core.ingest.operations.ConfigAWSCommand in project geowave by locationtech.

the class GeoWaveGrpcIT method init.

protected void init() throws Exception {
    Configurator.setRootLevel(Level.WARN);
    ZipUtils.unZipFile(new File(BasicKafkaIT.class.getClassLoader().getResource(TEST_DATA_ZIP_RESOURCE_PATH).toURI()), TestUtils.TEST_CASE_BASE);
    // set up the config file for the services
    configFile = File.createTempFile("test_config", null);
    GeoWaveGrpcServiceOptions.geowaveConfigFile = configFile;
    operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    // add a store and index manually before we try to ingest
    // this accurately simulates how the services will perform the ingest
    // from config file parameters (as opposed to programatic
    // creation/loading)
    final AddStoreCommand command = new AddStoreCommand();
    command.setParameters(GeoWaveGrpcTestUtils.storeName);
    command.setPluginOptions(dataStore);
    command.execute(operationParams);
    // finally add an output store for things like KDE etc
    final AddStoreCommand commandOut = new AddStoreCommand();
    commandOut.setParameters(GeoWaveGrpcTestUtils.outputStoreName);
    commandOut.setPluginOptions(dataStore);
    commandOut.execute(operationParams);
    // set up s3
    final ConfigAWSCommand configS3 = new ConfigAWSCommand();
    configS3.setS3UrlParameter("s3.amazonaws.com");
    configS3.execute(operationParams);
    // mimic starting the server from command line
    final StartGrpcServerCommand startCmd = new StartGrpcServerCommand();
    final StartGrpcServerCommandOptions grpcCmdOpts = new StartGrpcServerCommandOptions();
    grpcCmdOpts.setPort(GeoWaveGrpcServiceOptions.port);
    grpcCmdOpts.setNonBlocking(true);
    startCmd.setCommandOptions(grpcCmdOpts);
    startCmd.execute(operationParams);
    // fire up the client
    client = new GeoWaveGrpcTestClient(GeoWaveGrpcServiceOptions.host, GeoWaveGrpcServiceOptions.port);
    LOGGER.warn("-----------------------------------------");
    LOGGER.warn("*                                       *");
    LOGGER.warn("* FINISHED Init  *");
    LOGGER.warn("*         " + ((System.currentTimeMillis() - startMillis) / 1000) + "s elapsed.                 *");
    LOGGER.warn("*                                       *");
    LOGGER.warn("-----------------------------------------");
}