Examples with LocalToMapReduceToGeoWaveCommand org.locationtech.geowave.core.ingest.operations.LocalToMapReduceToGeoWaveCommand used on opensource projects

Example 1 with LocalToMapReduceToGeoWaveCommand

use of org.locationtech.geowave.core.ingest.operations.LocalToMapReduceToGeoWaveCommand in project geowave by locationtech.

the class MapReduceTestUtils method testMapReduceIngest.

public static void testMapReduceIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String format, final String ingestFilePath) throws Exception {
    // ingest gpx data directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    final Thread progressLogger = startProgressLogger();
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    // Ingest Formats
    final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // create temporary config file and use it for hdfs FS URL config
    final File configFile = File.createTempFile("test_mr", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final ConfigHDFSCommand configHdfs = new ConfigHDFSCommand();
    configHdfs.setHdfsUrlParameter(env.getHdfs());
    configHdfs.execute(operationParams);
    final LocalToMapReduceToGeoWaveCommand mrGw = new LocalToMapReduceToGeoWaveCommand();
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        String indexName = "testIndex" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    mrGw.setPluginFormats(ingestFormatOptions);
    mrGw.setParameters(ingestFilePath, env.getHdfsBaseDirectory(), "test-store", indexParam.toString());
    mrGw.getMapReduceOptions().setJobTrackerHostPort(env.getJobtracker());
    mrGw.execute(operationParams);
    progressLogger.interrupt();
}

Example 2 with LocalToMapReduceToGeoWaveCommand

use of org.locationtech.geowave.core.ingest.operations.LocalToMapReduceToGeoWaveCommand in project geowave by locationtech.

the class GeoWaveGrpcCoreIngestService method localToMapReduceToGeoWaveCommand.

@Override
public void localToMapReduceToGeoWaveCommand(final org.locationtech.geowave.service.grpc.protobuf.LocalToMapReduceToGeoWaveCommandParametersProtos request, final StreamObserver<org.locationtech.geowave.service.grpc.protobuf.GeoWaveReturnTypesProtos.VoidResponseProtos> responseObserver) {
    final LocalToMapReduceToGeoWaveCommand cmd = new LocalToMapReduceToGeoWaveCommand();
    final Map<FieldDescriptor, Object> m = request.getAllFields();
    GeoWaveGrpcServiceCommandUtil.setGrpcToCommandFields(m, cmd);
    final File configFile = GeoWaveGrpcServiceOptions.geowaveConfigFile;
    final OperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    cmd.prepare(params);
    LOGGER.info("Executing LocalToMapReduceToGeowaveCommand...");
    try {
        cmd.computeResults(params);
        final VoidResponseProtos resp = VoidResponseProtos.newBuilder().build();
        responseObserver.onNext(resp);
        responseObserver.onCompleted();
    } catch (final Exception e) {
        LOGGER.error("Exception encountered executing command", e);
        responseObserver.onError(e);
    }
}