Examples with KafkaToGeoWaveCommand org.locationtech.geowave.core.ingest.operations.KafkaToGeoWaveCommand used on opensource projects

Example 1 with KafkaToGeoWaveCommand

use of org.locationtech.geowave.core.ingest.operations.KafkaToGeoWaveCommand in project geowave by locationtech.

the class KafkaTestUtils method testKafkaIngest.

public static void testKafkaIngest(final DataStorePluginOptions options, final boolean spatialTemporal, final String ingestFilePath) throws Exception {
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    // // FIXME
    // final String[] args = StringUtils.split("-kafkaingest" +
    // " -f gpx -batchSize 1 -consumerTimeoutMs 5000 -reconnectOnTimeout
    // -groupId testGroup"
    // + " -autoOffsetReset smallest -fetchMessageMaxBytes " +
    // MAX_MESSAGE_BYTES +
    // " -zookeeperConnect " + zookeeper + " -" +
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin("gpx");
    // Indexes
    final IndexPluginOptions indexOption = new IndexPluginOptions();
    indexOption.selectPlugin((spatialTemporal ? "spatial_temporal" : "spatial"));
    // Execute Command
    final KafkaToGeoWaveCommand kafkaToGeowave = new KafkaToGeoWaveCommand();
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(options);
    addStore.execute(params);
    final IndexStore indexStore = options.createIndexStore();
    final DataStore dataStore = options.createDataStore();
    if (indexStore.getIndex("testIndex") == null) {
        indexOption.setName("testIndex");
        dataStore.addIndex(indexOption.createIndex(dataStore));
    }
    kafkaToGeowave.setPluginFormats(ingestFormatOptions);
    kafkaToGeowave.getKafkaOptions().setBootstrapServers(KafkaTestEnvironment.getInstance().getBootstrapServers());
    kafkaToGeowave.getKafkaOptions().setConsumerTimeoutMs("5000");
    kafkaToGeowave.getKafkaOptions().setReconnectOnTimeout(false);
    kafkaToGeowave.getKafkaOptions().setGroupId("testGroup");
    kafkaToGeowave.getKafkaOptions().setAutoOffsetReset("earliest");
    kafkaToGeowave.getKafkaOptions().setMaxPartitionFetchBytes(MAX_MESSAGE_BYTES);
    kafkaToGeowave.setParameters("test-store", "testIndex");
    kafkaToGeowave.execute(params);
    // finish.
    try {
        kafkaToGeowave.getDriver().waitFutures();
    } catch (InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with KafkaToGeoWaveCommand

use of org.locationtech.geowave.core.ingest.operations.KafkaToGeoWaveCommand in project geowave by locationtech.

the class GeoWaveGrpcCoreIngestService method kafkaToGeoWaveCommand.

@Override
public void kafkaToGeoWaveCommand(final org.locationtech.geowave.service.grpc.protobuf.KafkaToGeoWaveCommandParametersProtos request, final StreamObserver<org.locationtech.geowave.service.grpc.protobuf.GeoWaveReturnTypesProtos.VoidResponseProtos> responseObserver) {
    final KafkaToGeoWaveCommand cmd = new KafkaToGeoWaveCommand();
    final Map<FieldDescriptor, Object> m = request.getAllFields();
    GeoWaveGrpcServiceCommandUtil.setGrpcToCommandFields(m, cmd);
    final File configFile = GeoWaveGrpcServiceOptions.geowaveConfigFile;
    final OperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    cmd.prepare(params);
    LOGGER.info("Executing KafkaToGeowaveCommand...");
    try {
        cmd.computeResults(params);
        final VoidResponseProtos resp = VoidResponseProtos.newBuilder().build();
        responseObserver.onNext(resp);
        responseObserver.onCompleted();
    } catch (final Exception e) {
        LOGGER.error("Exception encountered executing command", e);
        responseObserver.onError(e);
    }
}