Examples with VectorIngestParametersProtos org.locationtech.geowave.service.grpc.protobuf.VectorIngestParametersProtos used on opensource projects

Example 1 with VectorIngestParametersProtos

use of org.locationtech.geowave.service.grpc.protobuf.VectorIngestParametersProtos in project geowave by locationtech.

the class GeoWaveGrpcTestClient method vectorIngest.

// Vector Service Methods
public void vectorIngest(final int minLat, final int maxLat, final int minLon, final int maxLon, final int latStepDegs, final int lonStepDegs) throws InterruptedException, UnsupportedEncodingException, ParseException {
    LOGGER.info("Performing Vector Ingest...");
    final VectorStoreParametersProtos baseParams = VectorStoreParametersProtos.newBuilder().setStoreName(GeoWaveGrpcTestUtils.storeName).setTypeName(GeoWaveGrpcTestUtils.typeName).setIndexName(GeoWaveGrpcTestUtils.indexName).build();
    final CountDownLatch finishLatch = new CountDownLatch(1);
    final StreamObserver<StringResponseProtos> responseObserver = new StreamObserver<StringResponseProtos>() {

        @Override
        public void onNext(final StringResponseProtos value) {
            try {
                numFeaturesProcessed = Integer.parseInt(value.getResponseValue());
            } catch (final NumberFormatException e) {
            }
            LOGGER.info(value.getResponseValue());
        }

        @Override
        public void onError(final Throwable t) {
            LOGGER.error("Error: Vector Ingest failed.", t);
            finishLatch.countDown();
        }

        @Override
        public void onCompleted() {
            LOGGER.info("Finished Vector Ingest...");
            finishLatch.countDown();
        }
    };
    final StreamObserver<VectorIngestParametersProtos> requestObserver = vectorAsyncStub.vectorIngest(responseObserver);
    // Build up and add features to the request here...
    final VectorIngestParametersProtos.Builder requestBuilder = VectorIngestParametersProtos.newBuilder();
    final FeatureAttributeProtos.Builder attBuilder = FeatureAttributeProtos.newBuilder();
    for (int longitude = minLon; longitude <= maxLon; longitude += lonStepDegs) {
        for (int latitude = minLat; latitude <= maxLat; latitude += latStepDegs) {
            attBuilder.setValGeometry(copyFrom(new WKBWriter().write(GeometryUtils.GEOMETRY_FACTORY.createPoint(new Coordinate(longitude, latitude)))));
            requestBuilder.putFeature("geometry", attBuilder.build());
            final TimeZone tz = TimeZone.getTimeZone("UTC");
            // Quoted "Z" to
            final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
            // indicate UTC,
            // no timezone offset
            df.setTimeZone(tz);
            attBuilder.setValDate(Timestamps.fromMillis((df.parse(GeoWaveGrpcTestUtils.temporalQueryStartTime).getTime() + df.parse(GeoWaveGrpcTestUtils.temporalQueryEndTime).getTime()) / 2));
            requestBuilder.putFeature("TimeStamp", attBuilder.build());
            attBuilder.setValDouble(latitude);
            requestBuilder.putFeature("Latitude", attBuilder.build());
            attBuilder.setValDouble(longitude);
            requestBuilder.putFeature("Longitude", attBuilder.build());
            final VectorIngestParametersProtos params = requestBuilder.setBaseParams(baseParams).build();
            requestObserver.onNext(params);
            if (finishLatch.getCount() == 0) {
                // be thrown away.
                return;
            }
        }
    }
    // Mark the end of requests
    requestObserver.onCompleted();
    // Receiving happens asynchronously
    if (!finishLatch.await(15, TimeUnit.MINUTES)) {
        LOGGER.warn("Vector Ingest can not finish within 5 minutes");
    }
}

Example 2 with VectorIngestParametersProtos

use of org.locationtech.geowave.service.grpc.protobuf.VectorIngestParametersProtos in project geowave by locationtech.

the class GeoWaveGrpcVectorService method vectorIngest.

@Override
public StreamObserver<VectorIngestParametersProtos> vectorIngest(final StreamObserver<StringResponseProtos> responseObserver) {
    return new StreamObserver<VectorIngestParametersProtos>() {

        private boolean firstFeature = true;

        private String storeName = null;

        private DataStore dataStore = null;

        private String typeName = null;

        private String indexName = null;

        private Writer<SimpleFeature> writer = null;

        private DataTypeAdapter adapter = null;

        private Index pIndex = null;

        private SimpleFeatureTypeBuilder typeBuilder = null;

        private SimpleFeatureBuilder featureBuilder = null;

        private static final int batchSize = 100;

        private int batchCount = 0;

        private int totalCount = 0;

        @Override
        public void onNext(final VectorIngestParametersProtos f) {
            if (firstFeature) {
                firstFeature = false;
                // parse top level required parameters
                storeName = f.getBaseParams().getStoreName();
                final StoreLoader storeLoader = new StoreLoader(storeName);
                typeName = f.getBaseParams().getTypeName();
                indexName = f.getBaseParams().getIndexName();
                // This only needs to happen once
                if (typeBuilder == null) {
                    typeBuilder = new SimpleFeatureTypeBuilder();
                    for (final Map.Entry<String, FeatureAttributeProtos> mapEntry : f.getFeatureMap().entrySet()) {
                        switch(mapEntry.getValue().getValueCase()) {
                            case VALSTRING:
                                {
                                    typeBuilder.add(mapEntry.getKey(), String.class);
                                    break;
                                }
                            case VALINT32:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Integer.class);
                                    break;
                                }
                            case VALINT64:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Long.class);
                                    break;
                                }
                            case VALFLOAT:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Float.class);
                                    break;
                                }
                            case VALDOUBLE:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Double.class);
                                    break;
                                }
                            case VALDATE:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Date.class);
                                    break;
                                }
                            case VALGEOMETRY:
                                {
                                    typeBuilder.add(mapEntry.getKey(), Geometry.class);
                                    break;
                                }
                            default:
                                break;
                        }
                        ;
                    }
                }
                // This a factory class that builds simple feature objects
                // based
                // on the
                // type
                typeBuilder.setName(typeName);
                final SimpleFeatureType featureType = typeBuilder.buildFeatureType();
                featureBuilder = new SimpleFeatureBuilder(featureType);
                // get a handle to the relevant stores
                if (!storeLoader.loadFromConfig(GeoWaveGrpcServiceOptions.geowaveConfigFile)) {
                    throw new ParameterException("Cannot find store name: " + storeLoader.getStoreName());
                }
                dataStore = storeLoader.createDataStore();
                final PersistentAdapterStore adapterStore = storeLoader.createAdapterStore();
                final InternalAdapterStore internalAdapterStore = storeLoader.createInternalAdapterStore();
                final Short internalAdapterId = internalAdapterStore.getAdapterId(typeName);
                if (internalAdapterId != null) {
                    adapter = adapterStore.getAdapter(internalAdapterId);
                } else {
                    adapter = null;
                }
                if (adapter == null) {
                    adapter = new FeatureDataAdapter(featureType);
                }
                // Load the Indexes
                final List<Index> indices = DataStoreUtils.loadIndices(storeLoader.createIndexStore(), indexName);
                // assuming one index for now
                // (PrimaryIndex)
                pIndex = indices.get(0);
                // indexStore.getIndex(indexId);
                if (pIndex == null) {
                    throw new ParameterException("Failed to instantiate primary index");
                }
                // create index writer to actually write data
                dataStore.addType(adapter, pIndex);
                writer = dataStore.createWriter(adapter.getTypeName());
            }
            // Set the values for all the attributes in the feature
            for (final Map.Entry<String, FeatureAttributeProtos> attribute : f.getFeatureMap().entrySet()) {
                switch(attribute.getValue().getValueCase()) {
                    case VALSTRING:
                        {
                            featureBuilder.set(attribute.getKey(), attribute.getValue().getValString());
                            break;
                        }
                    case VALINT32:
                        {
                            featureBuilder.set(attribute.getKey(), attribute.getValue().getValInt32());
                            break;
                        }
                    case VALINT64:
                        {
                            featureBuilder.set(attribute.getKey(), attribute.getValue().getValInt64());
                            break;
                        }
                    case VALFLOAT:
                        {
                            featureBuilder.set(attribute.getKey(), attribute.getValue().getValFloat());
                            break;
                        }
                    case VALDOUBLE:
                        {
                            featureBuilder.set(attribute.getKey(), attribute.getValue().getValDouble());
                            break;
                        }
                    case VALDATE:
                        {
                            featureBuilder.set(attribute.getKey(), new Date(Timestamps.toMillis(attribute.getValue().getValDate())));
                            break;
                        }
                    case VALGEOMETRY:
                        {
                            Geometry geom = null;
                            try {
                                geom = new WKBReader(JTSFactoryFinder.getGeometryFactory()).read(attribute.getValue().getValGeometry().toByteArray());
                            } catch (FactoryRegistryException | org.locationtech.jts.io.ParseException e) {
                                LOGGER.error("Failed to parse string for geometry", e);
                            }
                            if (geom != null) {
                                featureBuilder.set(attribute.getKey(), geom);
                            }
                            break;
                        }
                    default:
                        break;
                }
                ;
            }
            final SimpleFeature sf = featureBuilder.buildFeature(String.valueOf(totalCount));
            writer.write(sf);
            // onComplete and onError
            if (++batchCount >= batchSize) {
                // writer.flush();
                batchCount = 0;
            }
            final StringResponseProtos resp = StringResponseProtos.newBuilder().setResponseValue(String.valueOf(++totalCount)).build();
            responseObserver.onNext(resp);
        }

        @Override
        public void onError(final Throwable t) {
            LOGGER.error("Exception encountered during vectorIngest", t);
            writer.flush();
            writer.close();
            final StringResponseProtos resp = StringResponseProtos.newBuilder().setResponseValue("Error during ingest: ").build();
            responseObserver.onNext(resp);
            responseObserver.onCompleted();
        }

        @Override
        public void onCompleted() {
            writer.flush();
            writer.close();
            final StringResponseProtos resp = StringResponseProtos.newBuilder().setResponseValue("Ingest completed successfully").build();
            responseObserver.onNext(resp);
            responseObserver.onCompleted();
        }
    };
}