Examples with Writer org.locationtech.geowave.core.store.api.Writer used on opensource projects

Example 1 with Writer

use of org.locationtech.geowave.core.store.api.Writer in project geowave by locationtech.

the class KMeansUtils method writeClusterHulls.

public static DataTypeAdapter writeClusterHulls(final JavaRDD<Vector> inputCentroids, final KMeansModel clusterModel, final DataStorePluginOptions outputDataStore, final String hullAdapterName, final boolean computeMetadata) {
    final JavaPairRDD<Integer, Iterable<Vector>> groupByRdd = KMeansHullGenerator.groupByIndex(inputCentroids, clusterModel);
    final JavaPairRDD<Integer, Geometry> hullRdd = KMeansHullGenerator.generateHullsRDD(groupByRdd);
    final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder();
    typeBuilder.setName(hullAdapterName);
    typeBuilder.setNamespaceURI(BasicFeatureTypes.DEFAULT_NAMESPACE);
    try {
        typeBuilder.setCRS(CRS.decode("EPSG:4326", true));
    } catch (final FactoryException e) {
        LOGGER.error(e.getMessage(), e);
    }
    final AttributeTypeBuilder attrBuilder = new AttributeTypeBuilder();
    typeBuilder.add(attrBuilder.binding(Geometry.class).nillable(false).buildDescriptor(Geometry.class.getName().toString()));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("ClusterIndex"));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("Count"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Area"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Density"));
    final SimpleFeatureType sfType = typeBuilder.buildFeatureType();
    final SimpleFeatureBuilder sfBuilder = new SimpleFeatureBuilder(sfType);
    final FeatureDataAdapter featureAdapter = new FeatureDataAdapter(sfType);
    final DataStore featureStore = outputDataStore.createDataStore();
    final Index featureIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final PolygonAreaCalculator polyCalc = (computeMetadata ? new PolygonAreaCalculator() : null);
    featureStore.addType(featureAdapter, featureIndex);
    try (Writer writer = featureStore.createWriter(featureAdapter.getTypeName())) {
        for (final Tuple2<Integer, Geometry> hull : hullRdd.collect()) {
            final Integer index = hull._1;
            final Geometry geom = hull._2;
            sfBuilder.set(Geometry.class.getName(), geom);
            sfBuilder.set("ClusterIndex", index);
            int count = 0;
            double area = 0.0;
            double density = 0.0;
            if (computeMetadata) {
                for (final Iterable<Vector> points : groupByRdd.lookup(index)) {
                    final Vector[] pointVec = Iterables.toArray(points, Vector.class);
                    count += pointVec.length;
                }
                try {
                    // HP Fortify "NULL Pointer Dereference" false positive
                    // Exception handling will catch if polyCalc is null
                    area = polyCalc.getAreaDensify(geom);
                    density = count / area;
                } catch (final Exception e) {
                    LOGGER.error("Problem computing polygon area: " + e.getMessage());
                }
            }
            sfBuilder.set("Count", count);
            sfBuilder.set("Area", area);
            sfBuilder.set("Density", density);
            final SimpleFeature sf = sfBuilder.buildFeature("Hull-" + index);
            writer.write(sf);
        }
    }
    return featureAdapter;
}

Example 2 with Writer

use of org.locationtech.geowave.core.store.api.Writer in project geowave by locationtech.

the class KMeansDistortionMapReduceTest method ingest.

private void ingest(final DataStore dataStore, final FeatureDataAdapter adapter, final Index index, final SimpleFeature feature) throws IOException {
    dataStore.addType(adapter, index);
    try (Writer writer = dataStore.createWriter(adapter.getTypeName())) {
        writer.write(feature);
        writer.close();
    }
}

Example 3 with Writer

use of org.locationtech.geowave.core.store.api.Writer in project geowave by locationtech.

the class SqlResultsWriter method writeResults.

public void writeResults(String typeName) {
    if (typeName == null) {
        typeName = DEFAULT_TYPE_NAME;
        LOGGER.warn("Using default type name (adapter id): '" + DEFAULT_TYPE_NAME + "' for SQL output");
    }
    final StructType schema = results.schema();
    final SimpleFeatureType featureType = SchemaConverter.schemaToFeatureType(schema, typeName);
    final SimpleFeatureBuilder sfBuilder = new SimpleFeatureBuilder(featureType);
    final FeatureDataAdapter featureAdapter = new FeatureDataAdapter(featureType);
    final DataStore featureStore = outputDataStore.createDataStore();
    final Index featureIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    featureStore.addType(featureAdapter, featureIndex);
    try (Writer writer = featureStore.createWriter(featureAdapter.getTypeName())) {
        final List<Row> rows = results.collectAsList();
        for (int r = 0; r < rows.size(); r++) {
            final Row row = rows.get(r);
            for (int i = 0; i < schema.fields().length; i++) {
                final StructField field = schema.apply(i);
                final Object rowObj = row.apply(i);
                if (rowObj != null) {
                    if (field.name().equals("geom")) {
                        final Geometry geom = (Geometry) rowObj;
                        sfBuilder.set("geom", geom);
                    } else if (field.dataType() == DataTypes.TimestampType) {
                        final long millis = ((Timestamp) rowObj).getTime();
                        final Date date = new Date(millis);
                        sfBuilder.set(field.name(), date);
                    } else {
                        sfBuilder.set(field.name(), rowObj);
                    }
                }
            }
            final SimpleFeature sf = sfBuilder.buildFeature("result-" + nf.format(r));
            writer.write(sf);
        }
    }
}

Example 4 with Writer

use of org.locationtech.geowave.core.store.api.Writer in project geowave by locationtech.

the class IndexCompositeWriter method internalWrite.

protected WriteResults internalWrite(final T entry, final Function<Writer<T>, WriteResults> internalWriter) {
    final Map<String, List<SinglePartitionInsertionIds>> insertionIdsPerIndex = new HashMap<>();
    for (final Writer<T> indexWriter : writers) {
        final WriteResults ids = internalWriter.apply(indexWriter);
        for (final String indexName : ids.getWrittenIndexNames()) {
            List<SinglePartitionInsertionIds> partitionInsertionIds = insertionIdsPerIndex.get(indexName);
            if (partitionInsertionIds == null) {
                partitionInsertionIds = new ArrayList<>();
                insertionIdsPerIndex.put(indexName, partitionInsertionIds);
            }
            partitionInsertionIds.addAll(ids.getInsertionIdsWritten(indexName).getPartitionKeys());
        }
    }
    return new WriteResults(Maps.transformValues(insertionIdsPerIndex, v -> new InsertionIds(v)));
}

Example 5 with Writer

use of org.locationtech.geowave.core.store.api.Writer in project geowave by locationtech.

the class RasterIngestRunner method processPreviousScene.

@SuppressWarnings({ "rawtypes", "unchecked" })
protected void processPreviousScene() {
    if (!ingestOptions.isCoveragePerBand()) {
        // ingest as single image for all bands
        if (!lastSceneBands.isEmpty()) {
            // we are sorting by band name to ensure a consistent order for
            // bands
            final TreeMap<String, RasterBandData> sceneData = new TreeMap<>();
            Writer writer;
            // get coverage info, ensuring that all coverage names are the
            // same
            String coverageName = null;
            for (final SimpleFeature band : lastSceneBands) {
                RasterBandData bandData;
                try {
                    bandData = getBandData(band);
                    if (coverageName == null) {
                        coverageName = bandData.name;
                    } else if (!coverageName.equals(bandData.name)) {
                        LOGGER.warn("Unable to use band data as the band coverage name '" + bandData.name + "' is unexpectedly different from default name '" + coverageName + "'");
                    }
                    final String bandName = band.getAttribute(BandFeatureIterator.BAND_ATTRIBUTE_NAME).toString();
                    sceneData.put(bandName, bandData);
                } catch (IOException | TemplateException e) {
                    LOGGER.warn("Unable to read band data", e);
                }
            }
            if (coverageName == null) {
                LOGGER.warn("No valid bands found for scene");
                lastSceneBands.clear();
                return;
            }
            final GridCoverage2D mergedCoverage;
            if (sceneData.size() == 1) {
                mergedCoverage = sceneData.firstEntry().getValue().coverage;
            } else {
                final CoverageProcessor processor = CoverageProcessor.getInstance();
                final AbstractOperation op = (AbstractOperation) processor.getOperation("BandMerge");
                final ParameterValueGroup params = op.getParameters();
                final List<GridCoverage2D> sources = new ArrayList<>();
                for (final RasterBandData bandData : sceneData.values()) {
                    sources.add(bandData.coverage);
                }
                params.parameter("Sources").setValue(sources);
                params.parameter(BandMerge.TRANSFORM_CHOICE).setValue(TransformList.FIRST.toString());
                mergedCoverage = (GridCoverage2D) op.doOperation(params, null);
            }
            final String[] thisSceneBands = sceneData.keySet().toArray(new String[] {});
            if (bandsIngested == null) {
                // this means this is the first scene
                // setup adapter and other required info
                final Map<String, String> metadata = new HashMap<>();
                final double[][] noDataValues = new double[sceneData.size()][];
                int b = 0;
                // merge metadata from all readers
                for (final RasterBandData bandData : sceneData.values()) {
                    try {
                        final String[] metadataNames = bandData.reader.getMetadataNames();
                        if ((metadataNames != null) && (metadataNames.length > 0)) {
                            for (final String metadataName : metadataNames) {
                                metadata.put(metadataName, bandData.reader.getMetadataValue(metadataName));
                            }
                        }
                    } catch (final Exception e) {
                        LOGGER.warn("Unable to get metadata for coverage '" + coverageName + "'.", e);
                    }
                    noDataValues[b++] = new double[] { bandData.nodataValue };
                }
                final RasterDataAdapter adapter = new RasterDataAdapter(coverageName, metadata, mergedCoverage, ingestOptions.getTileSize(), ingestOptions.isCreatePyramid(), ingestOptions.isCreateHistogram(), noDataValues, new NoDataMergeStrategy());
                store.addType(adapter, indices);
                writer = store.createWriter(adapter.getTypeName());
                writerCache.put(coverageName, writer);
                bandsIngested = thisSceneBands;
            } else if (!Arrays.equals(bandsIngested, thisSceneBands)) {
                LOGGER.warn("The bands in this scene ('" + Arrays.toString(thisSceneBands) + "') differ from the previous scene ('" + Arrays.toString(bandsIngested) + "').  To merge bands all scenes must use the same bands.  Skipping scene'" + lastSceneBands.get(0).getAttribute(SceneFeatureIterator.ENTITY_ID_ATTRIBUTE_NAME) + "'.");
                lastSceneBands.clear();
                return;
            } else {
                writer = writerCache.get(coverageName);
                if (writer == null) {
                    LOGGER.warn("Unable to find writer for coverage '" + coverageName + "'.  Skipping scene'" + lastSceneBands.get(0).getAttribute(SceneFeatureIterator.ENTITY_ID_ATTRIBUTE_NAME) + "'.");
                    lastSceneBands.clear();
                    return;
                }
            }
            writer.write(mergedCoverage);
            lastSceneBands.clear();
        }
    }
}