use of io.georocket.util.JsonParserOperator in project georocket by georocket.
the class ImporterVerticle method importJSON.
/**
* Imports a JSON file from the given input stream into the store
* @param f the JSON file to read
* @param correlationId a unique identifier for this import process
* @param filename the name of the file currently being imported
* @param timestamp denotes when the import process has started
* @param layer the layer where the file should be stored (may be null)
* @param tags the list of tags to attach to the file (may be null)
* @param properties the map of properties to attach to the file (may be null)
* @return a single that will emit when the file has been imported
*/
protected Single<Integer> importJSON(ReadStream<Buffer> f, String correlationId, String filename, long timestamp, String layer, List<String> tags, Map<String, Object> properties) {
UTF8BomFilter bomFilter = new UTF8BomFilter();
StringWindow window = new StringWindow();
GeoJsonSplitter splitter = new GeoJsonSplitter(window);
AtomicInteger processing = new AtomicInteger(0);
return f.toObservable().map(buf -> (io.vertx.core.buffer.Buffer) buf.getDelegate()).map(bomFilter::filter).doOnNext(window::append).lift(new JsonParserOperator()).flatMap(splitter::onEventObservable).flatMapSingle(result -> {
IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, null);
return addToStoreWithPause(result, layer, indexMeta, f, processing);
}).count().toSingle();
}
use of io.georocket.util.JsonParserOperator in project georocket by georocket.
the class GeoJsonSplitterTest method split.
private List<Tuple2<GeoJsonChunkMeta, JsonObject>> split(String file) throws IOException {
byte[] json = IOUtils.toByteArray(GeoJsonSplitterTest.class.getResource(file));
List<Tuple2<GeoJsonChunkMeta, JsonObject>> chunks = new ArrayList<>();
StringWindow window = new StringWindow();
GeoJsonSplitter splitter = new GeoJsonSplitter(window);
Observable.just(json).map(Buffer::buffer).doOnNext(window::append).lift(new JsonParserOperator()).flatMap(splitter::onEventObservable).toBlocking().forEach(result -> {
JsonObject o = new JsonObject(result.getChunk());
chunks.add(Tuple.tuple((GeoJsonChunkMeta) result.getMeta(), o));
});
return chunks;
}
use of io.georocket.util.JsonParserOperator in project georocket by georocket.
the class IndexerVerticle method openChunkToDocument.
/**
* Open a chunk and convert it to an Elasticsearch document. Retry operation
* several times before failing.
* @param path the path to the chunk to open
* @param chunkMeta metadata about the chunk
* @param indexMeta metadata used to index the chunk
* @return an observable that emits the document
*/
private Observable<Map<String, Object>> openChunkToDocument(String path, ChunkMeta chunkMeta, IndexMeta indexMeta) {
return Observable.defer(() -> store.rxGetOne(path).flatMapObservable(chunk -> {
List<? extends IndexerFactory> factories;
Operator<? extends StreamEvent, Buffer> parserOperator;
// select indexers and parser depending on the mime type
String mimeType = chunkMeta.getMimeType();
if (belongsTo(mimeType, "application", "xml") || belongsTo(mimeType, "text", "xml")) {
factories = xmlIndexerFactories;
parserOperator = new XMLParserOperator();
} else if (belongsTo(mimeType, "application", "json")) {
factories = jsonIndexerFactories;
parserOperator = new JsonParserOperator();
} else {
return Observable.error(new NoStackTraceThrowable(String.format("Unexpected mime type '%s' while trying to index " + "chunk '%s'", mimeType, path)));
}
// call meta indexers
Map<String, Object> metaResults = new HashMap<>();
for (MetaIndexerFactory metaIndexerFactory : metaIndexerFactories) {
MetaIndexer metaIndexer = metaIndexerFactory.createIndexer();
metaIndexer.onIndexChunk(path, chunkMeta, indexMeta);
metaResults.putAll(metaIndexer.getResult());
}
// convert chunk to document and close it
return chunkToDocument(chunk, indexMeta.getFallbackCRSString(), parserOperator, factories).doAfterTerminate(chunk::close).doOnNext(doc -> doc.putAll(metaResults));
})).retryWhen(makeRetry());
}
use of io.georocket.util.JsonParserOperator in project georocket by georocket.
the class GeoJsonGenericAttributeIndexerTest method assertIndexed.
/**
* Indexes the given JSON file and checks if the result matches the
* expected properties map
* @param expected the expected properties map
* @param jsonFile the JSON file to parse
* @param context the current test context
* @throws IOException if the JSON file could not be read
*/
private void assertIndexed(Map<String, Object> expected, String jsonFile, TestContext context) throws IOException {
String json;
try (InputStream is = getClass().getResourceAsStream(jsonFile);
Scanner scanner = new Scanner(is, "UTF-8")) {
scanner.useDelimiter("\\A");
json = scanner.next();
}
GeoJsonGenericAttributeIndexer indexer = new GeoJsonGenericAttributeIndexer();
Map<String, Object> expectedMap = ImmutableMap.of("genAttrs", expected);
Async async = context.async();
Observable.just(Buffer.buffer(json)).lift(new JsonParserOperator()).doOnNext(indexer::onEvent).last().subscribe(r -> {
context.assertEquals(expectedMap, indexer.getResult());
async.complete();
}, err -> {
context.fail(err);
});
}
use of io.georocket.util.JsonParserOperator in project georocket by georocket.
the class GeoJsonBoundingBoxIndexerTest method assertIndexed.
/**
* Indexes the given JSON file and checks if the result matches the
* expected bounding box
* @param expected the expected bounding box
* @param jsonFile the JSON file to parse
* @param context the current test context
* @throws IOException if the JSON file could not be read
*/
private void assertIndexed(List<List<Double>> expected, String jsonFile, TestContext context) throws IOException {
String json;
try (InputStream is = getClass().getResourceAsStream(jsonFile);
Scanner scanner = new Scanner(is, "UTF-8")) {
scanner.useDelimiter("\\A");
json = scanner.next();
}
GeoJsonBoundingBoxIndexer indexer = new GeoJsonBoundingBoxIndexer();
Map<String, Object> expectedMap = ImmutableMap.of("bbox", ImmutableMap.of("type", "envelope", "coordinates", expected));
Async async = context.async();
Observable.just(Buffer.buffer(json)).lift(new JsonParserOperator()).doOnNext(indexer::onEvent).last().subscribe(r -> {
context.assertEquals(expectedMap, indexer.getResult());
async.complete();
}, err -> {
context.fail(err);
});
}
Aggregations