use of io.georocket.index.xml.MetaIndexerFactory in project georocket by georocket.
the class IndexerVerticle method openChunkToDocument.
/**
* Open a chunk and convert it to an Elasticsearch document. Retry operation
* several times before failing.
* @param path the path to the chunk to open
* @param chunkMeta metadata about the chunk
* @param indexMeta metadata used to index the chunk
* @return an observable that emits the document
*/
private Observable<Map<String, Object>> openChunkToDocument(String path, ChunkMeta chunkMeta, IndexMeta indexMeta) {
return Observable.defer(() -> store.rxGetOne(path).flatMapObservable(chunk -> {
List<? extends IndexerFactory> factories;
Operator<? extends StreamEvent, Buffer> parserOperator;
// select indexers and parser depending on the mime type
String mimeType = chunkMeta.getMimeType();
if (belongsTo(mimeType, "application", "xml") || belongsTo(mimeType, "text", "xml")) {
factories = xmlIndexerFactories;
parserOperator = new XMLParserOperator();
} else if (belongsTo(mimeType, "application", "json")) {
factories = jsonIndexerFactories;
parserOperator = new JsonParserOperator();
} else {
return Observable.error(new NoStackTraceThrowable(String.format("Unexpected mime type '%s' while trying to index " + "chunk '%s'", mimeType, path)));
}
// call meta indexers
Map<String, Object> metaResults = new HashMap<>();
for (MetaIndexerFactory metaIndexerFactory : metaIndexerFactories) {
MetaIndexer metaIndexer = metaIndexerFactory.createIndexer();
metaIndexer.onIndexChunk(path, chunkMeta, indexMeta);
metaResults.putAll(metaIndexer.getResult());
}
// convert chunk to document and close it
return chunkToDocument(chunk, indexMeta.getFallbackCRSString(), parserOperator, factories).doAfterTerminate(chunk::close).doOnNext(doc -> doc.putAll(metaResults));
})).retryWhen(makeRetry());
}
use of io.georocket.index.xml.MetaIndexerFactory in project georocket by georocket.
the class IndexerVerticle method start.
@Override
public void start(Future<Void> startFuture) {
// True if the indexer and other verticles should report their activities
// to the Vert.x event bus (mostly useful for GeoRocket plug-ins)
reportActivities = config().getBoolean(ConfigConstants.REPORT_ACTIVITIES, false);
maxBulkSize = config().getInteger(ConfigConstants.INDEX_MAX_BULK_SIZE, ConfigConstants.DEFAULT_INDEX_MAX_BULK_SIZE);
maxParallelInserts = config().getInteger(ConfigConstants.INDEX_MAX_PARALLEL_INSERTS, ConfigConstants.DEFAULT_INDEX_MAX_PARALLEL_INSERTS);
// load and copy all indexer factories now and not lazily to avoid
// concurrent modifications to the service loader's internal cache
indexerFactories = ImmutableList.copyOf(ServiceLoader.load(IndexerFactory.class));
xmlIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof XMLIndexerFactory).cast(XMLIndexerFactory.class));
jsonIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof JsonIndexerFactory).cast(JsonIndexerFactory.class));
metaIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof MetaIndexerFactory).cast(MetaIndexerFactory.class));
store = new RxStore(StoreFactory.createStore(getVertx()));
queryCompiler = createQueryCompiler();
queryCompiler.setQueryCompilers(indexerFactories);
new ElasticsearchClientFactory(vertx).createElasticsearchClient(INDEX_NAME).doOnNext(es -> {
client = es;
}).flatMap(v -> client.ensureIndex()).flatMap(v -> ensureMapping()).subscribe(es -> {
registerMessageConsumers();
startFuture.complete();
}, err -> {
startFuture.fail(err);
});
}
Aggregations