Search in sources :

Example 1 with RxStore

use of io.georocket.storage.RxStore in project georocket by georocket.

the class ImporterVerticle method start.

@Override
public void start() {
    log.info("Launching importer ...");
    reportActivities = config().getBoolean(ConfigConstants.REPORT_ACTIVITIES, false);
    store = new RxStore(StoreFactory.createStore(getVertx()));
    String storagePath = config().getString(ConfigConstants.STORAGE_FILE_PATH);
    incoming = storagePath + "/incoming";
    vertx.eventBus().<JsonObject>localConsumer(AddressConstants.IMPORTER_IMPORT).toObservable().onBackpressureBuffer().flatMapSingle(msg -> {
        // call onImport() but ignore errors. onImport() will handle errors for us.
        return onImport(msg).onErrorReturn(err -> null);
    }, false, MAX_PARALLEL_IMPORTS).subscribe(v -> {
    // ignore
    }, err -> {
        // This is bad. It will unsubscribe the consumer from the eventbus!
        // Should never happen anyhow. If it does, something else has
        // completely gone wrong.
        log.fatal("Could not import file", err);
    });
}
Also used : Buffer(io.vertx.rxjava.core.buffer.Buffer) IndexMeta(io.georocket.storage.IndexMeta) StringWindow(io.georocket.util.StringWindow) XMLParserOperator(io.georocket.util.XMLParserOperator) ChunkMeta(io.georocket.storage.ChunkMeta) Window(io.georocket.util.Window) LoggerFactory(io.vertx.core.logging.LoggerFactory) RxStore(io.georocket.storage.RxStore) AbstractVerticle(io.vertx.rxjava.core.AbstractVerticle) Single(rx.Single) JsonParserOperator(io.georocket.util.JsonParserOperator) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Result(io.georocket.input.Splitter.Result) Map(java.util.Map) FileSystem(io.vertx.rxjava.core.file.FileSystem) JsonObject(io.vertx.core.json.JsonObject) StoreFactory(io.georocket.storage.StoreFactory) UTF8BomFilter(io.georocket.util.UTF8BomFilter) Logger(io.vertx.core.logging.Logger) Message(io.vertx.rxjava.core.eventbus.Message) XMLSplitter(io.georocket.input.xml.XMLSplitter) OpenOptions(io.vertx.core.file.OpenOptions) GeoJsonSplitter(io.georocket.input.geojson.GeoJsonSplitter) NoStackTraceThrowable(io.vertx.core.impl.NoStackTraceThrowable) FirstLevelSplitter(io.georocket.input.xml.FirstLevelSplitter) Collectors(java.util.stream.Collectors) XMLCRSIndexer(io.georocket.index.xml.XMLCRSIndexer) JsonArray(io.vertx.core.json.JsonArray) List(java.util.List) Stream(java.util.stream.Stream) MimeTypeUtils.belongsTo(io.georocket.util.MimeTypeUtils.belongsTo) ReadStream(io.vertx.rxjava.core.streams.ReadStream) AddressConstants(io.georocket.constants.AddressConstants) RxUtils(io.georocket.util.RxUtils) ConfigConstants(io.georocket.constants.ConfigConstants) RxStore(io.georocket.storage.RxStore) JsonObject(io.vertx.core.json.JsonObject)

Example 2 with RxStore

use of io.georocket.storage.RxStore in project georocket by georocket.

the class IndexerVerticle method start.

@Override
public void start(Future<Void> startFuture) {
    // True if the indexer and other verticles should report their activities
    // to the Vert.x event bus (mostly useful for GeoRocket plug-ins)
    reportActivities = config().getBoolean(ConfigConstants.REPORT_ACTIVITIES, false);
    maxBulkSize = config().getInteger(ConfigConstants.INDEX_MAX_BULK_SIZE, ConfigConstants.DEFAULT_INDEX_MAX_BULK_SIZE);
    maxParallelInserts = config().getInteger(ConfigConstants.INDEX_MAX_PARALLEL_INSERTS, ConfigConstants.DEFAULT_INDEX_MAX_PARALLEL_INSERTS);
    // load and copy all indexer factories now and not lazily to avoid
    // concurrent modifications to the service loader's internal cache
    indexerFactories = ImmutableList.copyOf(ServiceLoader.load(IndexerFactory.class));
    xmlIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof XMLIndexerFactory).cast(XMLIndexerFactory.class));
    jsonIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof JsonIndexerFactory).cast(JsonIndexerFactory.class));
    metaIndexerFactories = ImmutableList.copyOf(Seq.seq(indexerFactories).filter(f -> f instanceof MetaIndexerFactory).cast(MetaIndexerFactory.class));
    store = new RxStore(StoreFactory.createStore(getVertx()));
    queryCompiler = createQueryCompiler();
    queryCompiler.setQueryCompilers(indexerFactories);
    new ElasticsearchClientFactory(vertx).createElasticsearchClient(INDEX_NAME).doOnNext(es -> {
        client = es;
    }).flatMap(v -> client.ensureIndex()).flatMap(v -> ensureMapping()).subscribe(es -> {
        registerMessageConsumers();
        startFuture.complete();
    }, err -> {
        startFuture.fail(err);
    });
}
Also used : MetaIndexer(io.georocket.index.xml.MetaIndexer) GeoJsonChunkMeta(io.georocket.storage.GeoJsonChunkMeta) IndexMeta(io.georocket.storage.IndexMeta) StreamEvent(io.georocket.util.StreamEvent) XMLChunkMeta(io.georocket.storage.XMLChunkMeta) XMLParserOperator(io.georocket.util.XMLParserOperator) ChunkMeta(io.georocket.storage.ChunkMeta) RxStore(io.georocket.storage.RxStore) Tuple2(org.jooq.lambda.tuple.Tuple2) Tuple3(org.jooq.lambda.tuple.Tuple3) JsonParserOperator(io.georocket.util.JsonParserOperator) Map(java.util.Map) JsonObject(io.vertx.core.json.JsonObject) Logger(io.vertx.core.logging.Logger) MetaIndexerFactory(io.georocket.index.xml.MetaIndexerFactory) Message(io.vertx.rxjava.core.eventbus.Message) ServiceLoader(java.util.ServiceLoader) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) List(java.util.List) ElasticsearchClientFactory(io.georocket.index.elasticsearch.ElasticsearchClientFactory) Stream(java.util.stream.Stream) Tuple(org.jooq.lambda.tuple.Tuple) MapUtils(io.georocket.util.MapUtils) Buffer(io.vertx.core.buffer.Buffer) MimeTypeUtils.belongsTo(io.georocket.util.MimeTypeUtils.belongsTo) RxHelper(io.vertx.rx.java.RxHelper) AddressConstants(io.georocket.constants.AddressConstants) ChunkReadStream(io.georocket.storage.ChunkReadStream) Operator(rx.Observable.Operator) HashMap(java.util.HashMap) Seq(org.jooq.lambda.Seq) DefaultQueryCompiler(io.georocket.query.DefaultQueryCompiler) LoggerFactory(io.vertx.core.logging.LoggerFactory) ArrayList(java.util.ArrayList) AbstractVerticle(io.vertx.rxjava.core.AbstractVerticle) Observable(rx.Observable) Func1(rx.functions.Func1) ImmutableList(com.google.common.collect.ImmutableList) XMLIndexerFactory(io.georocket.index.xml.XMLIndexerFactory) StoreFactory(io.georocket.storage.StoreFactory) JsonIndexerFactory(io.georocket.index.xml.JsonIndexerFactory) NoStackTraceThrowable(io.vertx.core.impl.NoStackTraceThrowable) ThrowableHelper.throwableToMessage(io.georocket.util.ThrowableHelper.throwableToMessage) StreamIndexer(io.georocket.index.xml.StreamIndexer) TimeUnit(java.util.concurrent.TimeUnit) JsonArray(io.vertx.core.json.JsonArray) ThrowableHelper.throwableToCode(io.georocket.util.ThrowableHelper.throwableToCode) ElasticsearchClient(io.georocket.index.elasticsearch.ElasticsearchClient) RxUtils(io.georocket.util.RxUtils) DefaultMetaIndexerFactory(io.georocket.index.generic.DefaultMetaIndexerFactory) ConfigConstants(io.georocket.constants.ConfigConstants) JsonChunkMeta(io.georocket.storage.JsonChunkMeta) ElasticsearchClientFactory(io.georocket.index.elasticsearch.ElasticsearchClientFactory) JsonIndexerFactory(io.georocket.index.xml.JsonIndexerFactory) MetaIndexerFactory(io.georocket.index.xml.MetaIndexerFactory) DefaultMetaIndexerFactory(io.georocket.index.generic.DefaultMetaIndexerFactory) RxStore(io.georocket.storage.RxStore) XMLIndexerFactory(io.georocket.index.xml.XMLIndexerFactory)

Aggregations

AddressConstants (io.georocket.constants.AddressConstants)2 ConfigConstants (io.georocket.constants.ConfigConstants)2 ChunkMeta (io.georocket.storage.ChunkMeta)2 IndexMeta (io.georocket.storage.IndexMeta)2 RxStore (io.georocket.storage.RxStore)2 StoreFactory (io.georocket.storage.StoreFactory)2 JsonParserOperator (io.georocket.util.JsonParserOperator)2 MimeTypeUtils.belongsTo (io.georocket.util.MimeTypeUtils.belongsTo)2 RxUtils (io.georocket.util.RxUtils)2 XMLParserOperator (io.georocket.util.XMLParserOperator)2 NoStackTraceThrowable (io.vertx.core.impl.NoStackTraceThrowable)2 JsonArray (io.vertx.core.json.JsonArray)2 JsonObject (io.vertx.core.json.JsonObject)2 Logger (io.vertx.core.logging.Logger)2 LoggerFactory (io.vertx.core.logging.LoggerFactory)2 AbstractVerticle (io.vertx.rxjava.core.AbstractVerticle)2 Message (io.vertx.rxjava.core.eventbus.Message)2 ImmutableList (com.google.common.collect.ImmutableList)1 ElasticsearchClient (io.georocket.index.elasticsearch.ElasticsearchClient)1 ElasticsearchClientFactory (io.georocket.index.elasticsearch.ElasticsearchClientFactory)1