Search in sources :

Example 1 with StoreCursor

use of io.georocket.storage.StoreCursor in project georocket by georocket.

the class StoreEndpoint method doMerge.

/**
 * Perform a search and merge all retrieved chunks using the given merger
 * @param merger the merger
 * @param data Data to merge into the response
 * @param out the response to write the merged chunks to
 * @return a single that will emit one item when all chunks have been merged
 */
private Single<Void> doMerge(Merger<ChunkMeta> merger, Single<StoreCursor> data, WriteStream<Buffer> out) {
    return data.map(RxStoreCursor::new).flatMapObservable(RxStoreCursor::toObservable).flatMap(p -> store.rxGetOne(p.getRight()).flatMapObservable(crs -> merger.merge(crs, p.getLeft(), out).map(// left: count, right: not_accepted
    v -> Pair.of(1L, 0L)).onErrorResumeNext(t -> {
        if (t instanceof IllegalStateException) {
            // ignore it, but emit a warning later
            return Observable.just(Pair.of(0L, 1L));
        }
        return Observable.error(t);
    }).doOnTerminate(() -> {
        // don't forget to close the chunk!
        crs.close();
    })), 1).defaultIfEmpty(Pair.of(0L, 0L)).reduce((p1, p2) -> Pair.of(p1.getLeft() + p2.getLeft(), p1.getRight() + p2.getRight())).flatMap(p -> {
        long count = p.getLeft();
        long notaccepted = p.getRight();
        if (notaccepted > 0) {
            log.warn("Could not merge " + notaccepted + " chunks " + "because the merger did not accept them. Most likely " + "these are new chunks that were added while the " + "merge was in progress. If this worries you, just " + "repeat the request.");
        }
        if (count > 0) {
            merger.finish(out);
            return Observable.just(null);
        } else {
            return Observable.error(new FileNotFoundException("Not Found"));
        }
    }).toSingle().map(v -> null);
}
Also used : Arrays(java.util.Arrays) Router(io.vertx.ext.web.Router) RxStoreCursor(io.georocket.storage.RxStoreCursor) RoutingContext(io.vertx.ext.web.RoutingContext) StringUtils(org.apache.commons.lang3.StringUtils) ChunkMeta(io.georocket.storage.ChunkMeta) RxStore(io.georocket.storage.RxStore) StoreCursor(io.georocket.storage.StoreCursor) Single(rx.Single) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) Pump(io.vertx.core.streams.Pump) JsonObject(io.vertx.core.json.JsonObject) Logger(io.vertx.core.logging.Logger) Splitter(com.google.common.base.Splitter) OpenOptions(io.vertx.core.file.OpenOptions) ContentType(org.apache.http.entity.ContentType) UUID(java.util.UUID) Future(io.vertx.core.Future) FileNotFoundException(java.io.FileNotFoundException) List(java.util.List) Buffer(io.vertx.core.buffer.Buffer) HttpServerResponse(io.vertx.core.http.HttpServerResponse) FileSystem(io.vertx.core.file.FileSystem) RxHelper(io.vertx.rx.java.RxHelper) MultiMerger(io.georocket.output.MultiMerger) Pattern(java.util.regex.Pattern) AddressConstants(io.georocket.constants.AddressConstants) AsyncFile(io.vertx.core.file.AsyncFile) HttpServerRequest(io.vertx.core.http.HttpServerRequest) MimeTypeUtils(io.georocket.util.MimeTypeUtils) HashMap(java.util.HashMap) LoggerFactory(io.vertx.core.logging.LoggerFactory) Observable(rx.Observable) ServerAPIException(io.georocket.ServerAPIException) WriteStream(io.vertx.core.streams.WriteStream) StoreFactory(io.georocket.storage.StoreFactory) AsyncResult(io.vertx.core.AsyncResult) HttpException(io.georocket.util.HttpException) ParseException(org.apache.http.ParseException) ObservableFuture(io.vertx.rx.java.ObservableFuture) Vertx(io.vertx.core.Vertx) IOException(java.io.IOException) StringEscapeUtils(org.apache.commons.text.StringEscapeUtils) RxAsyncCursor(io.georocket.storage.RxAsyncCursor) File(java.io.File) JsonArray(io.vertx.core.json.JsonArray) ObjectId(org.bson.types.ObjectId) Merger(io.georocket.output.Merger) Handler(io.vertx.core.Handler) ConfigConstants(io.georocket.constants.ConfigConstants) FileNotFoundException(java.io.FileNotFoundException) RxStoreCursor(io.georocket.storage.RxStoreCursor)

Example 2 with StoreCursor

use of io.georocket.storage.StoreCursor in project georocket by georocket.

the class StoreEndpoint method getChunks.

/**
 * Retrieve all chunks matching the specified query and path
 * @param context the routing context
 */
private void getChunks(RoutingContext context) {
    HttpServerResponse response = context.response();
    Single<StoreCursor> data = prepareCursor(context);
    // Our responses must always be chunked because we cannot calculate
    // the exact content-length beforehand. We perform two searches, one to
    // initialize the merger and one to do the actual merge. The problem is
    // that the result set may change between these two searches and so we
    // cannot calculate the content-length just from looking at the result
    // from the first search.
    response.setChunked(true);
    // perform two searches: first initialize the merger and then
    // merge all retrieved chunks
    Merger<ChunkMeta> merger = createMerger(context);
    initializeMerger(merger, data).flatMapSingle(v -> doMerge(merger, data, response)).subscribe(v -> {
        response.end();
    }, err -> {
        if (!(err instanceof FileNotFoundException)) {
            log.error("Could not perform query", err);
        }
        fail(response, err);
    });
}
Also used : Arrays(java.util.Arrays) Router(io.vertx.ext.web.Router) RxStoreCursor(io.georocket.storage.RxStoreCursor) RoutingContext(io.vertx.ext.web.RoutingContext) StringUtils(org.apache.commons.lang3.StringUtils) ChunkMeta(io.georocket.storage.ChunkMeta) RxStore(io.georocket.storage.RxStore) StoreCursor(io.georocket.storage.StoreCursor) Single(rx.Single) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) Pump(io.vertx.core.streams.Pump) JsonObject(io.vertx.core.json.JsonObject) Logger(io.vertx.core.logging.Logger) Splitter(com.google.common.base.Splitter) OpenOptions(io.vertx.core.file.OpenOptions) ContentType(org.apache.http.entity.ContentType) UUID(java.util.UUID) Future(io.vertx.core.Future) FileNotFoundException(java.io.FileNotFoundException) List(java.util.List) Buffer(io.vertx.core.buffer.Buffer) HttpServerResponse(io.vertx.core.http.HttpServerResponse) FileSystem(io.vertx.core.file.FileSystem) RxHelper(io.vertx.rx.java.RxHelper) MultiMerger(io.georocket.output.MultiMerger) Pattern(java.util.regex.Pattern) AddressConstants(io.georocket.constants.AddressConstants) AsyncFile(io.vertx.core.file.AsyncFile) HttpServerRequest(io.vertx.core.http.HttpServerRequest) MimeTypeUtils(io.georocket.util.MimeTypeUtils) HashMap(java.util.HashMap) LoggerFactory(io.vertx.core.logging.LoggerFactory) Observable(rx.Observable) ServerAPIException(io.georocket.ServerAPIException) WriteStream(io.vertx.core.streams.WriteStream) StoreFactory(io.georocket.storage.StoreFactory) AsyncResult(io.vertx.core.AsyncResult) HttpException(io.georocket.util.HttpException) ParseException(org.apache.http.ParseException) ObservableFuture(io.vertx.rx.java.ObservableFuture) Vertx(io.vertx.core.Vertx) IOException(java.io.IOException) StringEscapeUtils(org.apache.commons.text.StringEscapeUtils) RxAsyncCursor(io.georocket.storage.RxAsyncCursor) File(java.io.File) JsonArray(io.vertx.core.json.JsonArray) ObjectId(org.bson.types.ObjectId) Merger(io.georocket.output.Merger) Handler(io.vertx.core.Handler) ConfigConstants(io.georocket.constants.ConfigConstants) HttpServerResponse(io.vertx.core.http.HttpServerResponse) RxStoreCursor(io.georocket.storage.RxStoreCursor) StoreCursor(io.georocket.storage.StoreCursor) FileNotFoundException(java.io.FileNotFoundException) ChunkMeta(io.georocket.storage.ChunkMeta)

Example 3 with StoreCursor

use of io.georocket.storage.StoreCursor in project georocket by georocket.

the class IndexedStore method delete.

@Override
public void delete(String search, String path, Handler<AsyncResult<Void>> handler) {
    get(search, path, ar -> {
        if (ar.failed()) {
            Throwable cause = ar.cause();
            if (cause instanceof ReplyException) {
                // Cast to get access to the failure code
                ReplyException ex = (ReplyException) cause;
                if (ex.failureCode() == 404) {
                    handler.handle(Future.succeededFuture());
                    return;
                }
            }
            handler.handle(Future.failedFuture(ar.cause()));
        } else {
            StoreCursor cursor = ar.result();
            Queue<String> paths = new ArrayDeque<>();
            AtomicLong remaining = new AtomicLong(cursor.getInfo().getTotalHits());
            doDelete(cursor, paths, remaining, handler);
        }
    });
}
Also used : AtomicLong(java.util.concurrent.atomic.AtomicLong) StoreCursor(io.georocket.storage.StoreCursor) ReplyException(io.vertx.core.eventbus.ReplyException) ArrayDeque(java.util.ArrayDeque)

Aggregations

StoreCursor (io.georocket.storage.StoreCursor)3 Splitter (com.google.common.base.Splitter)2 ServerAPIException (io.georocket.ServerAPIException)2 AddressConstants (io.georocket.constants.AddressConstants)2 ConfigConstants (io.georocket.constants.ConfigConstants)2 Merger (io.georocket.output.Merger)2 MultiMerger (io.georocket.output.MultiMerger)2 ChunkMeta (io.georocket.storage.ChunkMeta)2 RxAsyncCursor (io.georocket.storage.RxAsyncCursor)2 RxStore (io.georocket.storage.RxStore)2 RxStoreCursor (io.georocket.storage.RxStoreCursor)2 StoreFactory (io.georocket.storage.StoreFactory)2 HttpException (io.georocket.util.HttpException)2 MimeTypeUtils (io.georocket.util.MimeTypeUtils)2 AsyncResult (io.vertx.core.AsyncResult)2 Future (io.vertx.core.Future)2 Handler (io.vertx.core.Handler)2 Vertx (io.vertx.core.Vertx)2 Buffer (io.vertx.core.buffer.Buffer)2 AsyncFile (io.vertx.core.file.AsyncFile)2