Examples with ChunkMeta io.georocket.storage.ChunkMeta used on opensource projects

Example 1 with ChunkMeta

use of io.georocket.storage.ChunkMeta in project georocket by georocket.

the class StoreEndpoint method doMerge.

/**
 * Perform a search and merge all retrieved chunks using the given merger
 * @param merger the merger
 * @param data Data to merge into the response
 * @param out the response to write the merged chunks to
 * @return a single that will emit one item when all chunks have been merged
 */
private Single<Void> doMerge(Merger<ChunkMeta> merger, Single<StoreCursor> data, WriteStream<Buffer> out) {
    return data.map(RxStoreCursor::new).flatMapObservable(RxStoreCursor::toObservable).flatMap(p -> store.rxGetOne(p.getRight()).flatMapObservable(crs -> merger.merge(crs, p.getLeft(), out).map(// left: count, right: not_accepted
    v -> Pair.of(1L, 0L)).onErrorResumeNext(t -> {
        if (t instanceof IllegalStateException) {
            // ignore it, but emit a warning later
            return Observable.just(Pair.of(0L, 1L));
        }
        return Observable.error(t);
    }).doOnTerminate(() -> {
        // don't forget to close the chunk!
        crs.close();
    })), 1).defaultIfEmpty(Pair.of(0L, 0L)).reduce((p1, p2) -> Pair.of(p1.getLeft() + p2.getLeft(), p1.getRight() + p2.getRight())).flatMap(p -> {
        long count = p.getLeft();
        long notaccepted = p.getRight();
        if (notaccepted > 0) {
            log.warn("Could not merge " + notaccepted + " chunks " + "because the merger did not accept them. Most likely " + "these are new chunks that were added while the " + "merge was in progress. If this worries you, just " + "repeat the request.");
        }
        if (count > 0) {
            merger.finish(out);
            return Observable.just(null);
        } else {
            return Observable.error(new FileNotFoundException("Not Found"));
        }
    }).toSingle().map(v -> null);
}

Example 2 with ChunkMeta

use of io.georocket.storage.ChunkMeta in project georocket by georocket.

the class MultiMergerTest method doMerge.

private void doMerge(TestContext context, Observable<Buffer> chunks, Observable<ChunkMeta> metas, String jsonContents) {
    MultiMerger m = new MultiMerger();
    BufferWriteStream bws = new BufferWriteStream();
    Async async = context.async();
    metas.flatMap(meta -> m.init(meta).map(v -> meta)).toList().flatMap(l -> chunks.map(DelegateChunkReadStream::new).<ChunkMeta, Pair<ChunkReadStream, ChunkMeta>>zipWith(l, Pair::of)).flatMap(p -> m.merge(p.getLeft(), p.getRight(), bws)).last().subscribe(v -> {
        m.finish(bws);
        context.assertEquals(jsonContents, bws.getBuffer().toString("utf-8"));
        async.complete();
    }, err -> {
        context.fail(err);
    });
}

Example 3 with ChunkMeta

use of io.georocket.storage.ChunkMeta in project georocket by georocket.

the class IndexerVerticle method onQuery.

/**
 * Write result of a query given the Elasticsearch response
 * @param body the message containing the query
 * @return an observable that emits the results of the query
 */
private Observable<JsonObject> onQuery(JsonObject body) {
    String search = body.getString("search");
    String path = body.getString("path");
    String scrollId = body.getString("scrollId");
    int pageSize = body.getInteger("size", 100);
    // one minute
    String timeout = "1m";
    JsonObject parameters = new JsonObject().put("size", pageSize);
    Observable<JsonObject> observable;
    if (scrollId == null) {
        // Execute a new search. Use a post_filter because we only want to get
        // a yes/no answer and no scoring (i.e. we only want to get matching
        // documents and not those that likely match). For the difference between
        // query and post_filter see the Elasticsearch documentation.
        JsonObject postFilter;
        try {
            postFilter = queryCompiler.compileQuery(search, path);
        } catch (Throwable t) {
            return Observable.error(t);
        }
        observable = client.beginScroll(TYPE_NAME, null, postFilter, parameters, timeout);
    } else {
        // continue searching
        observable = client.continueScroll(scrollId, timeout);
    }
    return observable.map(sr -> {
        // iterate through all hits and convert them to JSON
        JsonObject hits = sr.getJsonObject("hits");
        long totalHits = hits.getLong("total");
        JsonArray resultHits = new JsonArray();
        JsonArray hitsHits = hits.getJsonArray("hits");
        for (Object o : hitsHits) {
            JsonObject hit = (JsonObject) o;
            String id = hit.getString("_id");
            JsonObject source = hit.getJsonObject("_source");
            JsonObject jsonMeta = source.getJsonObject("chunkMeta");
            ChunkMeta meta = getMeta(jsonMeta);
            JsonObject obj = meta.toJsonObject().put("id", id);
            resultHits.add(obj);
        }
        // create result and send it to the client
        return new JsonObject().put("totalHits", totalHits).put("hits", resultHits).put("scrollId", sr.getString("_scroll_id"));
    });
}

Example 4 with ChunkMeta

use of io.georocket.storage.ChunkMeta in project georocket by georocket.

the class IndexerVerticle method onAdd.

/**
 * Will be called when chunks should be added to the index
 * @param messages the list of add messages that contain the paths to
 * the chunks to be indexed
 * @return an observable that completes when the operation has finished
 */
private Observable<Void> onAdd(List<Message<JsonObject>> messages) {
    return Observable.from(messages).flatMap(msg -> {
        // get path to chunk from message
        JsonObject body = msg.body();
        String path = body.getString("path");
        if (path == null) {
            msg.fail(400, "Missing path to the chunk to index");
            return Observable.empty();
        }
        // get chunk metadata
        JsonObject meta = body.getJsonObject("meta");
        if (meta == null) {
            msg.fail(400, "Missing metadata for chunk " + path);
            return Observable.empty();
        }
        // get tags
        JsonArray tagsArr = body.getJsonArray("tags");
        List<String> tags = tagsArr != null ? tagsArr.stream().flatMap(o -> o != null ? Stream.of(o.toString()) : Stream.of()).collect(Collectors.toList()) : null;
        // get properties
        JsonObject propertiesObj = body.getJsonObject("properties");
        Map<String, Object> properties = propertiesObj != null ? propertiesObj.getMap() : null;
        // get fallback CRS
        String fallbackCRSString = body.getString("fallbackCRSString");
        log.trace("Indexing " + path);
        String correlationId = body.getString("correlationId");
        String filename = body.getString("filename");
        long timestamp = body.getLong("timestamp", System.currentTimeMillis());
        ChunkMeta chunkMeta = getMeta(meta);
        IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, fallbackCRSString);
        // open chunk and create IndexRequest
        return openChunkToDocument(path, chunkMeta, indexMeta).map(doc -> Tuple.tuple(path, new JsonObject(doc), msg)).onErrorResumeNext(err -> {
            msg.fail(throwableToCode(err), throwableToMessage(err, ""));
            return Observable.empty();
        });
    }).toList().flatMap(l -> {
        if (!l.isEmpty()) {
            return insertDocuments(TYPE_NAME, l);
        }
        return Observable.empty();
    });
}

Example 5 with ChunkMeta

use of io.georocket.storage.ChunkMeta in project georocket by georocket.

the class StoreEndpoint method getChunks.

/**
 * Retrieve all chunks matching the specified query and path
 * @param context the routing context
 */
private void getChunks(RoutingContext context) {
    HttpServerResponse response = context.response();
    Single<StoreCursor> data = prepareCursor(context);
    // Our responses must always be chunked because we cannot calculate
    // the exact content-length beforehand. We perform two searches, one to
    // initialize the merger and one to do the actual merge. The problem is
    // that the result set may change between these two searches and so we
    // cannot calculate the content-length just from looking at the result
    // from the first search.
    response.setChunked(true);
    // perform two searches: first initialize the merger and then
    // merge all retrieved chunks
    Merger<ChunkMeta> merger = createMerger(context);
    initializeMerger(merger, data).flatMapSingle(v -> doMerge(merger, data, response)).subscribe(v -> {
        response.end();
    }, err -> {
        if (!(err instanceof FileNotFoundException)) {
            log.error("Could not perform query", err);
        }
        fail(response, err);
    });
}