Examples with IndexMeta io.georocket.storage.IndexMeta used on opensource projects

Example 1 with IndexMeta

use of io.georocket.storage.IndexMeta in project georocket by georocket.

the class ImporterVerticle method importJSON.

/**
 * Imports a JSON file from the given input stream into the store
 * @param f the JSON file to read
 * @param correlationId a unique identifier for this import process
 * @param filename the name of the file currently being imported
 * @param timestamp denotes when the import process has started
 * @param layer the layer where the file should be stored (may be null)
 * @param tags the list of tags to attach to the file (may be null)
 * @param properties the map of properties to attach to the file (may be null)
 * @return a single that will emit when the file has been imported
 */
protected Single<Integer> importJSON(ReadStream<Buffer> f, String correlationId, String filename, long timestamp, String layer, List<String> tags, Map<String, Object> properties) {
    UTF8BomFilter bomFilter = new UTF8BomFilter();
    StringWindow window = new StringWindow();
    GeoJsonSplitter splitter = new GeoJsonSplitter(window);
    AtomicInteger processing = new AtomicInteger(0);
    return f.toObservable().map(buf -> (io.vertx.core.buffer.Buffer) buf.getDelegate()).map(bomFilter::filter).doOnNext(window::append).lift(new JsonParserOperator()).flatMap(splitter::onEventObservable).flatMapSingle(result -> {
        IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, null);
        return addToStoreWithPause(result, layer, indexMeta, f, processing);
    }).count().toSingle();
}

Example 2 with IndexMeta

use of io.georocket.storage.IndexMeta in project georocket by georocket.

the class ImporterVerticle method importXML.

/**
 * Imports an XML file from the given input stream into the store
 * @param f the XML file to read
 * @param correlationId a unique identifier for this import process
 * @param filename the name of the file currently being imported
 * @param timestamp denotes when the import process has started
 * @param layer the layer where the file should be stored (may be null)
 * @param tags the list of tags to attach to the file (may be null)
 * @param properties the map of properties to attach to the file (may be null)
 * @param fallbackCRSString the CRS which should be used if the imported
 * file does not specify one (may be <code>null</code>)
 * @return a single that will emit when the file has been imported
 */
protected Single<Integer> importXML(ReadStream<Buffer> f, String correlationId, String filename, long timestamp, String layer, List<String> tags, Map<String, Object> properties, String fallbackCRSString) {
    UTF8BomFilter bomFilter = new UTF8BomFilter();
    Window window = new Window();
    XMLSplitter splitter = new FirstLevelSplitter(window);
    AtomicInteger processing = new AtomicInteger(0);
    XMLCRSIndexer crsIndexer = new XMLCRSIndexer();
    return f.toObservable().map(buf -> (io.vertx.core.buffer.Buffer) buf.getDelegate()).map(bomFilter::filter).doOnNext(window::append).lift(new XMLParserOperator()).doOnNext(e -> {
        // save the first CRS found in the file
        if (crsIndexer.getCRS() == null) {
            crsIndexer.onEvent(e);
        }
    }).flatMap(splitter::onEventObservable).flatMapSingle(result -> {
        String crsString = fallbackCRSString;
        if (crsIndexer.getCRS() != null) {
            crsString = crsIndexer.getCRS();
        }
        IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, crsString);
        return addToStoreWithPause(result, layer, indexMeta, f, processing);
    }).count().toSingle();
}

Example 3 with IndexMeta

use of io.georocket.storage.IndexMeta in project georocket by georocket.

the class IndexerVerticle method onAdd.

/**
 * Will be called when chunks should be added to the index
 * @param messages the list of add messages that contain the paths to
 * the chunks to be indexed
 * @return an observable that completes when the operation has finished
 */
private Observable<Void> onAdd(List<Message<JsonObject>> messages) {
    return Observable.from(messages).flatMap(msg -> {
        // get path to chunk from message
        JsonObject body = msg.body();
        String path = body.getString("path");
        if (path == null) {
            msg.fail(400, "Missing path to the chunk to index");
            return Observable.empty();
        }
        // get chunk metadata
        JsonObject meta = body.getJsonObject("meta");
        if (meta == null) {
            msg.fail(400, "Missing metadata for chunk " + path);
            return Observable.empty();
        }
        // get tags
        JsonArray tagsArr = body.getJsonArray("tags");
        List<String> tags = tagsArr != null ? tagsArr.stream().flatMap(o -> o != null ? Stream.of(o.toString()) : Stream.of()).collect(Collectors.toList()) : null;
        // get properties
        JsonObject propertiesObj = body.getJsonObject("properties");
        Map<String, Object> properties = propertiesObj != null ? propertiesObj.getMap() : null;
        // get fallback CRS
        String fallbackCRSString = body.getString("fallbackCRSString");
        log.trace("Indexing " + path);
        String correlationId = body.getString("correlationId");
        String filename = body.getString("filename");
        long timestamp = body.getLong("timestamp", System.currentTimeMillis());
        ChunkMeta chunkMeta = getMeta(meta);
        IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, fallbackCRSString);
        // open chunk and create IndexRequest
        return openChunkToDocument(path, chunkMeta, indexMeta).map(doc -> Tuple.tuple(path, new JsonObject(doc), msg)).onErrorResumeNext(err -> {
            msg.fail(throwableToCode(err), throwableToMessage(err, ""));
            return Observable.empty();
        });
    }).toList().flatMap(l -> {
        if (!l.isEmpty()) {
            return insertDocuments(TYPE_NAME, l);
        }
        return Observable.empty();
    });
}