Search in sources :

Example 1 with ObservableFuture

use of io.vertx.rx.java.ObservableFuture in project georocket by georocket.

the class StoreEndpoint method onPost.

/**
 * Handles the HTTP POST request
 * @param context the routing context
 */
private void onPost(RoutingContext context) {
    HttpServerRequest request = context.request();
    request.pause();
    String layer = getEndpointPath(context);
    String tagsStr = request.getParam("tags");
    String propertiesStr = request.getParam("props");
    String fallbackCRSString = request.getParam("fallbackCRS");
    List<String> tags = StringUtils.isNotEmpty(tagsStr) ? Splitter.on(',').trimResults().splitToList(tagsStr) : null;
    Map<String, Object> properties = new HashMap<>();
    if (StringUtils.isNotEmpty(propertiesStr)) {
        String regex = "(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":");
        String[] parts = propertiesStr.split(",");
        for (String part : parts) {
            part = part.trim();
            String[] property = part.split(regex);
            if (property.length != 2) {
                request.response().setStatusCode(400).end("Invalid property syntax: " + part);
                return;
            }
            String key = StringEscapeUtils.unescapeJava(property[0].trim());
            String value = StringEscapeUtils.unescapeJava(property[1].trim());
            properties.put(key, value);
        }
    }
    // get temporary filename
    String incoming = storagePath + "/incoming";
    String filename = new ObjectId().toString();
    String filepath = incoming + "/" + filename;
    String correlationId = UUID.randomUUID().toString();
    long startTime = System.currentTimeMillis();
    this.onReceivingFileStarted(correlationId, layer, startTime);
    // create directory for incoming files
    FileSystem fs = vertx.fileSystem();
    ObservableFuture<Void> observable = RxHelper.observableFuture();
    fs.mkdirs(incoming, observable.toHandler());
    observable.flatMap(v -> {
        // create temporary file
        ObservableFuture<AsyncFile> openObservable = RxHelper.observableFuture();
        fs.open(filepath, new OpenOptions(), openObservable.toHandler());
        return openObservable;
    }).flatMap(f -> {
        // write request body into temporary file
        ObservableFuture<Void> pumpObservable = RxHelper.observableFuture();
        Handler<AsyncResult<Void>> pumpHandler = pumpObservable.toHandler();
        Pump.pump(request, f).start();
        Handler<Throwable> errHandler = (Throwable t) -> {
            request.endHandler(null);
            f.close();
            pumpHandler.handle(Future.failedFuture(t));
        };
        f.exceptionHandler(errHandler);
        request.exceptionHandler(errHandler);
        request.endHandler(v -> {
            f.close();
            pumpHandler.handle(Future.succeededFuture());
        });
        request.resume();
        return pumpObservable;
    }).flatMap(v -> {
        String contentTypeHeader = request.getHeader("Content-Type");
        String mimeType = null;
        try {
            ContentType contentType = ContentType.parse(contentTypeHeader);
            mimeType = contentType.getMimeType();
        } catch (ParseException | IllegalArgumentException ex) {
        // mimeType already null
        }
        // detect content type of file to import
        if (mimeType == null || mimeType.trim().isEmpty() || mimeType.equals("application/octet-stream") || mimeType.equals("application/x-www-form-urlencoded")) {
            // fallback: if the client has not sent a Content-Type or if it's
            // a generic one, then try to guess it
            log.debug("Mime type '" + mimeType + "' is invalid or generic. " + "Trying to guess the right type.");
            return detectContentType(filepath).doOnNext(guessedType -> {
                log.debug("Guessed mime type '" + guessedType + "'.");
            });
        }
        return Observable.just(mimeType);
    }).subscribe(detectedContentType -> {
        long duration = System.currentTimeMillis() - startTime;
        this.onReceivingFileFinished(correlationId, duration, layer, null);
        // run importer
        JsonObject msg = new JsonObject().put("filename", filename).put("layer", layer).put("contentType", detectedContentType).put("correlationId", correlationId);
        if (tags != null) {
            msg.put("tags", new JsonArray(tags));
        }
        if (!properties.isEmpty()) {
            msg.put("properties", new JsonObject(properties));
        }
        if (fallbackCRSString != null) {
            msg.put("fallbackCRSString", fallbackCRSString);
        }
        request.response().setStatusCode(// Accepted
        202).putHeader("X-Correlation-Id", correlationId).setStatusMessage("Accepted file - importing in progress").end();
        // run importer
        vertx.eventBus().send(AddressConstants.IMPORTER_IMPORT, msg);
    }, err -> {
        long duration = System.currentTimeMillis() - startTime;
        this.onReceivingFileFinished(correlationId, duration, layer, err);
        fail(request.response(), err);
        err.printStackTrace();
        fs.delete(filepath, ar -> {
        });
    });
}
Also used : Arrays(java.util.Arrays) Router(io.vertx.ext.web.Router) RxStoreCursor(io.georocket.storage.RxStoreCursor) RoutingContext(io.vertx.ext.web.RoutingContext) StringUtils(org.apache.commons.lang3.StringUtils) ChunkMeta(io.georocket.storage.ChunkMeta) RxStore(io.georocket.storage.RxStore) StoreCursor(io.georocket.storage.StoreCursor) Single(rx.Single) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) Pump(io.vertx.core.streams.Pump) JsonObject(io.vertx.core.json.JsonObject) Logger(io.vertx.core.logging.Logger) Splitter(com.google.common.base.Splitter) OpenOptions(io.vertx.core.file.OpenOptions) ContentType(org.apache.http.entity.ContentType) UUID(java.util.UUID) Future(io.vertx.core.Future) FileNotFoundException(java.io.FileNotFoundException) List(java.util.List) Buffer(io.vertx.core.buffer.Buffer) HttpServerResponse(io.vertx.core.http.HttpServerResponse) FileSystem(io.vertx.core.file.FileSystem) RxHelper(io.vertx.rx.java.RxHelper) MultiMerger(io.georocket.output.MultiMerger) Pattern(java.util.regex.Pattern) AddressConstants(io.georocket.constants.AddressConstants) AsyncFile(io.vertx.core.file.AsyncFile) HttpServerRequest(io.vertx.core.http.HttpServerRequest) MimeTypeUtils(io.georocket.util.MimeTypeUtils) HashMap(java.util.HashMap) LoggerFactory(io.vertx.core.logging.LoggerFactory) Observable(rx.Observable) ServerAPIException(io.georocket.ServerAPIException) WriteStream(io.vertx.core.streams.WriteStream) StoreFactory(io.georocket.storage.StoreFactory) AsyncResult(io.vertx.core.AsyncResult) HttpException(io.georocket.util.HttpException) ParseException(org.apache.http.ParseException) ObservableFuture(io.vertx.rx.java.ObservableFuture) Vertx(io.vertx.core.Vertx) IOException(java.io.IOException) StringEscapeUtils(org.apache.commons.text.StringEscapeUtils) RxAsyncCursor(io.georocket.storage.RxAsyncCursor) File(java.io.File) JsonArray(io.vertx.core.json.JsonArray) ObjectId(org.bson.types.ObjectId) Merger(io.georocket.output.Merger) Handler(io.vertx.core.Handler) ConfigConstants(io.georocket.constants.ConfigConstants) OpenOptions(io.vertx.core.file.OpenOptions) ContentType(org.apache.http.entity.ContentType) HashMap(java.util.HashMap) ObjectId(org.bson.types.ObjectId) HttpServerRequest(io.vertx.core.http.HttpServerRequest) JsonObject(io.vertx.core.json.JsonObject) JsonArray(io.vertx.core.json.JsonArray) ObservableFuture(io.vertx.rx.java.ObservableFuture) FileSystem(io.vertx.core.file.FileSystem) JsonObject(io.vertx.core.json.JsonObject) AsyncResult(io.vertx.core.AsyncResult)

Example 2 with ObservableFuture

use of io.vertx.rx.java.ObservableFuture in project georocket by georocket.

the class ElasticsearchInstaller method extractArchive.

/**
 * Extract the Elasticsearch ZIP archive to a destination path
 * @param archivePath the path to the ZIP file
 * @param destPath the destination path
 * @param strip <code>true</code> if the first path element of all items in the
 * ZIP file should be stripped away.
 * @return emitting the path to the extracted contents (i.e.
 * <code>destPath</code>)
 */
private Single<String> extractArchive(String archivePath, String destPath, boolean strip) {
    ObservableFuture<String> observable = RxHelper.observableFuture();
    Handler<AsyncResult<String>> handler = observable.toHandler();
    // extract archive asynchronously
    vertx.executeBlocking(f -> {
        File archiveFile = new File(archivePath);
        File destFile = new File(destPath);
        destFile.mkdirs();
        try {
            extractZip(archiveFile, destFile, strip);
            f.complete();
        } catch (IOException e) {
            FileUtils.deleteQuietly(destFile);
            f.fail(e);
        }
    }, ar -> {
        if (ar.failed()) {
            handler.handle(Future.failedFuture(ar.cause()));
        } else {
            handler.handle(Future.succeededFuture(destPath));
        }
    });
    // set executable permissions for Elasticsearch binary
    return observable.doOnNext(path -> {
        if (!SystemUtils.IS_OS_WINDOWS) {
            log.info("Set executable permissions for \"bin/elasticsearch\"");
            File archiveFile = new File(path);
            File executable = new File(archiveFile, "bin/elasticsearch");
            executable.setExecutable(true);
        }
    }).toSingle();
}
Also used : Enumeration(java.util.Enumeration) AsyncFile(io.vertx.rxjava.core.file.AsyncFile) LoggerFactory(io.vertx.core.logging.LoggerFactory) Observable(rx.Observable) HttpClient(io.vertx.rxjava.core.http.HttpClient) Single(rx.Single) FileSystem(io.vertx.rxjava.core.file.FileSystem) ZipFile(java.util.zip.ZipFile) AsyncResult(io.vertx.core.AsyncResult) HttpClientOptions(io.vertx.core.http.HttpClientOptions) Logger(io.vertx.core.logging.Logger) ZipEntry(java.util.zip.ZipEntry) HttpException(io.georocket.util.HttpException) OutputStream(java.io.OutputStream) OpenOptions(io.vertx.core.file.OpenOptions) ObservableFuture(io.vertx.rx.java.ObservableFuture) SystemUtils(org.apache.commons.lang3.SystemUtils) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Future(io.vertx.core.Future) File(java.io.File) HttpClientRequest(io.vertx.rxjava.core.http.HttpClientRequest) IOUtils(org.apache.commons.io.IOUtils) RxHelper(io.vertx.rx.java.RxHelper) Handler(io.vertx.core.Handler) FilenameUtils(org.apache.commons.io.FilenameUtils) InputStream(java.io.InputStream) Vertx(io.vertx.rxjava.core.Vertx) IOException(java.io.IOException) AsyncResult(io.vertx.core.AsyncResult) AsyncFile(io.vertx.rxjava.core.file.AsyncFile) ZipFile(java.util.zip.ZipFile) File(java.io.File)

Example 3 with ObservableFuture

use of io.vertx.rx.java.ObservableFuture in project georocket by georocket.

the class FileStore method getOne.

@Override
public void getOne(String path, Handler<AsyncResult<ChunkReadStream>> handler) {
    String absolutePath = Paths.get(root, path).toString();
    // check if chunk exists
    FileSystem fs = vertx.fileSystem();
    ObservableFuture<Boolean> observable = RxHelper.observableFuture();
    fs.exists(absolutePath, observable.toHandler());
    observable.flatMap(exists -> {
        if (!exists) {
            return Observable.error(new FileNotFoundException("Could not find chunk: " + path));
        }
        return Observable.just(exists);
    }).flatMap(exists -> {
        // get chunk's size
        ObservableFuture<FileProps> propsObservable = RxHelper.observableFuture();
        fs.props(absolutePath, propsObservable.toHandler());
        return propsObservable;
    }).map(props -> props.size()).flatMap(size -> {
        // open chunk
        ObservableFuture<AsyncFile> openObservable = RxHelper.observableFuture();
        OpenOptions openOptions = new OpenOptions().setCreate(false).setWrite(false);
        fs.open(absolutePath, openOptions, openObservable.toHandler());
        return openObservable.map(f -> new FileChunkReadStream(size, f));
    }).subscribe(readStream -> {
        // send chunk to peer
        handler.handle(Future.succeededFuture(readStream));
    }, err -> {
        handler.handle(Future.failedFuture(err));
    });
}
Also used : PathUtils(io.georocket.util.PathUtils) AsyncFile(io.vertx.core.file.AsyncFile) OpenOptions(io.vertx.core.file.OpenOptions) ObservableFuture(io.vertx.rx.java.ObservableFuture) IndexedStore(io.georocket.storage.indexed.IndexedStore) Vertx(io.vertx.core.Vertx) Future(io.vertx.core.Future) FileNotFoundException(java.io.FileNotFoundException) FileProps(io.vertx.core.file.FileProps) Observable(rx.Observable) Buffer(io.vertx.core.buffer.Buffer) Paths(java.nio.file.Paths) FileSystem(io.vertx.core.file.FileSystem) RxHelper(io.vertx.rx.java.RxHelper) Preconditions(com.google.common.base.Preconditions) Queue(java.util.Queue) AsyncResult(io.vertx.core.AsyncResult) Handler(io.vertx.core.Handler) ChunkReadStream(io.georocket.storage.ChunkReadStream) ConfigConstants(io.georocket.constants.ConfigConstants) OpenOptions(io.vertx.core.file.OpenOptions) ObservableFuture(io.vertx.rx.java.ObservableFuture) FileSystem(io.vertx.core.file.FileSystem) FileNotFoundException(java.io.FileNotFoundException)

Example 4 with ObservableFuture

use of io.vertx.rx.java.ObservableFuture in project georocket by georocket.

the class ImportCommand method importFile.

/**
 * Upload a file to GeoRocket
 * @param path path to file to import
 * @param client the GeoRocket client
 * @param vertx the Vert.x instance
 * @return an observable that will emit when the file has been uploaded
 */
protected Observable<Void> importFile(String path, GeoRocketClient client, Vertx vertx) {
    // open file
    FileSystem fs = vertx.fileSystem();
    OpenOptions openOptions = new OpenOptions().setCreate(false).setWrite(false);
    return fs.rxOpen(path, openOptions).flatMap(f -> fs.rxProps(path).map(props -> Pair.of(f, props.size()))).flatMapObservable(f -> {
        ObservableFuture<Void> o = RxHelper.observableFuture();
        Handler<AsyncResult<Void>> handler = o.toHandler();
        AsyncFile file = f.getLeft().getDelegate();
        WriteStream<Buffer> out = client.getStore().startImport(layer, tags, properties, Optional.of(f.getRight()), fallbackCRS, handler);
        AtomicBoolean fileClosed = new AtomicBoolean();
        Pump pump = Pump.pump(file, out);
        file.endHandler(v -> {
            file.close();
            out.end();
            fileClosed.set(true);
        });
        Handler<Throwable> exceptionHandler = t -> {
            if (!fileClosed.get()) {
                file.endHandler(null);
                file.close();
            }
            handler.handle(Future.failedFuture(t));
        };
        file.exceptionHandler(exceptionHandler);
        out.exceptionHandler(exceptionHandler);
        pump.start();
        return o;
    });
}
Also used : OpenOptions(io.vertx.core.file.OpenOptions) AsyncFile(io.vertx.core.file.AsyncFile) Arrays(java.util.Arrays) OptionParserException(de.undercouch.underline.OptionParserException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayList(java.util.ArrayList) Observable(rx.Observable) Pair(org.apache.commons.lang3.tuple.Pair) FileSet(org.apache.tools.ant.types.FileSet) WriteStream(io.vertx.core.streams.WriteStream) FileSystem(io.vertx.rxjava.core.file.FileSystem) Project(org.apache.tools.ant.Project) UnknownAttributes(de.undercouch.underline.UnknownAttributes) Pump(io.vertx.core.streams.Pump) AsyncResult(io.vertx.core.AsyncResult) Splitter(com.google.common.base.Splitter) OptionDesc(de.undercouch.underline.OptionDesc) PrintWriter(java.io.PrintWriter) OpenOptions(io.vertx.core.file.OpenOptions) ObservableFuture(io.vertx.rx.java.ObservableFuture) SystemUtils(org.apache.commons.lang3.SystemUtils) InputReader(de.undercouch.underline.InputReader) DurationFormat(io.georocket.util.DurationFormat) IOException(java.io.IOException) DirectoryScanner(org.apache.tools.ant.DirectoryScanner) GeoRocketClient(io.georocket.client.GeoRocketClient) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) File(java.io.File) List(java.util.List) Stream(java.util.stream.Stream) Buffer(io.vertx.core.buffer.Buffer) Paths(java.nio.file.Paths) RxHelper(io.vertx.rx.java.RxHelper) Optional(java.util.Optional) Queue(java.util.Queue) ArrayDeque(java.util.ArrayDeque) ArgumentType(de.undercouch.underline.Option.ArgumentType) Handler(io.vertx.core.Handler) FilenameUtils(org.apache.commons.io.FilenameUtils) Vertx(io.vertx.rxjava.core.Vertx) Buffer(io.vertx.core.buffer.Buffer) Pump(io.vertx.core.streams.Pump) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) FileSystem(io.vertx.rxjava.core.file.FileSystem) AsyncFile(io.vertx.core.file.AsyncFile) AsyncResult(io.vertx.core.AsyncResult)

Aggregations

AsyncResult (io.vertx.core.AsyncResult)4 Future (io.vertx.core.Future)4 Handler (io.vertx.core.Handler)4 OpenOptions (io.vertx.core.file.OpenOptions)4 ObservableFuture (io.vertx.rx.java.ObservableFuture)4 RxHelper (io.vertx.rx.java.RxHelper)4 Buffer (io.vertx.core.buffer.Buffer)3 AsyncFile (io.vertx.core.file.AsyncFile)3 File (java.io.File)3 IOException (java.io.IOException)3 Observable (rx.Observable)3 Splitter (com.google.common.base.Splitter)2 ConfigConstants (io.georocket.constants.ConfigConstants)2 HttpException (io.georocket.util.HttpException)2 Vertx (io.vertx.core.Vertx)2 FileSystem (io.vertx.core.file.FileSystem)2 Logger (io.vertx.core.logging.Logger)2 LoggerFactory (io.vertx.core.logging.LoggerFactory)2 Pump (io.vertx.core.streams.Pump)2 WriteStream (io.vertx.core.streams.WriteStream)2