Search in sources :

Example 1 with FileSystem

use of io.vertx.rxjava.core.file.FileSystem in project georocket by georocket.

the class ElasticsearchInstaller method downloadNoCheck.

/**
 * Download and extract Elasticsearch to a given location no matter if the
 * destination folder exists or not.
 * @param downloadUrl the URL to the ZIP file containing Elasticsearch
 * @param destPath the path to the destination folder where the downloaded ZIP
 * file should be extracted to
 * @param strip <code>true</code> if the first path element of all items in the
 * ZIP file should be stripped away.
 * @return emitting the path to the extracted Elasticsearch
 */
private Single<String> downloadNoCheck(String downloadUrl, String destPath, boolean strip) {
    log.info("Downloading Elasticsearch ...");
    log.info("Source: " + downloadUrl);
    log.info("Dest: " + destPath);
    // download the archive, extract it and finally delete it
    return downloadArchive(downloadUrl).flatMap(archivePath -> {
        return extractArchive(archivePath, destPath, strip).doAfterTerminate(() -> {
            FileSystem fs = vertx.fileSystem();
            fs.deleteBlocking(archivePath);
        });
    });
}
Also used : FileSystem(io.vertx.rxjava.core.file.FileSystem)

Example 2 with FileSystem

use of io.vertx.rxjava.core.file.FileSystem in project georocket by georocket.

the class ElasticsearchInstaller method downloadArchive.

/**
 * Download the ZIP file containing Elasticsearch to a temporary location
 * @param downloadUrl the URL to the ZIP file
 * @return a single emitting the location of the temporary file
 */
private Single<String> downloadArchive(String downloadUrl) {
    // create temporary file
    File archiveFile;
    try {
        archiveFile = File.createTempFile("elasticsearch", "tmp");
    } catch (IOException e) {
        return Single.error(e);
    }
    // open temporary file
    FileSystem fs = vertx.fileSystem();
    String archivePath = archiveFile.getAbsolutePath();
    OpenOptions openOptions = new OpenOptions().setCreate(true).setWrite(true).setTruncateExisting(true);
    return fs.rxOpen(archivePath, openOptions).flatMap(file -> {
        return doDownload(downloadUrl, file).doAfterTerminate(() -> {
            file.flush();
            file.close();
        }).map(v -> archivePath);
    });
}
Also used : OpenOptions(io.vertx.core.file.OpenOptions) FileSystem(io.vertx.rxjava.core.file.FileSystem) IOException(java.io.IOException) AsyncFile(io.vertx.rxjava.core.file.AsyncFile) ZipFile(java.util.zip.ZipFile) File(java.io.File)

Example 3 with FileSystem

use of io.vertx.rxjava.core.file.FileSystem in project georocket by georocket.

the class ImporterVerticle method onImport.

/**
 * Receives a name of a file to import
 * @param msg the event bus message containing the filename
 * @return a single that will emit an item when the file has
 * been imported
 */
protected Single<Void> onImport(Message<JsonObject> msg) {
    JsonObject body = msg.body();
    String filename = body.getString("filename");
    String filepath = incoming + "/" + filename;
    String layer = body.getString("layer", "/");
    String contentType = body.getString("contentType");
    String correlationId = body.getString("correlationId");
    String fallbackCRSString = body.getString("fallbackCRSString");
    // get tags
    JsonArray tagsArr = body.getJsonArray("tags");
    List<String> tags = tagsArr != null ? tagsArr.stream().flatMap(o -> o != null ? Stream.of(o.toString()) : Stream.of()).collect(Collectors.toList()) : null;
    // get properties
    JsonObject propertiesObj = body.getJsonObject("properties");
    Map<String, Object> properties = propertiesObj != null ? propertiesObj.getMap() : null;
    // generate timestamp for this import
    long timestamp = System.currentTimeMillis();
    onImportingStarted(correlationId, filepath, contentType, layer, tagsArr, timestamp);
    FileSystem fs = vertx.fileSystem();
    OpenOptions openOptions = new OpenOptions().setCreate(false).setWrite(false);
    return fs.rxOpen(filepath, openOptions).flatMap(f -> importFile(contentType, f, correlationId, filename, timestamp, layer, tags, properties, fallbackCRSString).doAfterTerminate(() -> {
        // delete file from 'incoming' folder
        log.info("Deleting " + filepath + " from incoming folder");
        f.rxClose().flatMap(v -> fs.rxDelete(filepath)).subscribe(v -> {
        }, err -> {
            log.error("Could not delete file from 'incoming' folder", err);
        });
    })).doOnSuccess(chunkCount -> {
        onImportingFinished(correlationId, filepath, contentType, layer, chunkCount, System.currentTimeMillis() - timestamp, null);
    }).doOnError(err -> {
        onImportingFinished(correlationId, filepath, contentType, layer, null, System.currentTimeMillis() - timestamp, err);
    }).map(count -> null);
}
Also used : JsonArray(io.vertx.core.json.JsonArray) OpenOptions(io.vertx.core.file.OpenOptions) Buffer(io.vertx.rxjava.core.buffer.Buffer) IndexMeta(io.georocket.storage.IndexMeta) StringWindow(io.georocket.util.StringWindow) XMLParserOperator(io.georocket.util.XMLParserOperator) ChunkMeta(io.georocket.storage.ChunkMeta) Window(io.georocket.util.Window) LoggerFactory(io.vertx.core.logging.LoggerFactory) RxStore(io.georocket.storage.RxStore) AbstractVerticle(io.vertx.rxjava.core.AbstractVerticle) Single(rx.Single) JsonParserOperator(io.georocket.util.JsonParserOperator) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Result(io.georocket.input.Splitter.Result) Map(java.util.Map) FileSystem(io.vertx.rxjava.core.file.FileSystem) JsonObject(io.vertx.core.json.JsonObject) StoreFactory(io.georocket.storage.StoreFactory) UTF8BomFilter(io.georocket.util.UTF8BomFilter) Logger(io.vertx.core.logging.Logger) Message(io.vertx.rxjava.core.eventbus.Message) XMLSplitter(io.georocket.input.xml.XMLSplitter) OpenOptions(io.vertx.core.file.OpenOptions) GeoJsonSplitter(io.georocket.input.geojson.GeoJsonSplitter) NoStackTraceThrowable(io.vertx.core.impl.NoStackTraceThrowable) FirstLevelSplitter(io.georocket.input.xml.FirstLevelSplitter) Collectors(java.util.stream.Collectors) XMLCRSIndexer(io.georocket.index.xml.XMLCRSIndexer) JsonArray(io.vertx.core.json.JsonArray) List(java.util.List) Stream(java.util.stream.Stream) MimeTypeUtils.belongsTo(io.georocket.util.MimeTypeUtils.belongsTo) ReadStream(io.vertx.rxjava.core.streams.ReadStream) AddressConstants(io.georocket.constants.AddressConstants) RxUtils(io.georocket.util.RxUtils) ConfigConstants(io.georocket.constants.ConfigConstants) FileSystem(io.vertx.rxjava.core.file.FileSystem) JsonObject(io.vertx.core.json.JsonObject) JsonObject(io.vertx.core.json.JsonObject)

Example 4 with FileSystem

use of io.vertx.rxjava.core.file.FileSystem in project georocket by georocket.

the class ImportCommand method importFile.

/**
 * Upload a file to GeoRocket
 * @param path path to file to import
 * @param client the GeoRocket client
 * @param vertx the Vert.x instance
 * @return an observable that will emit when the file has been uploaded
 */
protected Observable<Void> importFile(String path, GeoRocketClient client, Vertx vertx) {
    // open file
    FileSystem fs = vertx.fileSystem();
    OpenOptions openOptions = new OpenOptions().setCreate(false).setWrite(false);
    return fs.rxOpen(path, openOptions).flatMap(f -> fs.rxProps(path).map(props -> Pair.of(f, props.size()))).flatMapObservable(f -> {
        ObservableFuture<Void> o = RxHelper.observableFuture();
        Handler<AsyncResult<Void>> handler = o.toHandler();
        AsyncFile file = f.getLeft().getDelegate();
        WriteStream<Buffer> out = client.getStore().startImport(layer, tags, properties, Optional.of(f.getRight()), fallbackCRS, handler);
        AtomicBoolean fileClosed = new AtomicBoolean();
        Pump pump = Pump.pump(file, out);
        file.endHandler(v -> {
            file.close();
            out.end();
            fileClosed.set(true);
        });
        Handler<Throwable> exceptionHandler = t -> {
            if (!fileClosed.get()) {
                file.endHandler(null);
                file.close();
            }
            handler.handle(Future.failedFuture(t));
        };
        file.exceptionHandler(exceptionHandler);
        out.exceptionHandler(exceptionHandler);
        pump.start();
        return o;
    });
}
Also used : OpenOptions(io.vertx.core.file.OpenOptions) AsyncFile(io.vertx.core.file.AsyncFile) Arrays(java.util.Arrays) OptionParserException(de.undercouch.underline.OptionParserException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayList(java.util.ArrayList) Observable(rx.Observable) Pair(org.apache.commons.lang3.tuple.Pair) FileSet(org.apache.tools.ant.types.FileSet) WriteStream(io.vertx.core.streams.WriteStream) FileSystem(io.vertx.rxjava.core.file.FileSystem) Project(org.apache.tools.ant.Project) UnknownAttributes(de.undercouch.underline.UnknownAttributes) Pump(io.vertx.core.streams.Pump) AsyncResult(io.vertx.core.AsyncResult) Splitter(com.google.common.base.Splitter) OptionDesc(de.undercouch.underline.OptionDesc) PrintWriter(java.io.PrintWriter) OpenOptions(io.vertx.core.file.OpenOptions) ObservableFuture(io.vertx.rx.java.ObservableFuture) SystemUtils(org.apache.commons.lang3.SystemUtils) InputReader(de.undercouch.underline.InputReader) DurationFormat(io.georocket.util.DurationFormat) IOException(java.io.IOException) DirectoryScanner(org.apache.tools.ant.DirectoryScanner) GeoRocketClient(io.georocket.client.GeoRocketClient) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) File(java.io.File) List(java.util.List) Stream(java.util.stream.Stream) Buffer(io.vertx.core.buffer.Buffer) Paths(java.nio.file.Paths) RxHelper(io.vertx.rx.java.RxHelper) Optional(java.util.Optional) Queue(java.util.Queue) ArrayDeque(java.util.ArrayDeque) ArgumentType(de.undercouch.underline.Option.ArgumentType) Handler(io.vertx.core.Handler) FilenameUtils(org.apache.commons.io.FilenameUtils) Vertx(io.vertx.rxjava.core.Vertx) Buffer(io.vertx.core.buffer.Buffer) Pump(io.vertx.core.streams.Pump) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) FileSystem(io.vertx.rxjava.core.file.FileSystem) AsyncFile(io.vertx.core.file.AsyncFile) AsyncResult(io.vertx.core.AsyncResult)

Aggregations

FileSystem (io.vertx.rxjava.core.file.FileSystem)4 OpenOptions (io.vertx.core.file.OpenOptions)3 File (java.io.File)2 IOException (java.io.IOException)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 Splitter (com.google.common.base.Splitter)1 InputReader (de.undercouch.underline.InputReader)1 ArgumentType (de.undercouch.underline.Option.ArgumentType)1 OptionDesc (de.undercouch.underline.OptionDesc)1 OptionParserException (de.undercouch.underline.OptionParserException)1 UnknownAttributes (de.undercouch.underline.UnknownAttributes)1 GeoRocketClient (io.georocket.client.GeoRocketClient)1 AddressConstants (io.georocket.constants.AddressConstants)1 ConfigConstants (io.georocket.constants.ConfigConstants)1 XMLCRSIndexer (io.georocket.index.xml.XMLCRSIndexer)1 Result (io.georocket.input.Splitter.Result)1 GeoJsonSplitter (io.georocket.input.geojson.GeoJsonSplitter)1 FirstLevelSplitter (io.georocket.input.xml.FirstLevelSplitter)1