Examples with OsmosisReader crosby.binary.osmosis.OsmosisReader used on opensource projects

Example 1 with OsmosisReader

use of crosby.binary.osmosis.OsmosisReader in project GeoGig by boundlessgeo.

the class OSMImportOp method parseDataFileAndInsert.

private OSMReport parseDataFileAndInsert(@Nullable File file, final InputStream dataIn, final EntityConverter converter) {
    final boolean pbf;
    final CompressionMethod compression;
    if (file == null) {
        pbf = false;
        compression = CompressionMethod.None;
    } else {
        pbf = file.getName().endsWith(".pbf");
        compression = resolveCompressionMethod(file);
    }
    RunnableSource reader;
    if (pbf) {
        reader = new OsmosisReader(dataIn);
    } else {
        reader = new org.locationtech.geogig.osm.internal.XmlReader(dataIn, true, compression);
    }
    final WorkingTree workTree = workingTree();
    if (!add) {
        workTree.delete(OSMUtils.NODE_TYPE_NAME);
        workTree.delete(OSMUtils.WAY_TYPE_NAME);
    }
    final int queueCapacity = 100 * 1000;
    final int timeout = 1;
    final TimeUnit timeoutUnit = TimeUnit.SECONDS;
    // With this iterator and the osm parsing happening on a separate thread, we follow a
    // producer/consumer approach so that the osm parse thread produces featrures into the
    // iterator's queue, and WorkingTree.insert consumes them on this thread
    QueueIterator<Feature> iterator = new QueueIterator<Feature>(queueCapacity, timeout, timeoutUnit);
    ProgressListener progressListener = getProgressListener();
    ConvertAndImportSink sink = new ConvertAndImportSink(converter, iterator, platform(), mapping, noRaw, new SubProgressListener(progressListener, 100));
    reader.setSink(sink);
    Thread readerThread = new Thread(reader, "osm-import-reader-thread");
    readerThread.start();
    Function<Feature, String> parentTreePathResolver = new Function<Feature, String>() {

        @Override
        public String apply(Feature input) {
            if (input instanceof MappedFeature) {
                return ((MappedFeature) input).getPath();
            }
            return input.getType().getName().getLocalPart();
        }
    };
    // used to set the task status name, but report no progress so it does not interfere
    // with the progress reported by the reader thread
    SubProgressListener noPorgressReportingListener = new SubProgressListener(progressListener, 0) {

        @Override
        public void setProgress(float progress) {
        // no-op
        }
    };
    workTree.insert(parentTreePathResolver, iterator, noPorgressReportingListener, null, null);
    if (sink.getCount() == 0) {
        throw new EmptyOSMDownloadException();
    }
    OSMReport report = new OSMReport(sink.getCount(), sink.getNodeCount(), sink.getWayCount(), sink.getUnprocessedCount(), sink.getLatestChangeset(), sink.getLatestTimestamp());
    return report;
}

Example 2 with OsmosisReader

use of crosby.binary.osmosis.OsmosisReader in project bboxdb by jnidzwetzki.

the class OSMDataConverter method start.

/**
 * Start the converter
 */
public void start() {
    try {
        // Open file handles
        for (final OSMType osmType : filter.keySet()) {
            final BufferedWriter bw = new BufferedWriter(new FileWriter(new File(output + File.separator + osmType.toString())));
            writerMap.put(osmType, bw);
        }
        System.out.format("Importing %s%n", filename);
        final OsmosisReader reader = new OsmosisReader(new FileInputStream(filename));
        reader.setSink(new Sink() {

            @Override
            public void close() {
            }

            @Override
            public void complete() {
            }

            @Override
            public void initialize(final Map<String, Object> metaData) {
            }

            @Override
            public void process(final EntityContainer entityContainer) {
                try {
                    if (entityContainer.getEntity() instanceof Node) {
                        // Nodes are cheap to handle, dispatching to another thread
                        // is more expensive
                        final Node node = (Node) entityContainer.getEntity();
                        handleNode(node);
                        statistics.incProcessedNodes();
                    } else if (entityContainer.getEntity() instanceof Way) {
                        // Ways are expensive to handle
                        final Way way = (Way) entityContainer.getEntity();
                        queue.put(way);
                        statistics.incProcessedWays();
                    }
                } catch (InterruptedException e) {
                    return;
                }
            }
        });
        // The way consumer
        for (int i = 0; i < CONSUMER_THREADS; i++) {
            threadPool.submit(new Consumer());
        }
        reader.run();
    } catch (IOException e) {
        logger.error("Got an exception during import", e);
    } finally {
        shutdown();
    }
}