use of io.georocket.util.Window in project georocket by georocket.
the class ImporterVerticle method importJSON.
/**
* Imports a JSON file from the given input stream into the store
* @param f the JSON file to read
* @param correlationId a unique identifier for this import process
* @param filename the name of the file currently being imported
* @param timestamp denotes when the import process has started
* @param layer the layer where the file should be stored (may be null)
* @param tags the list of tags to attach to the file (may be null)
* @param properties the map of properties to attach to the file (may be null)
* @return a single that will emit when the file has been imported
*/
protected Single<Integer> importJSON(ReadStream<Buffer> f, String correlationId, String filename, long timestamp, String layer, List<String> tags, Map<String, Object> properties) {
UTF8BomFilter bomFilter = new UTF8BomFilter();
StringWindow window = new StringWindow();
GeoJsonSplitter splitter = new GeoJsonSplitter(window);
AtomicInteger processing = new AtomicInteger(0);
return f.toObservable().map(buf -> (io.vertx.core.buffer.Buffer) buf.getDelegate()).map(bomFilter::filter).doOnNext(window::append).lift(new JsonParserOperator()).flatMap(splitter::onEventObservable).flatMapSingle(result -> {
IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, null);
return addToStoreWithPause(result, layer, indexMeta, f, processing);
}).count().toSingle();
}
use of io.georocket.util.Window in project georocket by georocket.
the class ImporterVerticle method importXML.
/**
* Imports an XML file from the given input stream into the store
* @param f the XML file to read
* @param correlationId a unique identifier for this import process
* @param filename the name of the file currently being imported
* @param timestamp denotes when the import process has started
* @param layer the layer where the file should be stored (may be null)
* @param tags the list of tags to attach to the file (may be null)
* @param properties the map of properties to attach to the file (may be null)
* @param fallbackCRSString the CRS which should be used if the imported
* file does not specify one (may be <code>null</code>)
* @return a single that will emit when the file has been imported
*/
protected Single<Integer> importXML(ReadStream<Buffer> f, String correlationId, String filename, long timestamp, String layer, List<String> tags, Map<String, Object> properties, String fallbackCRSString) {
UTF8BomFilter bomFilter = new UTF8BomFilter();
Window window = new Window();
XMLSplitter splitter = new FirstLevelSplitter(window);
AtomicInteger processing = new AtomicInteger(0);
XMLCRSIndexer crsIndexer = new XMLCRSIndexer();
return f.toObservable().map(buf -> (io.vertx.core.buffer.Buffer) buf.getDelegate()).map(bomFilter::filter).doOnNext(window::append).lift(new XMLParserOperator()).doOnNext(e -> {
// save the first CRS found in the file
if (crsIndexer.getCRS() == null) {
crsIndexer.onEvent(e);
}
}).flatMap(splitter::onEventObservable).flatMapSingle(result -> {
String crsString = fallbackCRSString;
if (crsIndexer.getCRS() != null) {
crsString = crsIndexer.getCRS();
}
IndexMeta indexMeta = new IndexMeta(correlationId, filename, timestamp, tags, properties, crsString);
return addToStoreWithPause(result, layer, indexMeta, f, processing);
}).count().toSingle();
}
use of io.georocket.util.Window in project georocket by georocket.
the class FirstLevelSplitterTest method split.
/**
* Use the {@link FirstLevelSplitter} and split an XML string
* @param xml the XML string
* @return the chunks created by the splitter
* @throws Exception if the XML string could not be parsed
*/
private List<Result<XMLChunkMeta>> split(String xml) throws Exception {
Window window = new Window();
window.append(Buffer.buffer(xml));
AsyncXMLInputFactory xmlInputFactory = new InputFactoryImpl();
AsyncXMLStreamReader<AsyncByteArrayFeeder> reader = xmlInputFactory.createAsyncForByteArray();
byte[] xmlBytes = xml.getBytes(StandardCharsets.UTF_8);
reader.getInputFeeder().feedInput(xmlBytes, 0, xmlBytes.length);
FirstLevelSplitter splitter = new FirstLevelSplitter(window);
List<Result<XMLChunkMeta>> chunks = new ArrayList<>();
while (reader.hasNext()) {
int event = reader.next();
if (event == AsyncXMLStreamReader.EVENT_INCOMPLETE) {
reader.close();
continue;
}
int pos = reader.getLocation().getCharacterOffset();
Result<XMLChunkMeta> chunk = splitter.onEvent(new XMLStreamEvent(event, pos, reader));
if (chunk != null) {
chunks.add(chunk);
}
}
return chunks;
}
Aggregations