Examples with StreamingParser de.lmu.ifi.dbs.elki.datasource.parser.StreamingParser used on opensource projects

Example 1 with StreamingParser

use of de.lmu.ifi.dbs.elki.datasource.parser.StreamingParser in project elki by elki-project.

the class ConcatenateFilesDatabaseConnection method loadData.

@Override
public MultipleObjectsBundle loadData() {
    MultipleObjectsBundle objects = new MultipleObjectsBundle();
    objects.appendColumn(TypeUtil.STRING, new ArrayList<>());
    for (File file : files) {
        String filestr = file.getPath();
        try (InputStream inputStream = // 
        FileUtil.tryGzipInput(new BufferedInputStream(new FileInputStream(file)))) {
            final BundleStreamSource source;
            if (parser instanceof StreamingParser) {
                final StreamingParser streamParser = (StreamingParser) parser;
                streamParser.initStream(inputStream);
                source = streamParser;
            } else {
                MultipleObjectsBundle parsingResult = parser.parse(inputStream);
                // normalize objects and transform labels
                source = parsingResult.asStream();
            }
            // NullPointerException on invalid streams
            BundleMeta meta = null;
            loop: for (Event e = source.nextEvent(); ; e = source.nextEvent()) {
                switch(e) {
                    case END_OF_STREAM:
                        break loop;
                    case META_CHANGED:
                        meta = source.getMeta();
                        for (int i = 0; i < meta.size(); i++) {
                            if (i + 1 >= objects.metaLength()) {
                                objects.appendColumn(meta.get(i), new ArrayList<>());
                            } else {
                                // Ensure compatibility:
                                if (!objects.meta(i + 1).isAssignableFromType(meta.get(i))) {
                                    throw new AbortException("Incompatible files loaded. Cannot concatenate with unaligned columns, please preprocess manually.");
                                }
                            }
                        }
                        // switch
                        break;
                    case NEXT_OBJECT:
                        Object[] o = new Object[objects.metaLength()];
                        o[0] = filestr;
                        for (int i = 0; i < meta.size(); i++) {
                            o[i + 1] = source.data(i);
                        }
                        objects.appendSimple(o);
                        // switch
                        break;
                }
            }
        } catch (IOException e) {
            throw new AbortException("Loading file " + filestr + " failed: " + e.toString(), e);
        }
    }
    parser.cleanup();
    // Invoke filters
    if (LOG.isDebugging()) {
        LOG.debugFine("Invoking filters.");
    }
    return invokeBundleFilters(objects);
}

Example 2 with StreamingParser

use of de.lmu.ifi.dbs.elki.datasource.parser.StreamingParser in project elki by elki-project.

the class InputStreamDatabaseConnection method loadData.

@Override
public MultipleObjectsBundle loadData() {
    // Run parser
    if (LOG.isDebugging()) {
        LOG.debugFine("Invoking parsers.");
    }
    // Streaming parsers may yield to stream filters immediately.
    if (parser instanceof StreamingParser) {
        final StreamingParser streamParser = (StreamingParser) parser;
        streamParser.initStream(in);
        // normalize objects and transform labels
        if (LOG.isDebugging()) {
            LOG.debugFine("Parsing as stream.");
        }
        Duration duration = LOG.isStatistics() ? LOG.newDuration(this.getClass().getName() + ".load").begin() : null;
        MultipleObjectsBundle objects = invokeStreamFilters(streamParser).asMultipleObjectsBundle();
        parser.cleanup();
        if (duration != null) {
            LOG.statistics(duration.end());
        }
        return objects;
    } else {
        // For non-streaming parsers, we first parse, then filter
        Duration duration = LOG.isStatistics() ? LOG.newDuration(this.getClass().getName() + ".parse").begin() : null;
        MultipleObjectsBundle parsingResult = parser.parse(in);
        parser.cleanup();
        if (duration != null) {
            LOG.statistics(duration.end());
        }
        // normalize objects and transform labels
        if (LOG.isDebugging()) {
            LOG.debugFine("Invoking filters.");
        }
        Duration fduration = LOG.isStatistics() ? LOG.newDuration(this.getClass().getName() + ".filter").begin() : null;
        MultipleObjectsBundle objects = invokeBundleFilters(parsingResult);
        if (fduration != null) {
            LOG.statistics(fduration.end());
        }
        return objects;
    }
}