Examples with StreamRecordReader org.apache.asterix.external.input.record.reader.stream.StreamRecordReader used on opensource projects

Example 1 with StreamRecordReader

use of org.apache.asterix.external.input.record.reader.stream.StreamRecordReader in project asterixdb by apache.

the class StreamRecordReaderProvider method initRecordReaders.

protected static Map<String, List<Pair<String[], Class>>> initRecordReaders() throws AsterixException {
    Map<String, List<Pair<String[], Class>>> recordReaders = new HashMap<>();
    ClassLoader cl = StreamRecordReaderProvider.class.getClassLoader();
    final Charset encoding = Charset.forName("UTF-8");
    try {
        Enumeration<URL> urls = cl.getResources(RESOURCE);
        for (URL url : Collections.list(urls)) {
            InputStream is = url.openStream();
            String config = IOUtils.toString(is, encoding);
            is.close();
            String[] classNames = config.split("\n");
            for (String className : classNames) {
                if (className.startsWith("#")) {
                    continue;
                }
                final Class<?> clazz = Class.forName(className);
                StreamRecordReader newInstance = (StreamRecordReader) clazz.getConstructor().newInstance();
                List<String> formats = newInstance.getRecordReaderFormats();
                String[] configs = newInstance.getRequiredConfigs().split(":");
                for (String format : formats) {
                    if (!recordReaders.containsKey(format)) {
                        recordReaders.put(format, new ArrayList<>());
                    }
                    recordReaders.get(format).add(Pair.of(configs, clazz));
                }
            }
        }
    } catch (IOException | ClassNotFoundException | InvocationTargetException | IllegalAccessException | NoSuchMethodException | InstantiationException e) {
        throw new AsterixException(e);
    }
    return recordReaders;
}

Example 2 with StreamRecordReader

use of org.apache.asterix.external.input.record.reader.stream.StreamRecordReader in project asterixdb by apache.

the class HDFSDataSourceFactory method createRecordReader.

/**
     * HDFS Datasource is a special case in two ways:
     * 1. It supports indexing.
     * 2. It returns input as a set of writable object that we sometimes internally transform into a byte stream
     * Hence, it can produce:
     * 1. StreamRecordReader: When we transform the input into a byte stream.
     * 2. Indexing Stream Record Reader: When we transform the input into a byte stream and perform indexing.
     * 3. HDFS Record Reader: When we simply pass the Writable object as it is to the parser.
     */
@Override
public IRecordReader<? extends Object> createRecordReader(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
    try {
        IExternalIndexer indexer = files == null ? null : ExternalIndexerProvider.getIndexer(configuration);
        if (recordReaderClazz != null) {
            StreamRecordReader streamReader = (StreamRecordReader) recordReaderClazz.getConstructor().newInstance();
            streamReader.configure(createInputStream(ctx, partition, indexer), configuration);
            if (indexer != null) {
                return new IndexingStreamRecordReader(streamReader, indexer);
            } else {
                return streamReader;
            }
        }
        restoreConfig(ctx);
        return new HDFSRecordReader<>(read, inputSplits, readSchedule, nodeName, conf, files, indexer);
    } catch (Exception e) {
        throw new HyracksDataException(e);
    }
}