Search in sources :

Example 1 with IRecordReaderFactory

use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.

the class DataflowControllerProvider method getDataflowController.

// TODO: Instead, use a factory just like data source and data parser.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static IDataFlowController getDataflowController(ARecordType recordType, IHyracksTaskContext ctx, int partition, IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory, Map<String, String> configuration, boolean indexingOp, boolean isFeed, FeedLogManager feedLogManager) throws HyracksDataException {
    try {
        switch(dataSourceFactory.getDataSourceType()) {
            case RECORDS:
                IRecordReaderFactory<?> recordReaderFactory = (IRecordReaderFactory<?>) dataSourceFactory;
                IRecordReader<?> recordReader = recordReaderFactory.createRecordReader(ctx, partition);
                IRecordDataParserFactory<?> recordParserFactory = (IRecordDataParserFactory<?>) dataParserFactory;
                IRecordDataParser<?> dataParser = recordParserFactory.createRecordParser(ctx);
                if (indexingOp) {
                    return new IndexingDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, ((IIndexingDatasource) recordReader).getIndexer());
                } else if (isFeed) {
                    FeedTupleForwarder tupleForwarder = (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager);
                    boolean isChangeFeed = ExternalDataUtils.isChangeFeed(configuration);
                    boolean isRecordWithMeta = ExternalDataUtils.isRecordWithMeta(configuration);
                    if (isRecordWithMeta) {
                        if (isChangeFeed) {
                            int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
                            return new ChangeFeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 2, (IRecordWithMetadataParser) dataParser, recordReader);
                        } else {
                            return new FeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, 2, (IRecordWithMetadataParser) dataParser, recordReader);
                        }
                    } else if (isChangeFeed) {
                        int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
                        return new ChangeFeedDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 1, (IRecordWithPKDataParser) dataParser, recordReader);
                    } else {
                        return new FeedRecordDataFlowController(ctx, tupleForwarder, feedLogManager, 1, dataParser, recordReader);
                    }
                } else {
                    return new RecordDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, 1);
                }
            case STREAM:
                IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
                AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
                IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
                IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
                streamParser.setInputStream(stream);
                if (isFeed) {
                    return new FeedStreamDataFlowController(ctx, (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager), feedLogManager, streamParser, stream);
                } else {
                    return new StreamDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, null), streamParser);
                }
            default:
                throw new RuntimeDataException(ErrorCode.PROVIDER_DATAFLOW_CONTROLLER_UNKNOWN_DATA_SOURCE, dataSourceFactory.getDataSourceType());
        }
    } catch (IOException | AsterixException e) {
        throw new HyracksDataException(e);
    }
}
Also used : ChangeFeedDataFlowController(org.apache.asterix.external.dataflow.ChangeFeedDataFlowController) IndexingDataFlowController(org.apache.asterix.external.dataflow.IndexingDataFlowController) IRecordWithMetadataParser(org.apache.asterix.external.api.IRecordWithMetadataParser) AsterixException(org.apache.asterix.common.exceptions.AsterixException) FeedTupleForwarder(org.apache.asterix.external.dataflow.FeedTupleForwarder) ChangeFeedWithMetaDataFlowController(org.apache.asterix.external.dataflow.ChangeFeedWithMetaDataFlowController) FeedWithMetaDataFlowController(org.apache.asterix.external.dataflow.FeedWithMetaDataFlowController) FeedStreamDataFlowController(org.apache.asterix.external.dataflow.FeedStreamDataFlowController) FeedStreamDataFlowController(org.apache.asterix.external.dataflow.FeedStreamDataFlowController) StreamDataFlowController(org.apache.asterix.external.dataflow.StreamDataFlowController) RuntimeDataException(org.apache.asterix.common.exceptions.RuntimeDataException) RecordDataFlowController(org.apache.asterix.external.dataflow.RecordDataFlowController) FeedRecordDataFlowController(org.apache.asterix.external.dataflow.FeedRecordDataFlowController) IStreamDataParser(org.apache.asterix.external.api.IStreamDataParser) IStreamDataParserFactory(org.apache.asterix.external.api.IStreamDataParserFactory) IRecordDataParserFactory(org.apache.asterix.external.api.IRecordDataParserFactory) FeedRecordDataFlowController(org.apache.asterix.external.dataflow.FeedRecordDataFlowController) IInputStreamFactory(org.apache.asterix.external.api.IInputStreamFactory) IOException(java.io.IOException) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) IRecordReaderFactory(org.apache.asterix.external.api.IRecordReaderFactory) ChangeFeedWithMetaDataFlowController(org.apache.asterix.external.dataflow.ChangeFeedWithMetaDataFlowController) AsterixInputStream(org.apache.asterix.external.api.AsterixInputStream)

Example 2 with IRecordReaderFactory

use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.

the class DatasourceFactoryProvider method initFactories.

protected static Map<String, Class> initFactories() throws AsterixException {
    Map<String, Class> factories = new HashMap<>();
    ClassLoader cl = ParserFactoryProvider.class.getClassLoader();
    final Charset encoding = Charset.forName("UTF-8");
    try {
        Enumeration<URL> urls = cl.getResources(RESOURCE);
        for (URL url : Collections.list(urls)) {
            InputStream is = url.openStream();
            String config = IOUtils.toString(is, encoding);
            is.close();
            String[] classNames = config.split("\n");
            for (String className : classNames) {
                if (className.startsWith("#")) {
                    continue;
                }
                final Class<?> clazz = Class.forName(className);
                List<String> formats = ((IRecordReaderFactory) clazz.newInstance()).getRecordReaderNames();
                for (String format : formats) {
                    if (factories.containsKey(format)) {
                        throw new AsterixException(ErrorCode.PROVIDER_DATASOURCE_FACTORY_DUPLICATE_FORMAT_MAPPING, format);
                    }
                    factories.put(format, clazz);
                }
            }
        }
    } catch (IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
        throw new AsterixException(e);
    }
    return factories;
}
Also used : HashMap(java.util.HashMap) InputStream(java.io.InputStream) Charset(java.nio.charset.Charset) IOException(java.io.IOException) URL(java.net.URL) IRecordReaderFactory(org.apache.asterix.external.api.IRecordReaderFactory) AsterixException(org.apache.asterix.common.exceptions.AsterixException)

Example 3 with IRecordReaderFactory

use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.

the class ExternalDataUtils method createExternalRecordReaderFactory.

public static IRecordReaderFactory<?> createExternalRecordReaderFactory(ILibraryManager libraryManager, Map<String, String> configuration) throws AsterixException {
    String readerFactory = configuration.get(ExternalDataConstants.KEY_READER_FACTORY);
    if (readerFactory == null) {
        throw new AsterixException("to use " + ExternalDataConstants.EXTERNAL + " reader, the parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must be specified.");
    }
    String[] libraryAndFactory = readerFactory.split(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR);
    if (libraryAndFactory.length != 2) {
        throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must follow the format \"DataverseName.LibraryName#ReaderFactoryFullyQualifiedName\"");
    }
    String[] dataverseAndLibrary = libraryAndFactory[0].split(".");
    if (dataverseAndLibrary.length != 2) {
        throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must follow the format \"DataverseName.LibraryName#ReaderFactoryFullyQualifiedName\"");
    }
    ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverseAndLibrary[0], dataverseAndLibrary[1]);
    try {
        return (IRecordReaderFactory<?>) classLoader.loadClass(libraryAndFactory[1]).newInstance();
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        throw new AsterixException("Failed to create record reader factory", e);
    }
}
Also used : IRecordReaderFactory(org.apache.asterix.external.api.IRecordReaderFactory) AsterixException(org.apache.asterix.common.exceptions.AsterixException)

Aggregations

AsterixException (org.apache.asterix.common.exceptions.AsterixException)3 IRecordReaderFactory (org.apache.asterix.external.api.IRecordReaderFactory)3 IOException (java.io.IOException)2 InputStream (java.io.InputStream)1 URL (java.net.URL)1 Charset (java.nio.charset.Charset)1 HashMap (java.util.HashMap)1 RuntimeDataException (org.apache.asterix.common.exceptions.RuntimeDataException)1 AsterixInputStream (org.apache.asterix.external.api.AsterixInputStream)1 IInputStreamFactory (org.apache.asterix.external.api.IInputStreamFactory)1 IRecordDataParserFactory (org.apache.asterix.external.api.IRecordDataParserFactory)1 IRecordWithMetadataParser (org.apache.asterix.external.api.IRecordWithMetadataParser)1 IStreamDataParser (org.apache.asterix.external.api.IStreamDataParser)1 IStreamDataParserFactory (org.apache.asterix.external.api.IStreamDataParserFactory)1 ChangeFeedDataFlowController (org.apache.asterix.external.dataflow.ChangeFeedDataFlowController)1 ChangeFeedWithMetaDataFlowController (org.apache.asterix.external.dataflow.ChangeFeedWithMetaDataFlowController)1 FeedRecordDataFlowController (org.apache.asterix.external.dataflow.FeedRecordDataFlowController)1 FeedStreamDataFlowController (org.apache.asterix.external.dataflow.FeedStreamDataFlowController)1 FeedTupleForwarder (org.apache.asterix.external.dataflow.FeedTupleForwarder)1 FeedWithMetaDataFlowController (org.apache.asterix.external.dataflow.FeedWithMetaDataFlowController)1