use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.
the class DataflowControllerProvider method getDataflowController.
// TODO: Instead, use a factory just like data source and data parser.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static IDataFlowController getDataflowController(ARecordType recordType, IHyracksTaskContext ctx, int partition, IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory, Map<String, String> configuration, boolean indexingOp, boolean isFeed, FeedLogManager feedLogManager) throws HyracksDataException {
try {
switch(dataSourceFactory.getDataSourceType()) {
case RECORDS:
IRecordReaderFactory<?> recordReaderFactory = (IRecordReaderFactory<?>) dataSourceFactory;
IRecordReader<?> recordReader = recordReaderFactory.createRecordReader(ctx, partition);
IRecordDataParserFactory<?> recordParserFactory = (IRecordDataParserFactory<?>) dataParserFactory;
IRecordDataParser<?> dataParser = recordParserFactory.createRecordParser(ctx);
if (indexingOp) {
return new IndexingDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, ((IIndexingDatasource) recordReader).getIndexer());
} else if (isFeed) {
FeedTupleForwarder tupleForwarder = (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager);
boolean isChangeFeed = ExternalDataUtils.isChangeFeed(configuration);
boolean isRecordWithMeta = ExternalDataUtils.isRecordWithMeta(configuration);
if (isRecordWithMeta) {
if (isChangeFeed) {
int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
return new ChangeFeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 2, (IRecordWithMetadataParser) dataParser, recordReader);
} else {
return new FeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, 2, (IRecordWithMetadataParser) dataParser, recordReader);
}
} else if (isChangeFeed) {
int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
return new ChangeFeedDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 1, (IRecordWithPKDataParser) dataParser, recordReader);
} else {
return new FeedRecordDataFlowController(ctx, tupleForwarder, feedLogManager, 1, dataParser, recordReader);
}
} else {
return new RecordDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, 1);
}
case STREAM:
IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
streamParser.setInputStream(stream);
if (isFeed) {
return new FeedStreamDataFlowController(ctx, (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager), feedLogManager, streamParser, stream);
} else {
return new StreamDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, null), streamParser);
}
default:
throw new RuntimeDataException(ErrorCode.PROVIDER_DATAFLOW_CONTROLLER_UNKNOWN_DATA_SOURCE, dataSourceFactory.getDataSourceType());
}
} catch (IOException | AsterixException e) {
throw new HyracksDataException(e);
}
}
use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.
the class DatasourceFactoryProvider method initFactories.
protected static Map<String, Class> initFactories() throws AsterixException {
Map<String, Class> factories = new HashMap<>();
ClassLoader cl = ParserFactoryProvider.class.getClassLoader();
final Charset encoding = Charset.forName("UTF-8");
try {
Enumeration<URL> urls = cl.getResources(RESOURCE);
for (URL url : Collections.list(urls)) {
InputStream is = url.openStream();
String config = IOUtils.toString(is, encoding);
is.close();
String[] classNames = config.split("\n");
for (String className : classNames) {
if (className.startsWith("#")) {
continue;
}
final Class<?> clazz = Class.forName(className);
List<String> formats = ((IRecordReaderFactory) clazz.newInstance()).getRecordReaderNames();
for (String format : formats) {
if (factories.containsKey(format)) {
throw new AsterixException(ErrorCode.PROVIDER_DATASOURCE_FACTORY_DUPLICATE_FORMAT_MAPPING, format);
}
factories.put(format, clazz);
}
}
}
} catch (IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
throw new AsterixException(e);
}
return factories;
}
use of org.apache.asterix.external.api.IRecordReaderFactory in project asterixdb by apache.
the class ExternalDataUtils method createExternalRecordReaderFactory.
public static IRecordReaderFactory<?> createExternalRecordReaderFactory(ILibraryManager libraryManager, Map<String, String> configuration) throws AsterixException {
String readerFactory = configuration.get(ExternalDataConstants.KEY_READER_FACTORY);
if (readerFactory == null) {
throw new AsterixException("to use " + ExternalDataConstants.EXTERNAL + " reader, the parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must be specified.");
}
String[] libraryAndFactory = readerFactory.split(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR);
if (libraryAndFactory.length != 2) {
throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must follow the format \"DataverseName.LibraryName#ReaderFactoryFullyQualifiedName\"");
}
String[] dataverseAndLibrary = libraryAndFactory[0].split(".");
if (dataverseAndLibrary.length != 2) {
throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER_FACTORY + " must follow the format \"DataverseName.LibraryName#ReaderFactoryFullyQualifiedName\"");
}
ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverseAndLibrary[0], dataverseAndLibrary[1]);
try {
return (IRecordReaderFactory<?>) classLoader.loadClass(libraryAndFactory[1]).newInstance();
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new AsterixException("Failed to create record reader factory", e);
}
}
Aggregations