Examples with FeedTupleForwarder org.apache.asterix.external.dataflow.FeedTupleForwarder used on opensource projects

Example 1 with FeedTupleForwarder

use of org.apache.asterix.external.dataflow.FeedTupleForwarder in project asterixdb by apache.

the class DataflowControllerProvider method getDataflowController.

// TODO: Instead, use a factory just like data source and data parser.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static IDataFlowController getDataflowController(ARecordType recordType, IHyracksTaskContext ctx, int partition, IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory, Map<String, String> configuration, boolean indexingOp, boolean isFeed, FeedLogManager feedLogManager) throws HyracksDataException {
    try {
        switch(dataSourceFactory.getDataSourceType()) {
            case RECORDS:
                IRecordReaderFactory<?> recordReaderFactory = (IRecordReaderFactory<?>) dataSourceFactory;
                IRecordReader<?> recordReader = recordReaderFactory.createRecordReader(ctx, partition);
                IRecordDataParserFactory<?> recordParserFactory = (IRecordDataParserFactory<?>) dataParserFactory;
                IRecordDataParser<?> dataParser = recordParserFactory.createRecordParser(ctx);
                if (indexingOp) {
                    return new IndexingDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, ((IIndexingDatasource) recordReader).getIndexer());
                } else if (isFeed) {
                    FeedTupleForwarder tupleForwarder = (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager);
                    boolean isChangeFeed = ExternalDataUtils.isChangeFeed(configuration);
                    boolean isRecordWithMeta = ExternalDataUtils.isRecordWithMeta(configuration);
                    if (isRecordWithMeta) {
                        if (isChangeFeed) {
                            int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
                            return new ChangeFeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 2, (IRecordWithMetadataParser) dataParser, recordReader);
                        } else {
                            return new FeedWithMetaDataFlowController(ctx, tupleForwarder, feedLogManager, 2, (IRecordWithMetadataParser) dataParser, recordReader);
                        }
                    } else if (isChangeFeed) {
                        int numOfKeys = ExternalDataUtils.getNumberOfKeys(configuration);
                        return new ChangeFeedDataFlowController(ctx, tupleForwarder, feedLogManager, numOfKeys + 1, (IRecordWithPKDataParser) dataParser, recordReader);
                    } else {
                        return new FeedRecordDataFlowController(ctx, tupleForwarder, feedLogManager, 1, dataParser, recordReader);
                    }
                } else {
                    return new RecordDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, feedLogManager), dataParser, recordReader, 1);
                }
            case STREAM:
                IInputStreamFactory streamFactory = (IInputStreamFactory) dataSourceFactory;
                AsterixInputStream stream = streamFactory.createInputStream(ctx, partition);
                IStreamDataParserFactory streamParserFactory = (IStreamDataParserFactory) dataParserFactory;
                IStreamDataParser streamParser = streamParserFactory.createInputStreamParser(ctx, partition);
                streamParser.setInputStream(stream);
                if (isFeed) {
                    return new FeedStreamDataFlowController(ctx, (FeedTupleForwarder) DataflowUtils.getTupleForwarder(configuration, feedLogManager), feedLogManager, streamParser, stream);
                } else {
                    return new StreamDataFlowController(ctx, DataflowUtils.getTupleForwarder(configuration, null), streamParser);
                }
            default:
                throw new RuntimeDataException(ErrorCode.PROVIDER_DATAFLOW_CONTROLLER_UNKNOWN_DATA_SOURCE, dataSourceFactory.getDataSourceType());
        }
    } catch (IOException | AsterixException e) {
        throw new HyracksDataException(e);
    }
}

Example 2 with FeedTupleForwarder

use of org.apache.asterix.external.dataflow.FeedTupleForwarder in project asterixdb by apache.

the class DataflowUtils method getTupleForwarder.

public static ITupleForwarder getTupleForwarder(Map<String, String> configuration, FeedLogManager feedLogManager) throws HyracksDataException {
    ITupleForwarder.TupleForwardPolicy policyType = null;
    String propValue = configuration.get(ITupleForwarder.FORWARD_POLICY);
    if (ExternalDataUtils.isFeed(configuration)) {
        // TODO pass this value in the configuration and avoid this check for feeds
        policyType = TupleForwardPolicy.FEED;
    } else if (propValue == null) {
        policyType = TupleForwardPolicy.FRAME_FULL;
    } else {
        policyType = TupleForwardPolicy.valueOf(propValue.trim().toUpperCase());
    }
    switch(policyType) {
        case FEED:
            return new FeedTupleForwarder(feedLogManager);
        case FRAME_FULL:
            return new FrameFullTupleForwarder();
        case COUNTER_TIMER_EXPIRED:
            return CounterTimerTupleForwarder.create(configuration);
        case RATE_CONTROLLED:
            return RateControlledTupleForwarder.create(configuration);
        default:
            throw new RuntimeDataException(ErrorCode.UTIL_DATAFLOW_UTILS_UNKNOWN_FORWARD_POLICY);
    }
}