Examples with ITupleForwarder org.apache.asterix.external.api.ITupleForwarder used on opensource projects

Example 1 with ITupleForwarder

use of org.apache.asterix.external.api.ITupleForwarder in project asterixdb by apache.

the class TestTypedAdapterFactory method createAdapter.

@Override
public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
    final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId();
    final ITupleParserFactory tupleParserFactory = new ITupleParserFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public ITupleParser createTupleParser(IHyracksTaskContext ctx) throws HyracksDataException {
            ADMDataParser parser;
            ITupleForwarder forwarder;
            ArrayTupleBuilder tb;
            IApplicationContext appCtx = (IApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
            ClusterPartition nodePartition = appCtx.getMetadataProperties().getNodePartitions().get(nodeId)[0];
            parser = new ADMDataParser(outputType, true);
            forwarder = DataflowUtils.getTupleForwarder(configuration, FeedUtils.getFeedLogManager(ctx, FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration), ExternalDataUtils.getFeedName(configuration), nodeId, nodePartition)));
            tb = new ArrayTupleBuilder(1);
            return new ITupleParser() {

                @Override
                public void parse(InputStream in, IFrameWriter writer) throws HyracksDataException {
                    try {
                        parser.setInputStream(in);
                        forwarder.initialize(ctx, writer);
                        while (true) {
                            tb.reset();
                            if (!parser.parse(tb.getDataOutput())) {
                                break;
                            }
                            tb.addFieldEndOffset();
                            forwarder.addTuple(tb);
                        }
                        forwarder.close();
                    } catch (Exception e) {
                        throw new HyracksDataException(e);
                    }
                }
            };
        }
    };
    try {
        return new TestTypedAdapter(tupleParserFactory, outputType, ctx, configuration, partition);
    } catch (IOException e) {
        throw new HyracksDataException(e);
    }
}

Example 2 with ITupleForwarder

use of org.apache.asterix.external.api.ITupleForwarder in project asterixdb by apache.

the class DataflowUtils method getTupleForwarder.

public static ITupleForwarder getTupleForwarder(Map<String, String> configuration, FeedLogManager feedLogManager) throws HyracksDataException {
    ITupleForwarder.TupleForwardPolicy policyType = null;
    String propValue = configuration.get(ITupleForwarder.FORWARD_POLICY);
    if (ExternalDataUtils.isFeed(configuration)) {
        // TODO pass this value in the configuration and avoid this check for feeds
        policyType = TupleForwardPolicy.FEED;
    } else if (propValue == null) {
        policyType = TupleForwardPolicy.FRAME_FULL;
    } else {
        policyType = TupleForwardPolicy.valueOf(propValue.trim().toUpperCase());
    }
    switch(policyType) {
        case FEED:
            return new FeedTupleForwarder(feedLogManager);
        case FRAME_FULL:
            return new FrameFullTupleForwarder();
        case COUNTER_TIMER_EXPIRED:
            return CounterTimerTupleForwarder.create(configuration);
        case RATE_CONTROLLED:
            return RateControlledTupleForwarder.create(configuration);
        default:
            throw new RuntimeDataException(ErrorCode.UTIL_DATAFLOW_UTILS_UNKNOWN_FORWARD_POLICY);
    }
}