Examples with FeedAdapter org.apache.asterix.external.dataset.adapter.FeedAdapter used on opensource projects

Example 1 with FeedAdapter

use of org.apache.asterix.external.dataset.adapter.FeedAdapter in project asterixdb by apache.

the class GenericAdapterFactory method createAdapter.

/**
     * Runs on each node controller (after serialization-deserialization)
     */
@Override
public synchronized IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
    INCServiceContext serviceCtx = ctx.getJobletContext().getServiceContext();
    INcApplicationContext appCtx = (INcApplicationContext) serviceCtx.getApplicationContext();
    try {
        restoreExternalObjects(serviceCtx, appCtx.getLibraryManager());
    } catch (Exception e) {
        LOGGER.log(Level.INFO, "Failure restoring external objects", e);
        throw HyracksDataException.create(e);
    }
    if (isFeed) {
        if (feedLogManager == null) {
            feedLogManager = FeedUtils.getFeedLogManager(ctx, partition, feedLogFileSplits);
        }
        feedLogManager.touch();
    }
    IDataFlowController controller = DataflowControllerProvider.getDataflowController(recordType, ctx, partition, dataSourceFactory, dataParserFactory, configuration, indexingOp, isFeed, feedLogManager);
    if (isFeed) {
        return new FeedAdapter((AbstractFeedDataFlowController) controller);
    } else {
        return new GenericAdapter(controller);
    }
}

Example 2 with FeedAdapter

use of org.apache.asterix.external.dataset.adapter.FeedAdapter in project asterixdb by apache.

the class FeedIntakeOperatorNodePushable method start.

@Override
protected void start() throws HyracksDataException, InterruptedException {
    try {
        writer.open();
        Thread.currentThread().setName("Intake Thread");
        FeedAdapter adapter = (FeedAdapter) adapterFactory.createAdapter(ctx, partition);
        adapterRuntimeManager = new AdapterRuntimeManager(ctx, runtimeId.getEntityId(), adapter, writer, partition);
        IFrame message = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        /*
             * Set null feed message. Feed pipeline carries with it a message with each frame
             * Initially, the message is set to a null message that can be changed by feed adapters.
             * One use case is adapters which consume data sources that allow restartability. Such adapters
             * can propagate progress information through the ingestion pipeline to storage nodes
             */
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        adapterRuntimeManager.start();
        synchronized (adapterRuntimeManager) {
            while (!adapterRuntimeManager.isDone()) {
                adapterRuntimeManager.wait();
            }
        }
        if (adapterRuntimeManager.isFailed()) {
            throw new RuntimeDataException(ErrorCode.OPERATORS_FEED_INTAKE_OPERATOR_NODE_PUSHABLE_FAIL_AT_INGESTION);
        }
    } catch (Exception e) {
        /*
             * An Interrupted Exception is thrown if the Intake job cannot progress further due to failure of another
             * node involved in the Hyracks job. As the Intake job involves only the intake operator, the exception is
             * indicative of a failure at the sibling intake operator location. The surviving intake partitions must
             * continue to live and receive data from the external source.
             */
        writer.fail();
        throw e;
    } finally {
        writer.close();
    }
}