Examples with IAdapterFactory org.apache.asterix.external.api.IAdapterFactory used on opensource projects

Example 6 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class FeedIntakeOperatorDescriptor method createExternalAdapterFactory.

private IAdapterFactory createExternalAdapterFactory(IHyracksTaskContext ctx) throws HyracksDataException {
    IAdapterFactory adapterFactory;
    INcApplicationContext runtimeCtx = (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
    ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
    ClassLoader classLoader = libraryManager.getLibraryClassLoader(feedId.getDataverse(), adaptorLibraryName);
    if (classLoader != null) {
        try {
            adapterFactory = (IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
            adapterFactory.setOutputType(adapterOutputType);
            adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration);
        } catch (Exception e) {
            throw new HyracksDataException(e);
        }
    } else {
        RuntimeDataException err = new RuntimeDataException(ErrorCode.OPERATORS_FEED_INTAKE_OPERATOR_DESCRIPTOR_CLASSLOADER_NOT_CONFIGURED, adaptorLibraryName, feedId.getDataverse());
        LOGGER.severe(err.getMessage());
        throw err;
    }
    return adapterFactory;
}

Example 7 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class FeedOperations method buildFeedIntakeJobSpec.

private static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(Feed feed, MetadataProvider metadataProvider, FeedPolicyAccessor policyAccessor) throws Exception {
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    spec.setFrameSize(metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize());
    IAdapterFactory adapterFactory;
    IOperatorDescriptor feedIngestor;
    AlgebricksPartitionConstraint ingesterPc;
    Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> t = metadataProvider.buildFeedIntakeRuntime(spec, feed, policyAccessor);
    feedIngestor = t.first;
    ingesterPc = t.second;
    adapterFactory = t.third;
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, feedIngestor, ingesterPc);
    NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, ingesterPc);
    spec.connect(new OneToOneConnectorDescriptor(spec), feedIngestor, 0, nullSink, 0);
    spec.addRoot(nullSink);
    return Pair.of(spec, adapterFactory);
}

Example 8 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class FeedMetadataUtil method getPrimaryFeedFactoryAndOutput.

@SuppressWarnings("rawtypes")
public static Triple<IAdapterFactory, RecordDescriptor, AdapterType> getPrimaryFeedFactoryAndOutput(Feed feed, FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx) throws AlgebricksException {
    // This method needs to be re-visited
    String adapterName = null;
    DatasourceAdapter adapterEntity = null;
    String adapterFactoryClassname = null;
    IAdapterFactory adapterFactory = null;
    ARecordType adapterOutputType = null;
    ARecordType metaType = null;
    Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> feedProps = null;
    IDataSourceAdapter.AdapterType adapterType = null;
    try {
        adapterName = feed.getAdapterName();
        Map<String, String> configuration = feed.getAdapterConfiguration();
        configuration.putAll(policyAccessor.getFeedPolicy());
        adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
        metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
        ExternalDataUtils.prepareFeed(configuration, feed.getDataverseName(), feed.getFeedName());
        // Get adapter from metadata dataset <Metadata dataverse>
        adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, adapterName);
        // Get adapter from metadata dataset <The feed dataverse>
        if (adapterEntity == null) {
            adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, feed.getDataverseName(), adapterName);
        }
        if (adapterEntity != null) {
            adapterType = adapterEntity.getType();
            adapterFactoryClassname = adapterEntity.getClassname();
            switch(adapterType) {
                case INTERNAL:
                    adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
                    break;
                case EXTERNAL:
                    String[] anameComponents = adapterName.split("#");
                    String libraryName = anameComponents[0];
                    ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
                    adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
                    break;
                default:
                    throw new AsterixException("Unknown Adapter type " + adapterType);
            }
            adapterFactory.setOutputType(adapterOutputType);
            adapterFactory.setMetaType(metaType);
            adapterFactory.configure(appCtx.getServiceContext(), configuration);
        } else {
            adapterFactory = AdapterFactoryProvider.getAdapterFactory(appCtx.getServiceContext(), adapterName, configuration, adapterOutputType, metaType);
            adapterType = IDataSourceAdapter.AdapterType.INTERNAL;
        }
        if (metaType == null) {
            metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
        }
        if (adapterOutputType == null) {
            if (!configuration.containsKey(ExternalDataConstants.KEY_TYPE_NAME)) {
                throw new AsterixException("Unspecified feed output data type");
            }
            adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
        }
        int numOfOutputs = 1;
        if (metaType != null) {
            numOfOutputs++;
        }
        if (ExternalDataUtils.isChangeFeed(configuration)) {
            // get number of PKs
            numOfOutputs += ExternalDataUtils.getNumberOfKeys(configuration);
        }
        ISerializerDeserializer[] serdes = new ISerializerDeserializer[numOfOutputs];
        int i = 0;
        serdes[i++] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(adapterOutputType);
        if (metaType != null) {
            serdes[i++] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType);
        }
        if (ExternalDataUtils.isChangeFeed(configuration)) {
            getSerdesForPKs(serdes, configuration, metaType, adapterOutputType, i);
        }
        feedProps = new Triple<>(adapterFactory, new RecordDescriptor(serdes), adapterType);
    } catch (Exception e) {
        throw new AlgebricksException("unable to create adapter", e);
    }
    return feedProps;
}

Example 9 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class ExternalIndexingOperations method getIndexingOperator.

/**
     * This method create an indexing operator that index records in HDFS
     *
     * @param jobSpec
     * @param itemType
     * @param dataset
     * @param files
     * @param indexerDesc
     * @return
     * @throws AlgebricksException
     * @throws HyracksDataException
     * @throws Exception
     */
private static Pair<ExternalScanOperatorDescriptor, AlgebricksPartitionConstraint> getIndexingOperator(MetadataProvider metadataProvider, JobSpecification jobSpec, IAType itemType, Dataset dataset, List<ExternalFile> files, RecordDescriptor indexerDesc) throws HyracksDataException, AlgebricksException {
    ExternalDatasetDetails externalDatasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
    Map<String, String> configuration = externalDatasetDetails.getProperties();
    IAdapterFactory adapterFactory = AdapterFactoryProvider.getIndexingAdapterFactory(metadataProvider.getApplicationContext().getServiceContext(), externalDatasetDetails.getAdapter(), configuration, (ARecordType) itemType, files, true, null);
    return new Pair<>(new ExternalScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory), adapterFactory.getPartitionConstraint());
}

Example 10 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class MetadataProvider method buildFeedIntakeRuntime.

public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime(JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception {
    Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> factoryOutput;
    factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx, getApplicationContext());
    ARecordType recordType = FeedMetadataUtil.getOutputType(primaryFeed, primaryFeed.getAdapterConfiguration(), ExternalDataConstants.KEY_TYPE_NAME);
    IAdapterFactory adapterFactory = factoryOutput.first;
    FeedIntakeOperatorDescriptor feedIngestor = null;
    switch(factoryOutput.third) {
        case INTERNAL:
            feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, primaryFeed, adapterFactory, recordType, policyAccessor, factoryOutput.second);
            break;
        case EXTERNAL:
            String libraryName = primaryFeed.getAdapterName().trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[0];
            feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, primaryFeed, libraryName, adapterFactory.getClass().getName(), recordType, policyAccessor, factoryOutput.second);
            break;
        default:
            break;
    }
    AlgebricksPartitionConstraint partitionConstraint = adapterFactory.getPartitionConstraint();
    return new Triple<>(feedIngestor, partitionConstraint, adapterFactory);
}