Examples with IAdapterFactory org.apache.asterix.external.api.IAdapterFactory used on opensource projects

Example 1 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class FeedOperations method buildStartFeedJob.

public static Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> buildStartFeedJob(SessionOutput sessionOutput, MetadataProvider metadataProvider, Feed feed, List<FeedConnection> feedConnections, ILangCompilationProvider compilationProvider, IStorageComponentProvider storageComponentProvider, DefaultStatementExecutorFactory qtFactory, IHyracksClientConnection hcc) throws Exception {
    FeedPolicyAccessor fpa = new FeedPolicyAccessor(new HashMap<>());
    // TODO: Change the default Datasource to use all possible partitions
    Pair<JobSpecification, IAdapterFactory> intakeInfo = buildFeedIntakeJobSpec(feed, metadataProvider, fpa);
    //TODO: Add feed policy accessor
    List<JobSpecification> jobsList = new ArrayList<>();
    // Construct the ingestion Job
    JobSpecification intakeJob = intakeInfo.getLeft();
    IAdapterFactory ingestionAdaptorFactory = intakeInfo.getRight();
    String[] ingestionLocations = ingestionAdaptorFactory.getPartitionConstraint().getLocations();
    // Add connection job
    for (FeedConnection feedConnection : feedConnections) {
        JobSpecification connectionJob = getConnectionJob(sessionOutput, metadataProvider, feedConnection, ingestionLocations, compilationProvider, storageComponentProvider, qtFactory, hcc);
        jobsList.add(connectionJob);
    }
    return Pair.of(combineIntakeCollectJobs(metadataProvider, feed, intakeJob, jobsList, feedConnections, ingestionLocations), intakeInfo.getRight().getPartitionConstraint());
}

Example 2 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class LoadableDataSource method buildDatasourceScanRuntime.

@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
    LoadableDataSource alds = (LoadableDataSource) dataSource;
    ARecordType itemType = (ARecordType) alds.getLoadedType();
    IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(), alds.getAdapter(), alds.getAdapterProperties(), itemType, null);
    RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
    return metadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc);
}

Example 3 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class MetadataProvider method getConfiguredAdapterFactory.

protected IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName, Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException {
    try {
        configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName());
        IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType);
        // check to see if dataset is indexed
        Index filesIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX));
        if (filesIndex != null && filesIndex.getPendingOp() == 0) {
            // get files
            List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset);
            Iterator<ExternalFile> iterator = files.iterator();
            while (iterator.hasNext()) {
                if (iterator.next().getPendingOp() != ExternalFilePendingOp.NO_OP) {
                    iterator.remove();
                }
            }
        }
        return adapterFactory;
    } catch (Exception e) {
        throw new AlgebricksException("Unable to create adapter", e);
    }
}

Example 4 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class DatasetDataSource method buildDatasourceScanRuntime.

@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
    switch(dataset.getDatasetType()) {
        case EXTERNAL:
            Dataset externalDataset = ((DatasetDataSource) dataSource).getDataset();
            String itemTypeName = externalDataset.getItemTypeName();
            IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
            ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
            IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
            return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory, NonTaggedDataFormat.INSTANCE);
        case INTERNAL:
            DataSourceId id = getId();
            String dataverseName = id.getDataverseName();
            String datasetName = id.getDatasourceName();
            Index primaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName, datasetName);
            int[] minFilterFieldIndexes = createFilterIndexes(minFilterVars, opSchema);
            int[] maxFilterFieldIndexes = createFilterIndexes(maxFilterVars, opSchema);
            return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, ((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null, true, true, minFilterFieldIndexes, maxFilterFieldIndexes);
        default:
            throw new AlgebricksException("Unknown datasource type");
    }
}

Example 5 with IAdapterFactory

use of org.apache.asterix.external.api.IAdapterFactory in project asterixdb by apache.

the class FeedMetadataUtil method validateFeed.

public static void validateFeed(Feed feed, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx) throws AsterixException {
    try {
        String adapterName = feed.getAdapterName();
        Map<String, String> configuration = feed.getAdapterConfiguration();
        ARecordType adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
        ARecordType metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
        ExternalDataUtils.prepareFeed(configuration, feed.getDataverseName(), feed.getFeedName());
        // Get adapter from metadata dataset <Metadata dataverse>
        DatasourceAdapter adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, adapterName);
        // Get adapter from metadata dataset <The feed dataverse>
        if (adapterEntity == null) {
            adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, feed.getDataverseName(), adapterName);
        }
        AdapterType adapterType;
        IAdapterFactory adapterFactory;
        if (adapterEntity != null) {
            adapterType = adapterEntity.getType();
            String adapterFactoryClassname = adapterEntity.getClassname();
            switch(adapterType) {
                case INTERNAL:
                    adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
                    break;
                case EXTERNAL:
                    String[] anameComponents = adapterName.split("#");
                    String libraryName = anameComponents[0];
                    ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
                    adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
                    break;
                default:
                    throw new AsterixException("Unknown Adapter type " + adapterType);
            }
            adapterFactory.setOutputType(adapterOutputType);
            adapterFactory.setMetaType(metaType);
            adapterFactory.configure(appCtx.getServiceContext(), configuration);
        } else {
            AdapterFactoryProvider.getAdapterFactory(appCtx.getServiceContext(), adapterName, configuration, adapterOutputType, metaType);
        }
        if (metaType == null && configuration.containsKey(ExternalDataConstants.KEY_META_TYPE_NAME)) {
            metaType = getOutputType(feed, configuration, ExternalDataConstants.KEY_META_TYPE_NAME);
            if (metaType == null) {
                throw new AsterixException("Unknown specified feed meta output data type " + configuration.get(ExternalDataConstants.KEY_META_TYPE_NAME));
            }
        }
        if (adapterOutputType == null) {
            if (!configuration.containsKey(ExternalDataConstants.KEY_TYPE_NAME)) {
                throw new AsterixException("Unspecified feed output data type");
            }
            adapterOutputType = getOutputType(feed, configuration, ExternalDataConstants.KEY_TYPE_NAME);
            if (adapterOutputType == null) {
                throw new AsterixException("Unknown specified feed output data type " + configuration.get(ExternalDataConstants.KEY_TYPE_NAME));
            }
        }
    } catch (Exception e) {
        throw new AsterixException("Invalid feed parameters. Exception Message:" + e.getMessage(), e);
    }
}