Search in sources :

Example 6 with IResourceFactory

use of org.apache.hyracks.storage.common.IResourceFactory in project asterixdb by apache.

the class ExternalIndexingOperations method buildFilesIndexCreateJobSpec.

public static JobSpecification buildFilesIndexCreateJobSpec(Dataset dataset, List<ExternalFile> externalFilesSnapshot, MetadataProvider metadataProvider) throws AlgebricksException {
    IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
    ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
    Map<String, String> mergePolicyProperties = compactionInfo.second;
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, IndexingConstants.getFilesIndexName(dataset.getDatasetName()));
    IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
    String fileIndexName = IndexingConstants.getFilesIndexName(dataset.getDatasetName());
    Index fileIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(), dataset.getDatasetName(), fileIndexName);
    ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
    IResourceFactory resourceFactory = dataset.getResourceFactory(metadataProvider, fileIndex, recordType, null, mergePolicyFactory, mergePolicyProperties);
    IIndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), secondaryFileSplitProvider, resourceFactory, !dataset.isTemp());
    IIndexDataflowHelperFactory dataflowHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), secondaryFileSplitProvider);
    ExternalFilesIndexCreateOperatorDescriptor externalFilesOp = new ExternalFilesIndexCreateOperatorDescriptor(spec, indexBuilderFactory, dataflowHelperFactory, externalFilesSnapshot);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, externalFilesOp, secondarySplitsAndConstraint.second);
    spec.addRoot(externalFilesOp);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
}
Also used : IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) ConnectorPolicyAssignmentPolicy(org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy) IFileSplitProvider(org.apache.hyracks.dataflow.std.file.IFileSplitProvider) IIndexBuilderFactory(org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory) IndexBuilderFactory(org.apache.hyracks.storage.am.common.build.IndexBuilderFactory) Index(org.apache.asterix.metadata.entities.Index) ExternalFilesIndexCreateOperatorDescriptor(org.apache.asterix.external.operators.ExternalFilesIndexCreateOperatorDescriptor) IIndexBuilderFactory(org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory) IIndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint) JobSpecification(org.apache.hyracks.api.job.JobSpecification) Map(java.util.Map) ARecordType(org.apache.asterix.om.types.ARecordType) IResourceFactory(org.apache.hyracks.storage.common.IResourceFactory) IIndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory) IndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory) ILSMMergePolicyFactory(org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory)

Example 7 with IResourceFactory

use of org.apache.hyracks.storage.common.IResourceFactory in project asterixdb by apache.

the class Dataset method getResourceFactory.

/**
     * Create the index dataflow helper factory for a particular index on the dataset
     *
     * @param mdProvider
     *            metadata provider to get metadata information, components, and runtimes
     * @param index
     *            the index to get the dataflow helper factory for
     * @param recordType
     *            the record type for the dataset
     * @param metaType
     *            the meta type for the dataset
     * @param mergePolicyFactory
     *            the merge policy factory of the dataset
     * @param mergePolicyProperties
     *            the merge policy properties for the dataset
     * @return indexDataflowHelperFactory
     *         an instance of {@link org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory}
     * @throws AlgebricksException
     *             if dataflow helper factory could not be created
     */
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties) throws AlgebricksException {
    ITypeTraits[] filterTypeTraits = DatasetUtil.computeFilterTypeTraits(this, recordType);
    IBinaryComparatorFactory[] filterCmpFactories = DatasetUtil.computeFilterBinaryComparatorFactories(this, recordType, mdProvider.getStorageComponentProvider().getComparatorFactoryProvider());
    IResourceFactory resourceFactory;
    switch(index.getIndexType()) {
        case BTREE:
            resourceFactory = bTreeResourceFactoryProvider.getResourceFactory(mdProvider, this, index, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories);
            break;
        case RTREE:
            resourceFactory = rTreeResourceFactoryProvider.getResourceFactory(mdProvider, this, index, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories);
            break;
        case LENGTH_PARTITIONED_NGRAM_INVIX:
        case LENGTH_PARTITIONED_WORD_INVIX:
        case SINGLE_PARTITION_NGRAM_INVIX:
        case SINGLE_PARTITION_WORD_INVIX:
            resourceFactory = invertedIndexResourceFactoryProvider.getResourceFactory(mdProvider, this, index, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories);
            break;
        default:
            throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType().toString());
    }
    return new DatasetLocalResourceFactory(datasetId, resourceFactory);
}
Also used : CompilationException(org.apache.asterix.common.exceptions.CompilationException) DatasetLocalResourceFactory(org.apache.asterix.transaction.management.resource.DatasetLocalResourceFactory) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) IResourceFactory(org.apache.hyracks.storage.common.IResourceFactory)

Aggregations

IResourceFactory (org.apache.hyracks.storage.common.IResourceFactory)7 IndexBuilderFactory (org.apache.hyracks.storage.am.common.build.IndexBuilderFactory)6 JobSpecification (org.apache.hyracks.api.job.JobSpecification)5 IIndexBuilderFactory (org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory)4 IndexCreateOperatorDescriptor (org.apache.hyracks.storage.am.common.dataflow.IndexCreateOperatorDescriptor)4 Map (java.util.Map)2 IStorageComponentProvider (org.apache.asterix.common.context.IStorageComponentProvider)2 Index (org.apache.asterix.metadata.entities.Index)2 ARecordType (org.apache.asterix.om.types.ARecordType)2 AlgebricksPartitionConstraint (org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint)2 ConnectorPolicyAssignmentPolicy (org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy)2 IFileSplitProvider (org.apache.hyracks.dataflow.std.file.IFileSplitProvider)2 ILSMMergePolicyFactory (org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory)2 CompilationException (org.apache.asterix.common.exceptions.CompilationException)1 ExternalFilesIndexCreateOperatorDescriptor (org.apache.asterix.external.operators.ExternalFilesIndexCreateOperatorDescriptor)1 MetadataProvider (org.apache.asterix.metadata.declared.MetadataProvider)1 Dataverse (org.apache.asterix.metadata.entities.Dataverse)1 NonTaggedDataFormat (org.apache.asterix.runtime.formats.NonTaggedDataFormat)1 DatasetLocalResourceFactory (org.apache.asterix.transaction.management.resource.DatasetLocalResourceFactory)1 IHyracksTaskContext (org.apache.hyracks.api.context.IHyracksTaskContext)1