Search in sources :

Example 6 with INodeDomain

use of org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method addPartitioningEnforcers.

private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp, IPhysicalPropertiesVector required, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, IOptimizationContext context) throws AlgebricksException {
    if (pp != null) {
        IPhysicalOperator pop;
        switch(pp.getPartitioningType()) {
            case UNPARTITIONED:
                {
                    List<OrderColumn> ordCols = computeOrderColumns(deliveredByChild);
                    if (ordCols.isEmpty()) {
                        pop = new RandomMergeExchangePOperator();
                    } else {
                        if (op.getAnnotations().containsKey(OperatorAnnotations.USE_RANGE_CONNECTOR)) {
                            IRangeMap rangeMap = (IRangeMap) op.getAnnotations().get(OperatorAnnotations.USE_RANGE_CONNECTOR);
                            pop = new RangePartitionMergeExchangePOperator(ordCols, domain, rangeMap);
                        } else {
                            OrderColumn[] sortColumns = new OrderColumn[ordCols.size()];
                            sortColumns = ordCols.toArray(sortColumns);
                            pop = new SortMergeExchangePOperator(sortColumns);
                        }
                    }
                    break;
                }
            case UNORDERED_PARTITIONED:
                {
                    List<LogicalVariable> varList = new ArrayList<>(((UnorderedPartitionedProperty) pp).getColumnSet());
                    String hashMergeHint = context.getMetadataProvider().getConfig().get(HASH_MERGE);
                    if (hashMergeHint == null || !hashMergeHint.equalsIgnoreCase(TRUE_CONSTANT)) {
                        pop = new HashPartitionExchangePOperator(varList, domain);
                        break;
                    }
                    List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
                    List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
                    boolean propWasSet = false;
                    pop = null;
                    if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
                        Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
                        List<FunctionalDependency> fds = context.getFDList(c);
                        if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
                            List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals, cldLocals);
                            pop = new HashPartitionMergeExchangePOperator(orderColumns, varList, domain);
                            propWasSet = true;
                        }
                    }
                    if (!propWasSet) {
                        pop = new HashPartitionExchangePOperator(varList, domain);
                    }
                    break;
                }
            case ORDERED_PARTITIONED:
                {
                    pop = new RangePartitionExchangePOperator(((OrderedPartitionedProperty) pp).getOrderColumns(), domain, null);
                    break;
                }
            case BROADCAST:
                {
                    pop = new BroadcastExchangePOperator(domain);
                    break;
                }
            case RANDOM:
                {
                    RandomPartitioningProperty rpp = (RandomPartitioningProperty) pp;
                    INodeDomain nd = rpp.getNodeDomain();
                    pop = new RandomPartitionExchangePOperator(nd);
                    break;
                }
            default:
                {
                    throw new NotImplementedException("Enforcer for " + pp.getPartitioningType() + " partitioning type has not been implemented.");
                }
        }
        Mutable<ILogicalOperator> ci = op.getInputs().get(i);
        ExchangeOperator exchg = new ExchangeOperator();
        exchg.setPhysicalOperator(pop);
        setNewOp(ci, exchg, context);
        exchg.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(exchg, context);
        context.computeAndSetTypeEnvironmentForOperator(exchg);
        if (AlgebricksConfig.DEBUG) {
            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added partitioning enforcer " + exchg.getPhysicalOperator() + ".\n");
            printOp((AbstractLogicalOperator) op);
        }
    }
}
Also used : UnorderedPartitionedProperty(org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty) AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) RandomPartitionExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.RandomPartitionExchangePOperator) NotImplementedException(org.apache.hyracks.algebricks.common.exceptions.NotImplementedException) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) BroadcastExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.BroadcastExchangePOperator) INodeDomain(org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain) ExchangeOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator) RandomMergeExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.RandomMergeExchangePOperator) HashPartitionMergeExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator) RangePartitionMergeExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.RangePartitionMergeExchangePOperator) HashPartitionExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator) IRangeMap(org.apache.hyracks.dataflow.common.data.partition.range.IRangeMap) IPhysicalOperator(org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SortMergeExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator) RangePartitionExchangePOperator(org.apache.hyracks.algebricks.core.algebra.operators.physical.RangePartitionExchangePOperator) RandomPartitioningProperty(org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty) IRangeMap(org.apache.hyracks.dataflow.common.data.partition.range.IRangeMap) Map(java.util.Map)

Example 7 with INodeDomain

use of org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain in project asterixdb by apache.

the class MetadataManagerUtil method lookupSourceInMetadata.

public static DataSource lookupSourceInMetadata(MetadataTransactionContext mdTxnCtx, DataSourceId aqlId) throws AlgebricksException {
    Dataset dataset = findDataset(mdTxnCtx, aqlId.getDataverseName(), aqlId.getDatasourceName());
    if (dataset == null) {
        throw new AlgebricksException("Datasource with id " + aqlId + " was not found.");
    }
    IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
    IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
    INodeDomain domain = findNodeDomain(mdTxnCtx, dataset.getNodeGroupName());
    byte datasourceType = dataset.getDatasetType().equals(DatasetType.EXTERNAL) ? DataSource.Type.EXTERNAL_DATASET : DataSource.Type.INTERNAL_DATASET;
    return new DatasetDataSource(aqlId, dataset, itemType, metaItemType, datasourceType, dataset.getDatasetDetails(), domain);
}
Also used : Dataset(org.apache.asterix.metadata.entities.Dataset) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) INodeDomain(org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain) IAType(org.apache.asterix.om.types.IAType)

Aggregations

INodeDomain (org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain)7 ILogicalOperator (org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)4 AbstractLogicalOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator)4 Dataset (org.apache.asterix.metadata.entities.Dataset)3 AlgebricksException (org.apache.hyracks.algebricks.common.exceptions.AlgebricksException)3 ArrayList (java.util.ArrayList)2 List (java.util.List)2 DataSourceId (org.apache.asterix.metadata.declared.DataSourceId)2 IAType (org.apache.asterix.om.types.IAType)2 Mutable (org.apache.commons.lang3.mutable.Mutable)2 NotImplementedException (org.apache.hyracks.algebricks.common.exceptions.NotImplementedException)2 ILogicalPlan (org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan)2 AbstractOperatorWithNestedPlans (org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans)2 ExchangeOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator)2 RandomPartitionExchangePOperator (org.apache.hyracks.algebricks.core.algebra.operators.physical.RandomPartitionExchangePOperator)2 PhysicalRequirements (org.apache.hyracks.algebricks.core.algebra.properties.PhysicalRequirements)2 StructuralPropertiesVector (org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector)2 LinkedList (java.util.LinkedList)1 Map (java.util.Map)1 BTreeSearchPOperator (org.apache.asterix.algebra.operators.physical.BTreeSearchPOperator)1