Examples with OrderColumn org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn used on opensource projects

Example 1 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class IntersectPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    IntersectOperator intersectOp = (IntersectOperator) iop;
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[intersectOp.getNumInput()];
    for (int i = 0; i < intersectOp.getNumInput(); i++) {
        List<ILocalStructuralProperty> localProps = new ArrayList<>();
        List<OrderColumn> orderColumns = new ArrayList<>();
        for (LogicalVariable column : intersectOp.getInputVariables(i)) {
            orderColumns.add(new OrderColumn(column, OrderOperator.IOrder.OrderKind.ASC));
        }
        localProps.add(new LocalOrderProperty(orderColumns));
        IPartitioningProperty pp = null;
        if (intersectOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
            Set<LogicalVariable> partitioningVariables = new HashSet<>(intersectOp.getInputVariables(i));
            pp = new UnorderedPartitionedProperty(partitioningVariables, null);
        }
        pv[i] = new StructuralPropertiesVector(pp, localProps);
    }
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 2 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class AbstractPreclusteredGroupByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    GroupByOperator gby = (GroupByOperator) op;
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    if (gby.isGroupAll() && gby.isGlobal()) {
        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
        } else {
            return emptyUnaryRequirements();
        }
    }
    List<ILocalStructuralProperty> localProps = new ArrayList<>();
    Set<LogicalVariable> gbvars = new ListSet<>(columnList);
    LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList));
    boolean goon = true;
    for (ILogicalPlan p : gby.getNestedPlans()) {
        // groupings
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
            if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                IPhysicalOperator pop2 = op2.getPhysicalOperator();
                if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
                    List<LogicalVariable> gbyColumns = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
                    List<LogicalVariable> sndOrder = new ArrayList<>();
                    sndOrder.addAll(gbyColumns);
                    Set<LogicalVariable> freeVars = new HashSet<>();
                    try {
                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, freeVars);
                    } catch (AlgebricksException e) {
                        throw new IllegalStateException(e);
                    }
                    // Only considers group key variables defined out-side the outer-most group-by operator.
                    sndOrder.retainAll(freeVars);
                    groupProp.getColumnSet().addAll(sndOrder);
                    groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
                    goon = false;
                    break;
                }
            }
        }
        if (!goon) {
            break;
        }
    }
    localProps.add(groupProp);
    if (reqdByParent != null) {
        // propagate parent requirements
        List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
        if (lpPar != null) {
            boolean allOk = true;
            List<ILocalStructuralProperty> props = new ArrayList<>(lpPar.size());
            for (ILocalStructuralProperty prop : lpPar) {
                if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
                    allOk = false;
                    break;
                }
                LocalOrderProperty lop = (LocalOrderProperty) prop;
                List<OrderColumn> orderColumns = new ArrayList<>();
                List<OrderColumn> ords = lop.getOrderColumns();
                for (OrderColumn ord : ords) {
                    Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
                    if (p == null) {
                        p = getDecorPairByRhsVar(gby, ord.getColumn());
                        if (p == null) {
                            allOk = false;
                            break;
                        }
                    }
                    ILogicalExpression e = p.second.getValue();
                    if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                        throw new IllegalStateException("Right hand side of group-by assignment should have been normalized to a variable reference.");
                    }
                    LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
                    orderColumns.add(new OrderColumn(v, ord.getOrder()));
                }
                props.add(new LocalOrderProperty(orderColumns));
            }
            List<FunctionalDependency> fdList = new ArrayList<>();
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
                List<LogicalVariable> hd = gby.getGbyVarList();
                List<LogicalVariable> tl = new ArrayList<>();
                tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
                fdList.add(new FunctionalDependency(hd, tl));
            }
            if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, new HashMap<>(), fdList)) {
                localProps = props;
            }
        }
    }
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 3 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateLoad.

@Override
public ILogicalPlan translateLoad(ICompiledDmlStatement stmt) throws AlgebricksException {
    CompiledLoadFromFileStatement clffs = (CompiledLoadFromFileStatement) stmt;
    Dataset dataset = metadataProvider.findDataset(clffs.getDataverseName(), clffs.getDatasetName());
    if (dataset == null) {
        // This would never happen since we check for this in AqlTranslator
        throw new AlgebricksException("Unable to load dataset " + clffs.getDatasetName() + " since it does not exist");
    }
    IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
    IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
    DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
    List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
    if (dataset.hasMetaPart()) {
        throw new AlgebricksException(dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records");
    }
    LoadableDataSource lds;
    try {
        lds = new LoadableDataSource(dataset, itemType, metaItemType, clffs.getAdapter(), clffs.getProperties());
    } catch (IOException e) {
        throw new AlgebricksException(e);
    }
    // etsOp is a dummy input operator used to keep the compiler happy. it
    // could be removed but would result in
    // the need to fix many rewrite rules that assume that datasourcescan
    // operators always have input.
    ILogicalOperator etsOp = new EmptyTupleSourceOperator();
    // Add a logical variable for the record.
    List<LogicalVariable> payloadVars = new ArrayList<>();
    payloadVars.add(context.newVar());
    // Create a scan operator and make the empty tuple source its input
    DataSourceScanOperator dssOp = new DataSourceScanOperator(payloadVars, lds);
    dssOp.getInputs().add(new MutableObject<>(etsOp));
    ILogicalExpression payloadExpr = new VariableReferenceExpression(payloadVars.get(0));
    Mutable<ILogicalExpression> payloadRef = new MutableObject<>(payloadExpr);
    // Creating the assign to extract the PK out of the record
    ArrayList<LogicalVariable> pkVars = new ArrayList<>();
    ArrayList<Mutable<ILogicalExpression>> pkExprs = new ArrayList<>();
    List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
    LogicalVariable payloadVar = payloadVars.get(0);
    for (List<String> keyFieldName : partitionKeys) {
        PlanTranslationUtil.prepareVarAndExpression(keyFieldName, payloadVar, pkVars, pkExprs, varRefsForLoading, context);
    }
    AssignOperator assign = new AssignOperator(pkVars, pkExprs);
    assign.getInputs().add(new MutableObject<>(dssOp));
    // If the input is pre-sorted, we set the ordering property explicitly in the assign
    if (clffs.alreadySorted()) {
        List<OrderColumn> orderColumns = new ArrayList<>();
        for (int i = 0; i < pkVars.size(); ++i) {
            orderColumns.add(new OrderColumn(pkVars.get(i), OrderKind.ASC));
        }
        assign.setExplicitOrderingProperty(new LocalOrderProperty(orderColumns));
    }
    List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
    List<LogicalVariable> additionalFilteringVars;
    List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
    List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
    AssignOperator additionalFilteringAssign = null;
    if (additionalFilteringField != null) {
        additionalFilteringVars = new ArrayList<>();
        additionalFilteringAssignExpressions = new ArrayList<>();
        additionalFilteringExpressions = new ArrayList<>();
        PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
        additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
    }
    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true);
    insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    if (additionalFilteringAssign != null) {
        additionalFilteringAssign.getInputs().add(new MutableObject<>(assign));
        insertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
    } else {
        insertOp.getInputs().add(new MutableObject<>(assign));
    }
    ILogicalOperator leafOperator = new SinkOperator();
    leafOperator.getInputs().add(new MutableObject<>(insertOp));
    return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
}

Example 4 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method computeOrderColumns.

private List<OrderColumn> computeOrderColumns(IPhysicalPropertiesVector pv) {
    List<OrderColumn> ordCols = new ArrayList<>();
    List<ILocalStructuralProperty> localProps = pv.getLocalProperties();
    if (localProps == null || localProps.isEmpty()) {
        return new ArrayList<>();
    } else {
        for (ILocalStructuralProperty p : localProps) {
            if (p.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
                LocalOrderProperty lop = (LocalOrderProperty) p;
                ordCols.addAll(lop.getOrderColumns());
            } else {
                return new ArrayList<>();
            }
        }
        return ordCols;
    }
}

Example 5 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class RangePartitionExchangePOperator method createConnectorDescriptor.

@Override
public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
    int n = partitioningFields.size();
    int[] sortFields = new int[n];
    IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
    INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
    INormalizedKeyComputerFactory nkcf = null;
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    int i = 0;
    for (OrderColumn oc : partitioningFields) {
        LogicalVariable var = oc.getColumn();
        sortFields[i] = opSchema.findVariable(var);
        Object type = env.getVarType(var);
        OrderKind order = oc.getOrder();
        if (i == 0 && nkcfProvider != null && type != null) {
            nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
        }
        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
        comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
        i++;
    }
    ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
    IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
    return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
}