Examples with OrderColumn org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn used on opensource projects

Example 11 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class StableSortPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
    int n = sortColumns.length;
    int[] sortFields = new int[n];
    IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
    INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
    INormalizedKeyComputerFactory nkcf = null;
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    int i = 0;
    for (OrderColumn oc : sortColumns) {
        LogicalVariable var = oc.getColumn();
        sortFields[i] = opSchema.findVariable(var);
        Object type = env.getVarType(var);
        OrderKind order = oc.getOrder();
        if (i == 0 && nkcfProvider != null && type != null) {
            nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
        }
        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
        comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
        i++;
    }
    // topK == -1 means that a topK value is not provided.
    if (topK == -1) {
        ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames, sortFields, nkcf, comps, recDescriptor);
        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
        ILogicalOperator src = op.getInputs().get(0).getValue();
        builder.contributeGraphEdge(src, 0, op, 0);
    } else {
        // Since topK value is provided, topK optimization is possible.
        // We call topKSorter instead of calling ExternalSortOperator.
        TopKSorterOperatorDescriptor sortOpDesc = new TopKSorterOperatorDescriptor(spec, maxNumberOfFrames, topK, sortFields, nkcf, comps, recDescriptor);
        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
        ILogicalOperator src = op.getInputs().get(0).getValue();
        builder.contributeGraphEdge(src, 0, op, 0);
    }
}

Example 12 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class PreSortedDistinctByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>();
    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
    for (LogicalVariable column : columnList) {
        orderColumns.add(new OrderColumn(column, OrderKind.ASC));
    }
    localProps.add(new LocalOrderProperty(orderColumns));
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 13 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class SortGroupByPOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
    List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
    GroupByOperator gOp = (GroupByOperator) op;
    Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
    List<OrderColumn> ocs = new ArrayList<OrderColumn>();
    if (!columnSet.isEmpty()) {
        propsLocal.add(new LocalGroupingProperty(columnSet));
    }
    for (OrderColumn oc : orderColumns) {
        ocs.add(oc);
    }
    propsLocal.add(new LocalOrderProperty(ocs));
    for (ILogicalPlan p : gOp.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            ILogicalOperator rOp = r.getValue();
            propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
        }
    }
    ILogicalOperator op2 = op.getInputs().get(0).getValue();
    IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
    deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
}

Example 14 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class RemoveUnnecessarySortMergeExchange method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getPhysicalOperator() == null || (op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE && op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE)) {
        return false;
    }
    Mutable<ILogicalOperator> currentOpRef = op1.getInputs().get(0);
    AbstractLogicalOperator currentOp = (AbstractLogicalOperator) currentOpRef.getValue();
    // Goes down the pipeline to find a qualified SortMergeExchange to eliminate.
    while (currentOp != null) {
        IPhysicalOperator physicalOp = currentOp.getPhysicalOperator();
        if (physicalOp == null) {
            return false;
        } else if (physicalOp.getOperatorTag() == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
            break;
        } else if (!currentOp.isMap() || currentOp.getOperatorTag() == LogicalOperatorTag.UNNEST || currentOp.getOperatorTag() == LogicalOperatorTag.LIMIT) {
            // we need to use his new property in logical operator to check order sensitivity.
            return false;
        } else if (currentOp.getInputs().size() == 1) {
            currentOpRef = currentOp.getInputs().get(0);
            currentOp = (AbstractLogicalOperator) currentOpRef.getValue();
        } else {
            currentOp = null;
        }
    }
    if (currentOp == null) {
        // There is no such qualified SortMergeExchange.
        return false;
    }
    if (op1.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE) {
        // If op1 is a hash_partition_merge_exchange, the sort_merge_exchange can be simply removed.
        currentOpRef.setValue(currentOp.getInputs().get(0).getValue());
        op1.computeDeliveredPhysicalProperties(context);
        return true;
    }
    // Checks whether sort columns in the SortMergeExchange are still available at op1.
    // If yes, we use HashMergeExchange; otherwise, we use HashExchange.
    SortMergeExchangePOperator sme = (SortMergeExchangePOperator) currentOp.getPhysicalOperator();
    HashPartitionExchangePOperator hpe = (HashPartitionExchangePOperator) op1.getPhysicalOperator();
    Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(op1, liveVars);
    boolean usingHashMergeExchange = true;
    for (OrderColumn oc : sme.getSortColumns()) {
        if (!liveVars.contains(oc.getColumn())) {
            usingHashMergeExchange = false;
        }
    }
    if (usingHashMergeExchange) {
        // Add sort columns from the SortMergeExchange into a new HashMergeExchange.
        List<OrderColumn> ocList = new ArrayList<OrderColumn>();
        for (OrderColumn oc : sme.getSortColumns()) {
            ocList.add(oc);
        }
        HashPartitionMergeExchangePOperator hpme = new HashPartitionMergeExchangePOperator(ocList, hpe.getHashFields(), hpe.getDomain());
        op1.setPhysicalOperator(hpme);
    }
    // Remove the SortMergeExchange op.
    currentOpRef.setValue(currentOp.getInputs().get(0).getValue());
    // Re-compute delivered properties at op1.
    op1.computeDeliveredPhysicalProperties(context);
    return true;
}

Example 15 with OrderColumn

use of org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method addLocalEnforcers.

private void addLocalEnforcers(AbstractLogicalOperator op, int i, List<ILocalStructuralProperty> localProperties, boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
    if (AlgebricksConfig.DEBUG) {
        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Adding local enforcers for local props = " + localProperties + "\n");
    }
    if (localProperties == null || localProperties.isEmpty()) {
        return;
    }
    Mutable<ILogicalOperator> topOp = new MutableObject<>();
    topOp.setValue(op.getInputs().get(i).getValue());
    LinkedList<LocalOrderProperty> oList = new LinkedList<>();
    for (ILocalStructuralProperty prop : localProperties) {
        switch(prop.getPropertyType()) {
            case LOCAL_ORDER_PROPERTY:
                {
                    oList.add((LocalOrderProperty) prop);
                    break;
                }
            case LOCAL_GROUPING_PROPERTY:
                {
                    LocalGroupingProperty g = (LocalGroupingProperty) prop;
                    Collection<LogicalVariable> vars = (g.getPreferredOrderEnforcer() != null) ? g.getPreferredOrderEnforcer() : g.getColumnSet();
                    List<OrderColumn> orderColumns = new ArrayList<>();
                    for (LogicalVariable v : vars) {
                        OrderColumn oc = new OrderColumn(v, OrderKind.ASC);
                        orderColumns.add(oc);
                    }
                    LocalOrderProperty lop = new LocalOrderProperty(orderColumns);
                    oList.add(lop);
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
    }
    if (!oList.isEmpty()) {
        topOp = enforceOrderProperties(oList, topOp, nestedPlan, context);
    }
    op.getInputs().set(i, topOp);
    OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getValue(), context);
    OperatorManipulationUtil.setOperatorMode(op);
    printOp((AbstractLogicalOperator) topOp.getValue());
}