Examples with AlgebricksPartitionConstraint org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint used on opensource projects

Example 31 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class SinkWritePOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    WriteOperator write = (WriteOperator) op;
    int[] columns = new int[write.getExpressions().size()];
    int i = 0;
    for (Mutable<ILogicalExpression> exprRef : write.getExpressions()) {
        ILogicalExpression expr = exprRef.getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
            throw new NotImplementedException("Only writing variable expressions is supported.");
        }
        VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
        LogicalVariable v = varRef.getVariableReference();
        columns[i++] = inputSchemas[0].findVariable(v);
    }
    RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op), context, columns);
    IMetadataProvider<?, ?> mp = context.getMetadataProvider();
    Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> runtime = mp.getWriteFileRuntime(write.getDataSink(), columns, pf, inputDesc);
    builder.contributeMicroOperator(write, runtime.first, recDesc, runtime.second);
    ILogicalOperator src = write.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, write, 0);
}

Example 32 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class PigletMetadataProvider method getWriteFileRuntime.

@Override
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink, int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) throws AlgebricksException {
    PigletFileDataSink ds = (PigletFileDataSink) sink;
    FileSplit[] fileSplits = ds.getFileSplits();
    String[] locations = new String[fileSplits.length];
    for (int i = 0; i < fileSplits.length; ++i) {
        locations[i] = fileSplits[i].getNodeName();
    }
    IPushRuntimeFactory prf;
    try {
        prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0].getFile(null), PrinterBasedWriterFactory.INSTANCE, inputDesc);
        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
        return new Pair<>(prf, constraint);
    } catch (HyracksDataException e) {
        throw new AlgebricksException(e);
    }
}

Example 33 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class JobBuilder method setPartitionConstraintsBottomup.

private void setPartitionConstraintsBottomup(OperatorDescriptorId opId, Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp, boolean finalPass) {
    List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
    AlgebricksPartitionConstraint opConstraint = null;
    IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
    if (opInputs != null) {
        for (IConnectorDescriptor conn : opInputs) {
            ConnectorDescriptorId cid = conn.getConnectorId();
            org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec.getConnectorOperatorMap().get(cid);
            IOperatorDescriptor src = p.getLeft().getLeft();
            // Pre-order DFS
            setPartitionConstraintsBottomup(src.getOperatorId(), tgtConstraints, opDesc, finalPass);
            TargetConstraint constraint = tgtConstraints.get(conn);
            if (constraint != null) {
                switch(constraint) {
                    case ONE:
                        opConstraint = countOneLocation;
                        break;
                    case SAME_COUNT:
                        opConstraint = partitionConstraintMap.get(src);
                        break;
                }
            }
        }
    }
    if (partitionConstraintMap.get(opDesc) == null) {
        if (finalPass && opConstraint == null && (opInputs == null || opInputs.isEmpty())) {
            opConstraint = countOneLocation;
        }
        if (finalPass && opConstraint == null) {
            opConstraint = clusterLocations;
        }
        // Sets up the location constraint.
        if (opConstraint != null) {
            partitionConstraintMap.put(opDesc, opConstraint);
            AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, opConstraint);
        }
    }
}

Example 34 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class JobBuilder method contributeAlgebricksPartitionConstraint.

@Override
public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apcArg) {
    AlgebricksPartitionConstraint apc = apcArg;
    if (apc.getPartitionConstraintType() == PartitionConstraintType.COUNT) {
        AlgebricksCountPartitionConstraint constraint = (AlgebricksCountPartitionConstraint) apc;
        if (constraint.getCount() == 1) {
            apc = countOneLocation;
        }
    }
    partitionConstraintMap.put(opDesc, apc);
}

Example 35 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class StoragePathUtil method splitProviderAndPartitionConstraints.

public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints(FileSplit[] splits) {
    IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
    String[] loc = new String[splits.length];
    for (int p = 0; p < splits.length; p++) {
        loc[p] = splits[p].getNodeName();
    }
    AlgebricksPartitionConstraint pc = new AlgebricksAbsolutePartitionConstraint(loc);
    return new Pair<>(splitProvider, pc);
}