Examples with OperatorDescriptorId org.apache.hyracks.api.dataflow.OperatorDescriptorId used on opensource projects

Example 1 with OperatorDescriptorId

use of org.apache.hyracks.api.dataflow.OperatorDescriptorId in project asterixdb by apache.

the class TestNodeController method getInsertPipeline.

public Pair<LSMInsertDeleteOperatorNodePushable, CommitRuntime> getInsertPipeline(IHyracksTaskContext ctx, Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields, int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, StorageComponentProvider storageComponentProvider) throws AlgebricksException, HyracksDataException {
    PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, storageComponentProvider);
    IndexOperation op = IndexOperation.INSERT;
    IModificationOperationCallbackFactory modOpCallbackFactory = new PrimaryIndexModificationOperationCallbackFactory(getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, Operation.get(op), ResourceType.LSM_BTREE);
    IRecordDescriptorProvider recordDescProvider = primaryIndexInfo.getInsertRecordDescriptorProvider();
    IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), primaryIndexInfo.fileSplitProvider);
    LSMInsertDeleteOperatorNodePushable insertOp = new LSMInsertDeleteOperatorNodePushable(ctx, PARTITION, primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider.getInputRecordDescriptor(new ActivityId(new OperatorDescriptorId(0), 0), 0), op, true, indexHelperFactory, modOpCallbackFactory, null);
    CommitRuntime commitOp = new CommitRuntime(ctx, getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, false, true, PARTITION, true);
    insertOp.setOutputFrameWriter(0, commitOp, primaryIndexInfo.rDesc);
    commitOp.setInputRecordDescriptor(0, primaryIndexInfo.rDesc);
    return Pair.of(insertOp, commitOp);
}

Example 2 with OperatorDescriptorId

use of org.apache.hyracks.api.dataflow.OperatorDescriptorId in project asterixdb by apache.

the class AbstractPhysicalOperator method buildPipelineWithProjection.

private AlgebricksPipeline buildPipelineWithProjection(ILogicalPlan p, IOperatorSchema outerPlanSchema, AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, PlanCompiler pc) throws AlgebricksException {
    if (p.getRoots().size() > 1) {
        throw new NotImplementedException("Nested plans with several roots are not supported.");
    }
    JobSpecification nestedJob = pc.compilePlan(p, outerPlanSchema, null);
    ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue();
    JobGenContext context = pc.getContext();
    IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan);
    opSchema.addAllVariables(topOpInSubplanScm);
    Map<OperatorDescriptorId, IOperatorDescriptor> opMap = nestedJob.getOperatorMap();
    if (opMap.size() != 1) {
        throw new AlgebricksException("Attempting to construct a nested plan with " + opMap.size() + " operator descriptors. Currently, nested plans can only consist in linear pipelines of Asterix micro operators.");
    }
    for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> opEntry : opMap.entrySet()) {
        IOperatorDescriptor opd = opEntry.getValue();
        if (!(opd instanceof AlgebricksMetaOperatorDescriptor)) {
            throw new AlgebricksException("Can only generate Hyracks jobs for pipelinable Asterix nested plans, not for " + opd.getClass().getName());
        }
        AlgebricksMetaOperatorDescriptor amod = (AlgebricksMetaOperatorDescriptor) opd;
        return amod.getPipeline();
    // we suppose that the top operator in the subplan already does the
    // projection for us
    }
    throw new IllegalStateException();
}

Example 3 with OperatorDescriptorId

use of org.apache.hyracks.api.dataflow.OperatorDescriptorId in project asterixdb by apache.

the class JobSpecification method createOperatorDescriptorId.

@Override
public OperatorDescriptorId createOperatorDescriptorId(IOperatorDescriptor op) {
    OperatorDescriptorId odId = new OperatorDescriptorId(operatorIdCounter++);
    op.setOperatorId(odId);
    opMap.put(odId, op);
    return odId;
}

Example 4 with OperatorDescriptorId

use of org.apache.hyracks.api.dataflow.OperatorDescriptorId in project asterixdb by apache.

the class CCNCFunctions method writeTaskAttemptId.

private static void writeTaskAttemptId(DataOutputStream dos, TaskAttemptId taId) throws IOException {
    TaskId tid = taId.getTaskId();
    ActivityId aid = tid.getActivityId();
    OperatorDescriptorId odId = aid.getOperatorDescriptorId();
    dos.writeInt(odId.getId());
    dos.writeInt(aid.getLocalId());
    dos.writeInt(tid.getPartition());
    dos.writeInt(taId.getAttempt());
}

Example 5 with OperatorDescriptorId

use of org.apache.hyracks.api.dataflow.OperatorDescriptorId in project asterixdb by apache.

the class CCNCFunctions method readTaskAttemptId.

private static TaskAttemptId readTaskAttemptId(DataInputStream dis) throws IOException {
    int odid = dis.readInt();
    int aid = dis.readInt();
    int partition = dis.readInt();
    int attempt = dis.readInt();
    TaskAttemptId taId = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(odid), aid), partition), attempt);
    return taId;
}