Examples with ActivityId org.apache.hyracks.api.dataflow.ActivityId used on opensource projects

Example 36 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class JobExecutor method assignTaskLocations.

private void assignTaskLocations(TaskCluster tc, Map<String, List<TaskAttemptDescriptor>> taskAttemptMap) throws HyracksException {
    ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
    Task[] tasks = tc.getTasks();
    List<TaskClusterAttempt> tcAttempts = tc.getAttempts();
    int attempts = tcAttempts.size();
    TaskClusterAttempt tcAttempt = new TaskClusterAttempt(tc, attempts);
    Map<TaskId, TaskAttempt> taskAttempts = new HashMap<>();
    Map<TaskId, LValueConstraintExpression> locationMap = new HashMap<>();
    for (int i = 0; i < tasks.length; ++i) {
        Task ts = tasks[i];
        TaskId tid = ts.getTaskId();
        TaskAttempt taskAttempt = new TaskAttempt(tcAttempt, new TaskAttemptId(new TaskId(tid.getActivityId(), tid.getPartition()), attempts), ts);
        taskAttempt.setStatus(TaskAttempt.TaskStatus.INITIALIZED, null);
        locationMap.put(tid, new PartitionLocationExpression(tid.getActivityId().getOperatorDescriptorId(), tid.getPartition()));
        taskAttempts.put(tid, taskAttempt);
    }
    tcAttempt.setTaskAttempts(taskAttempts);
    solver.solve(locationMap.values());
    for (int i = 0; i < tasks.length; ++i) {
        Task ts = tasks[i];
        TaskId tid = ts.getTaskId();
        TaskAttempt taskAttempt = taskAttempts.get(tid);
        String nodeId = assignLocation(acg, locationMap, tid, taskAttempt);
        taskAttempt.setNodeId(nodeId);
        taskAttempt.setStatus(TaskAttempt.TaskStatus.RUNNING, null);
        taskAttempt.setStartTime(System.currentTimeMillis());
        List<TaskAttemptDescriptor> tads = taskAttemptMap.get(nodeId);
        if (tads == null) {
            tads = new ArrayList<>();
            taskAttemptMap.put(nodeId, tads);
        }
        OperatorDescriptorId opId = tid.getActivityId().getOperatorDescriptorId();
        jobRun.registerOperatorLocation(opId, tid.getPartition(), nodeId);
        ActivityPartitionDetails apd = ts.getActivityPlan().getActivityPartitionDetails();
        TaskAttemptDescriptor tad = new TaskAttemptDescriptor(taskAttempt.getTaskAttemptId(), apd.getPartitionCount(), apd.getInputPartitionCounts(), apd.getOutputPartitionCounts());
        tads.add(tad);
    }
    tcAttempt.initializePendingTaskCounter();
    tcAttempts.add(tcAttempt);
    /**
         * Improvement for reducing master/slave message communications, for each TaskAttemptDescriptor,
         * we set the NetworkAddress[][] partitionLocations, in which each row is for an incoming connector descriptor
         * and each column is for an input channel of the connector.
         */
    INodeManager nodeManager = ccs.getNodeManager();
    for (Map.Entry<String, List<TaskAttemptDescriptor>> e : taskAttemptMap.entrySet()) {
        List<TaskAttemptDescriptor> tads = e.getValue();
        for (TaskAttemptDescriptor tad : tads) {
            TaskAttemptId taid = tad.getTaskAttemptId();
            int attempt = taid.getAttempt();
            TaskId tid = taid.getTaskId();
            ActivityId aid = tid.getActivityId();
            List<IConnectorDescriptor> inConnectors = acg.getActivityInputs(aid);
            int[] inPartitionCounts = tad.getInputPartitionCounts();
            if (inPartitionCounts == null) {
                continue;
            }
            NetworkAddress[][] partitionLocations = new NetworkAddress[inPartitionCounts.length][];
            for (int i = 0; i < inPartitionCounts.length; ++i) {
                ConnectorDescriptorId cdId = inConnectors.get(i).getConnectorId();
                IConnectorPolicy policy = jobRun.getConnectorPolicyMap().get(cdId);
                /**
                     * carry sender location information into a task
                     * when it is not the case that it is an re-attempt and the send-side
                     * is materialized blocking.
                     */
                if (attempt > 0 && policy.materializeOnSendSide() && policy.consumerWaitsForProducerToFinish()) {
                    continue;
                }
                ActivityId producerAid = acg.getProducerActivity(cdId);
                partitionLocations[i] = new NetworkAddress[inPartitionCounts[i]];
                for (int j = 0; j < inPartitionCounts[i]; ++j) {
                    TaskId producerTaskId = new TaskId(producerAid, j);
                    String nodeId = findTaskLocation(producerTaskId);
                    partitionLocations[i][j] = nodeManager.getNodeControllerState(nodeId).getDataPort();
                }
            }
            tad.setInputPartitionLocations(partitionLocations);
        }
    }
    tcAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.RUNNING);
    tcAttempt.setStartTime(System.currentTimeMillis());
    inProgressTaskClusters.add(tc);
}

Example 37 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ExternalGroupOperatorDescriptor method contributeActivities.

/*
     * (non-Javadoc)
     *
     * @see
     * org.apache.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
     * (org.apache.hyracks.api.dataflow.IActivityGraphBuilder)
     */
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
    AggregateActivity aggregateAct = new AggregateActivity(new ActivityId(getOperatorId(), AGGREGATE_ACTIVITY_ID));
    MergeActivity mergeAct = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
    builder.addActivity(this, aggregateAct);
    builder.addSourceEdge(0, aggregateAct, 0);
    builder.addActivity(this, mergeAct);
    builder.addTargetEdge(0, mergeAct, 0);
    builder.addBlockingEdge(aggregateAct, mergeAct);
}

Example 38 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class SplitVectorOperatorDescriptor method contributeActivities.

@Override
public void contributeActivities(IActivityGraphBuilder builder) {
    CollectActivity ca = new CollectActivity(new ActivityId(odId, COLLECT_ACTIVITY_ID));
    SplitActivity sa = new SplitActivity(new ActivityId(odId, SPLIT_ACTIVITY_ID));
    builder.addActivity(this, ca);
    builder.addSourceEdge(0, ca, 0);
    builder.addActivity(this, sa);
    builder.addTargetEdge(0, sa, 0);
    builder.addBlockingEdge(ca, sa);
}

Example 39 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class NestedLoopJoinOperatorDescriptor method contributeActivities.

@Override
public void contributeActivities(IActivityGraphBuilder builder) {
    ActivityId jcaId = new ActivityId(getOperatorId(), JOIN_CACHE_ACTIVITY_ID);
    ActivityId nljAid = new ActivityId(getOperatorId(), NL_JOIN_ACTIVITY_ID);
    JoinCacheActivityNode jc = new JoinCacheActivityNode(jcaId, nljAid);
    NestedLoopJoinActivityNode nlj = new NestedLoopJoinActivityNode(nljAid);
    builder.addActivity(this, jc);
    builder.addSourceEdge(1, jc, 0);
    builder.addActivity(this, nlj);
    builder.addSourceEdge(0, nlj, 0);
    builder.addTargetEdge(0, nlj, 0);
    builder.addBlockingEdge(jc, nlj);
}

Example 40 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class InMemorySortOperatorDescriptor method contributeActivities.

@Override
public void contributeActivities(IActivityGraphBuilder builder) {
    SortActivity sa = new SortActivity(new ActivityId(odId, SORT_ACTIVITY_ID));
    MergeActivity ma = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
    builder.addActivity(this, sa);
    builder.addSourceEdge(0, sa, 0);
    builder.addActivity(this, ma);
    builder.addTargetEdge(0, ma, 0);
    builder.addBlockingEdge(sa, ma);
}