Examples with TaskId org.apache.hyracks.api.dataflow.TaskId used on opensource projects

Example 1 with TaskId

use of org.apache.hyracks.api.dataflow.TaskId in project asterixdb by apache.

the class JobExecutor method assignLocation.

private String assignLocation(ActivityClusterGraph acg, Map<TaskId, LValueConstraintExpression> locationMap, TaskId tid, TaskAttempt taskAttempt) throws HyracksException {
    ActivityId aid = tid.getActivityId();
    ActivityCluster ac = acg.getActivityMap().get(aid);
    Set<ActivityId> blockers = ac.getBlocked2BlockerMap().get(aid);
    String nodeId = null;
    if (blockers != null) {
        for (ActivityId blocker : blockers) {
            nodeId = findTaskLocation(new TaskId(blocker, tid.getPartition()));
            if (nodeId != null) {
                break;
            }
        }
    }
    INodeManager nodeManager = ccs.getNodeManager();
    Collection<String> liveNodes = nodeManager.getAllNodeIds();
    if (nodeId == null) {
        LValueConstraintExpression pLocationExpr = locationMap.get(tid);
        Object location = solver.getValue(pLocationExpr);
        if (location == null) {
            // pick any
            nodeId = liveNodes.toArray(new String[liveNodes.size()])[random.nextInt(1) % liveNodes.size()];
        } else if (location instanceof String) {
            nodeId = (String) location;
        } else if (location instanceof String[]) {
            for (String choice : (String[]) location) {
                if (liveNodes.contains(choice)) {
                    nodeId = choice;
                    break;
                }
            }
            if (nodeId == null) {
                throw new HyracksException("No satisfiable location found for " + taskAttempt.getTaskAttemptId());
            }
        } else {
            throw new HyracksException("Unknown type of value for " + pLocationExpr + ": " + location + "(" + location.getClass() + ")");
        }
    }
    if (nodeId == null) {
        throw new HyracksException("No satisfiable location found for " + taskAttempt.getTaskAttemptId());
    }
    if (!liveNodes.contains(nodeId)) {
        throw new HyracksException("Node " + nodeId + " not live");
    }
    return nodeId;
}

Example 2 with TaskId

use of org.apache.hyracks.api.dataflow.TaskId in project asterixdb by apache.

the class CCNCFunctions method writeTaskAttemptId.

private static void writeTaskAttemptId(DataOutputStream dos, TaskAttemptId taId) throws IOException {
    TaskId tid = taId.getTaskId();
    ActivityId aid = tid.getActivityId();
    OperatorDescriptorId odId = aid.getOperatorDescriptorId();
    dos.writeInt(odId.getId());
    dos.writeInt(aid.getLocalId());
    dos.writeInt(tid.getPartition());
    dos.writeInt(taId.getAttempt());
}

Example 3 with TaskId

use of org.apache.hyracks.api.dataflow.TaskId in project asterixdb by apache.

the class CCNCFunctions method readTaskAttemptId.

private static TaskAttemptId readTaskAttemptId(DataInputStream dis) throws IOException {
    int odid = dis.readInt();
    int aid = dis.readInt();
    int partition = dis.readInt();
    int attempt = dis.readInt();
    TaskAttemptId taId = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(odid), aid), partition), attempt);
    return taId;
}

Example 4 with TaskId

use of org.apache.hyracks.api.dataflow.TaskId in project asterixdb by apache.

the class ActivityClusterPlanner method buildActivityPlanMap.

private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
    Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<>();
    Set<ActivityId> depAnIds = new HashSet<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        depAnIds.clear();
        getDependencyActivityIds(depAnIds, anId, ac);
        ActivityPartitionDetails apd = pcMap.get(anId);
        Task[] tasks = new Task[apd.getPartitionCount()];
        ActivityPlan activityPlan = new ActivityPlan(apd);
        for (int i = 0; i < tasks.length; ++i) {
            TaskId tid = new TaskId(anId, i);
            tasks[i] = new Task(tid, activityPlan);
            for (ActivityId danId : depAnIds) {
                ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
                ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
                assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for " + "dependency AC: Encountered no plan for ActivityID " + danId;
                Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
                assert dATasks != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for" + " dependency AC: Encountered no plan for ActivityID " + danId;
                assert dATasks.length == tasks.length : "Dependency activity partitioned differently from " + "dependent: " + dATasks.length + " != " + tasks.length;
                Task dTask = dATasks[i];
                TaskId dTaskId = dTask.getTaskId();
                tasks[i].getDependencies().add(dTaskId);
                dTask.getDependents().add(tid);
            }
        }
        activityPlan.setTasks(tasks);
        activityPlanMap.put(anId, activityPlan);
    }
    return activityPlanMap;
}

Example 5 with TaskId

use of org.apache.hyracks.api.dataflow.TaskId in project asterixdb by apache.

the class StartTasksWork method run.

@Override
public void run() {
    Task task = null;
    try {
        NCServiceContext serviceCtx = ncs.getContext();
        Joblet joblet = getOrCreateLocalJoblet(deploymentId, jobId, serviceCtx, acgBytes);
        final ActivityClusterGraph acg = joblet.getActivityClusterGraph();
        IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {

            @Override
            public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) {
                ActivityCluster ac = acg.getActivityMap().get(aid);
                IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex);
                return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }

            @Override
            public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) {
                ActivityCluster ac = acg.getActivityMap().get(aid);
                IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex);
                return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }
        };
        for (TaskAttemptDescriptor td : taskDescriptors) {
            TaskAttemptId taId = td.getTaskAttemptId();
            TaskId tid = taId.getTaskId();
            ActivityId aid = tid.getActivityId();
            ActivityCluster ac = acg.getActivityMap().get(aid);
            IActivity han = ac.getActivityMap().get(aid);
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("Initializing " + taId + " -> " + han);
            }
            final int partition = tid.getPartition();
            List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(aid);
            task = new Task(joblet, taId, han.getClass().getName(), ncs.getExecutor(), ncs, createInputChannels(td, inputs));
            IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
            List<IPartitionCollector> collectors = new ArrayList<>();
            if (inputs != null) {
                for (int i = 0; i < inputs.size(); ++i) {
                    IConnectorDescriptor conn = inputs.get(i);
                    IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
                    if (LOGGER.isLoggable(Level.INFO)) {
                        LOGGER.info("input: " + i + ": " + conn.getConnectorId());
                    }
                    RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                    IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn, recordDesc, cPolicy);
                    collectors.add(collector);
                }
            }
            List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(aid);
            if (outputs != null) {
                for (int i = 0; i < outputs.size(); ++i) {
                    final IConnectorDescriptor conn = outputs.get(i);
                    RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                    IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
                    IPartitionWriterFactory pwFactory = createPartitionWriterFactory(task, cPolicy, jobId, conn, partition, taId, flags);
                    if (LOGGER.isLoggable(Level.INFO)) {
                        LOGGER.info("output: " + i + ": " + conn.getConnectorId());
                    }
                    IFrameWriter writer = conn.createPartitioner(task, recordDesc, pwFactory, partition, td.getPartitionCount(), td.getOutputPartitionCounts()[i]);
                    operator.setOutputFrameWriter(i, writer, recordDesc);
                }
            }
            task.setTaskRuntime(collectors.toArray(new IPartitionCollector[collectors.size()]), operator);
            joblet.addTask(task);
            task.start();
        }
    } catch (Exception e) {
        LOGGER.log(Level.WARNING, "Failure starting a task", e);
        // notify cc of start task failure
        List<Exception> exceptions = new ArrayList<>();
        ExceptionUtils.setNodeIds(exceptions, ncs.getId());
        ncs.getWorkQueue().schedule(new NotifyTaskFailureWork(ncs, task, exceptions));
    }
}