Search in sources :

Example 1 with IConnectorPolicy

use of org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy in project asterixdb by apache.

the class JobExecutor method findDoomedTaskClusters.

private boolean findDoomedTaskClusters(TaskCluster tc, Set<TaskCluster> doomedTaskClusters) {
    if (doomedTaskClusters.contains(tc)) {
        return true;
    }
    TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
    if (lastAttempt != null) {
        switch(lastAttempt.getStatus()) {
            case ABORTED:
            case FAILED:
            case COMPLETED:
                return false;
            default:
                break;
        }
    }
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicyMap = jobRun.getConnectorPolicyMap();
    PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
    boolean doomed = false;
    for (TaskCluster depTC : tc.getDependencyTaskClusters()) {
        if (findDoomedTaskClusters(depTC, doomedTaskClusters)) {
            doomed = true;
        }
    }
    for (PartitionId pid : tc.getRequiredPartitions()) {
        ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
        IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
        PartitionState maxState = pmm.getMaximumAvailableState(pid);
        if ((maxState == null || (cPolicy.consumerWaitsForProducerToFinish() && maxState != PartitionState.COMMITTED)) && findDoomedTaskClusters(partitionProducingTaskClusterMap.get(pid), doomedTaskClusters)) {
            doomed = true;
        }
    }
    if (doomed) {
        doomedTaskClusters.add(tc);
    }
    return doomed;
}
Also used : TaskClusterAttempt(org.apache.hyracks.control.cc.job.TaskClusterAttempt) IConnectorPolicy(org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy) ConnectorDescriptorId(org.apache.hyracks.api.dataflow.ConnectorDescriptorId) TaskCluster(org.apache.hyracks.control.cc.job.TaskCluster) PartitionState(org.apache.hyracks.control.common.job.PartitionState) PartitionMatchMaker(org.apache.hyracks.control.cc.partitions.PartitionMatchMaker) PartitionId(org.apache.hyracks.api.partitions.PartitionId)

Example 2 with IConnectorPolicy

use of org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy in project asterixdb by apache.

the class ActivityClusterPlanner method assignConnectorPolicy.

private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
    Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap = new HashMap<>();
    Set<ActivityId> activities = ac.getActivityMap().keySet();
    BitSet targetBitmap = new BitSet();
    for (ActivityId a1 : activities) {
        Task[] ac1TaskStates = taskMap.get(a1).getTasks();
        int nProducers = ac1TaskStates.length;
        List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
        if (outputConns == null) {
            continue;
        }
        for (IConnectorDescriptor c : outputConns) {
            ConnectorDescriptorId cdId = c.getConnectorId();
            ActivityId a2 = ac.getConsumerActivity(cdId);
            Task[] ac2TaskStates = taskMap.get(a2).getTasks();
            int nConsumers = ac2TaskStates.length;
            int[] fanouts = new int[nProducers];
            if (c.allProducersToAllConsumers()) {
                for (int i = 0; i < nProducers; ++i) {
                    fanouts[i] = nConsumers;
                }
            } else {
                for (int i = 0; i < nProducers; ++i) {
                    c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
                    fanouts[i] = targetBitmap.cardinality();
                }
            }
            IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
            cPolicyMap.put(cdId, cp);
        }
    }
    executor.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
Also used : IConnectorDescriptor(org.apache.hyracks.api.dataflow.IConnectorDescriptor) Task(org.apache.hyracks.control.cc.job.Task) HashMap(java.util.HashMap) IConnectorPolicy(org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy) ActivityId(org.apache.hyracks.api.dataflow.ActivityId) ConnectorDescriptorId(org.apache.hyracks.api.dataflow.ConnectorDescriptorId) BitSet(java.util.BitSet)

Example 3 with IConnectorPolicy

use of org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy in project asterixdb by apache.

the class StartTasksWork method run.

@Override
public void run() {
    Task task = null;
    try {
        NCServiceContext serviceCtx = ncs.getContext();
        Joblet joblet = getOrCreateLocalJoblet(deploymentId, jobId, serviceCtx, acgBytes);
        final ActivityClusterGraph acg = joblet.getActivityClusterGraph();
        IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {

            @Override
            public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) {
                ActivityCluster ac = acg.getActivityMap().get(aid);
                IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex);
                return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }

            @Override
            public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) {
                ActivityCluster ac = acg.getActivityMap().get(aid);
                IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex);
                return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }
        };
        for (TaskAttemptDescriptor td : taskDescriptors) {
            TaskAttemptId taId = td.getTaskAttemptId();
            TaskId tid = taId.getTaskId();
            ActivityId aid = tid.getActivityId();
            ActivityCluster ac = acg.getActivityMap().get(aid);
            IActivity han = ac.getActivityMap().get(aid);
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("Initializing " + taId + " -> " + han);
            }
            final int partition = tid.getPartition();
            List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(aid);
            task = new Task(joblet, taId, han.getClass().getName(), ncs.getExecutor(), ncs, createInputChannels(td, inputs));
            IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
            List<IPartitionCollector> collectors = new ArrayList<>();
            if (inputs != null) {
                for (int i = 0; i < inputs.size(); ++i) {
                    IConnectorDescriptor conn = inputs.get(i);
                    IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
                    if (LOGGER.isLoggable(Level.INFO)) {
                        LOGGER.info("input: " + i + ": " + conn.getConnectorId());
                    }
                    RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                    IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn, recordDesc, cPolicy);
                    collectors.add(collector);
                }
            }
            List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(aid);
            if (outputs != null) {
                for (int i = 0; i < outputs.size(); ++i) {
                    final IConnectorDescriptor conn = outputs.get(i);
                    RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                    IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
                    IPartitionWriterFactory pwFactory = createPartitionWriterFactory(task, cPolicy, jobId, conn, partition, taId, flags);
                    if (LOGGER.isLoggable(Level.INFO)) {
                        LOGGER.info("output: " + i + ": " + conn.getConnectorId());
                    }
                    IFrameWriter writer = conn.createPartitioner(task, recordDesc, pwFactory, partition, td.getPartitionCount(), td.getOutputPartitionCounts()[i]);
                    operator.setOutputFrameWriter(i, writer, recordDesc);
                }
            }
            task.setTaskRuntime(collectors.toArray(new IPartitionCollector[collectors.size()]), operator);
            joblet.addTask(task);
            task.start();
        }
    } catch (Exception e) {
        LOGGER.log(Level.WARNING, "Failure starting a task", e);
        // notify cc of start task failure
        List<Exception> exceptions = new ArrayList<>();
        ExceptionUtils.setNodeIds(exceptions, ncs.getId());
        ncs.getWorkQueue().schedule(new NotifyTaskFailureWork(ncs, task, exceptions));
    }
}
Also used : IFrameWriter(org.apache.hyracks.api.comm.IFrameWriter) Task(org.apache.hyracks.control.nc.Task) TaskId(org.apache.hyracks.api.dataflow.TaskId) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ActivityId(org.apache.hyracks.api.dataflow.ActivityId) ArrayList(java.util.ArrayList) IRecordDescriptorProvider(org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider) Joblet(org.apache.hyracks.control.nc.Joblet) IActivity(org.apache.hyracks.api.dataflow.IActivity) NCServiceContext(org.apache.hyracks.control.nc.application.NCServiceContext) INCServiceContext(org.apache.hyracks.api.application.INCServiceContext) List(java.util.List) ArrayList(java.util.ArrayList) IConnectorDescriptor(org.apache.hyracks.api.dataflow.IConnectorDescriptor) IPartitionCollector(org.apache.hyracks.api.comm.IPartitionCollector) TaskAttemptId(org.apache.hyracks.api.dataflow.TaskAttemptId) IConnectorPolicy(org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy) IPartitionWriterFactory(org.apache.hyracks.api.comm.IPartitionWriterFactory) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) UnknownHostException(java.net.UnknownHostException) HyracksException(org.apache.hyracks.api.exceptions.HyracksException) ActivityCluster(org.apache.hyracks.api.job.ActivityCluster) TaskAttemptDescriptor(org.apache.hyracks.control.common.job.TaskAttemptDescriptor) ActivityClusterGraph(org.apache.hyracks.api.job.ActivityClusterGraph) IOperatorNodePushable(org.apache.hyracks.api.dataflow.IOperatorNodePushable)

Example 4 with IConnectorPolicy

use of org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy in project asterixdb by apache.

the class ActivityClusterPlanner method buildConnectorPolicyAwareTaskClusters.

private TaskCluster[] buildConnectorPolicyAwareTaskClusters(ActivityCluster ac, Map<ActivityId, ActivityPlan> activityPlanMap, Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
    Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        ActivityPlan ap = activityPlanMap.get(anId);
        Task[] tasks = ap.getTasks();
        for (Task t : tasks) {
            Set<TaskId> cluster = new HashSet<>();
            TaskId tid = t.getTaskId();
            cluster.add(tid);
            taskClusterMap.put(tid, cluster);
        }
    }
    JobRun jobRun = executor.getJobRun();
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
    for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e : taskConnectivity.entrySet()) {
        Set<TaskId> cluster = taskClusterMap.get(e.getKey());
        for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
            IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
            if (cPolicy.requiresProducerConsumerCoscheduling()) {
                cluster.add(p.getLeft());
            }
        }
    }
    /*
         * We compute the transitive closure of this (producer-consumer) relation to find the largest set of
         * tasks that need to be co-scheduled.
         */
    int counter = 0;
    TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
    Map<TaskId, Integer> ordinalMap = new HashMap<>();
    for (TaskId tid : taskClusterMap.keySet()) {
        ordinalList[counter] = tid;
        ordinalMap.put(tid, counter);
        ++counter;
    }
    int n = ordinalList.length;
    BitSet[] paths = new BitSet[n];
    for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
        int i = ordinalMap.get(e.getKey());
        BitSet bsi = paths[i];
        if (bsi == null) {
            bsi = new BitSet(n);
            paths[i] = bsi;
        }
        for (TaskId ttid : e.getValue()) {
            int j = ordinalMap.get(ttid);
            paths[i].set(j);
            BitSet bsj = paths[j];
            if (bsj == null) {
                bsj = new BitSet(n);
                paths[j] = bsj;
            }
            bsj.set(i);
        }
    }
    for (int k = 0; k < n; ++k) {
        for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
            for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
                paths[i].set(j, paths[k].get(j));
                paths[j].set(i, paths[i].get(j));
            }
        }
    }
    BitSet pending = new BitSet(n);
    pending.set(0, n);
    List<List<TaskId>> clusters = new ArrayList<>();
    for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
        List<TaskId> cluster = new ArrayList<>();
        for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
            cluster.add(ordinalList[j]);
            pending.clear(j);
        }
        clusters.add(cluster);
    }
    List<TaskCluster> tcSet = new ArrayList<>();
    counter = 0;
    for (List<TaskId> cluster : clusters) {
        List<Task> taskStates = new ArrayList<>();
        for (TaskId tid : cluster) {
            taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
        }
        TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), counter++), ac, taskStates.toArray(new Task[taskStates.size()]));
        tcSet.add(tc);
        for (TaskId tid : cluster) {
            activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()].setTaskCluster(tc);
        }
    }
    return tcSet.toArray(new TaskCluster[tcSet.size()]);
}
Also used : Task(org.apache.hyracks.control.cc.job.Task) TaskId(org.apache.hyracks.api.dataflow.TaskId) HashSet(java.util.HashSet) Set(java.util.Set) BitSet(java.util.BitSet) HashMap(java.util.HashMap) ActivityId(org.apache.hyracks.api.dataflow.ActivityId) ConnectorDescriptorId(org.apache.hyracks.api.dataflow.ConnectorDescriptorId) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) HashSet(java.util.HashSet) IConnectorPolicy(org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy) ActivityPlan(org.apache.hyracks.control.cc.job.ActivityPlan) BitSet(java.util.BitSet) TaskCluster(org.apache.hyracks.control.cc.job.TaskCluster) HashMap(java.util.HashMap) Map(java.util.Map) TaskClusterId(org.apache.hyracks.control.cc.job.TaskClusterId) JobRun(org.apache.hyracks.control.cc.job.JobRun)

Example 5 with IConnectorPolicy

use of org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy in project asterixdb by apache.

the class JobExecutor method assignRunnabilityRank.

/*
     * Runnability rank has the following semantics
     * Runnability(Runnable TaskCluster depending on completed TaskClusters) = {RUNNABLE, 0}
     * Runnability(Runnable TaskCluster) = max(Rank(Dependent TaskClusters)) + 1
     * Runnability(Non-schedulable TaskCluster) = {NOT_RUNNABLE, _}
     */
private Runnability assignRunnabilityRank(TaskCluster goal, Map<TaskCluster, Runnability> runnabilityMap) {
    if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine("Computing runnability: " + goal);
    }
    if (runnabilityMap.containsKey(goal)) {
        return runnabilityMap.get(goal);
    }
    TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(goal);
    if (lastAttempt != null) {
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Last Attempt Status: " + lastAttempt.getStatus());
        }
        if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
            Runnability runnability = new Runnability(Runnability.Tag.COMPLETED, Integer.MIN_VALUE);
            runnabilityMap.put(goal, runnability);
            return runnability;
        }
        if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.RUNNING) {
            Runnability runnability = new Runnability(Runnability.Tag.RUNNING, Integer.MIN_VALUE);
            runnabilityMap.put(goal, runnability);
            return runnability;
        }
    }
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicyMap = jobRun.getConnectorPolicyMap();
    PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
    Runnability aggregateRunnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
    for (PartitionId pid : goal.getRequiredPartitions()) {
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Inspecting required partition: " + pid);
        }
        Runnability runnability;
        ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
        IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
        PartitionState maxState = pmm.getMaximumAvailableState(pid);
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Policy: " + cPolicy + " maxState: " + maxState);
        }
        if (PartitionState.COMMITTED.equals(maxState)) {
            runnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
        } else if (PartitionState.STARTED.equals(maxState) && !cPolicy.consumerWaitsForProducerToFinish()) {
            runnability = new Runnability(Runnability.Tag.RUNNABLE, 1);
        } else {
            runnability = assignRunnabilityRank(partitionProducingTaskClusterMap.get(pid), runnabilityMap);
            switch(runnability.getTag()) {
                case RUNNABLE:
                    if (cPolicy.consumerWaitsForProducerToFinish()) {
                        runnability = new Runnability(Runnability.Tag.NOT_RUNNABLE, Integer.MAX_VALUE);
                    } else {
                        runnability = new Runnability(Runnability.Tag.RUNNABLE, runnability.getPriority() + 1);
                    }
                    break;
                case NOT_RUNNABLE:
                    break;
                case RUNNING:
                    if (cPolicy.consumerWaitsForProducerToFinish()) {
                        runnability = new Runnability(Runnability.Tag.NOT_RUNNABLE, Integer.MAX_VALUE);
                    } else {
                        runnability = new Runnability(Runnability.Tag.RUNNABLE, 1);
                    }
                    break;
                default:
                    break;
            }
        }
        aggregateRunnability = Runnability.getWorstCase(aggregateRunnability, runnability);
        if (aggregateRunnability.getTag() == Runnability.Tag.NOT_RUNNABLE) {
            // already not runnable -- cannot get better. bail.
            break;
        }
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("aggregateRunnability: " + aggregateRunnability);
        }
    }
    runnabilityMap.put(goal, aggregateRunnability);
    return aggregateRunnability;
}
Also used : TaskClusterAttempt(org.apache.hyracks.control.cc.job.TaskClusterAttempt) IConnectorPolicy(org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy) ConnectorDescriptorId(org.apache.hyracks.api.dataflow.ConnectorDescriptorId) PartitionState(org.apache.hyracks.control.common.job.PartitionState) PartitionMatchMaker(org.apache.hyracks.control.cc.partitions.PartitionMatchMaker) PartitionId(org.apache.hyracks.api.partitions.PartitionId)

Aggregations

IConnectorPolicy (org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy)7 ConnectorDescriptorId (org.apache.hyracks.api.dataflow.ConnectorDescriptorId)6 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)4 List (java.util.List)4 ActivityId (org.apache.hyracks.api.dataflow.ActivityId)4 Map (java.util.Map)3 IConnectorDescriptor (org.apache.hyracks.api.dataflow.IConnectorDescriptor)3 TaskId (org.apache.hyracks.api.dataflow.TaskId)3 ActivityClusterGraph (org.apache.hyracks.api.job.ActivityClusterGraph)3 Task (org.apache.hyracks.control.cc.job.Task)3 TaskClusterAttempt (org.apache.hyracks.control.cc.job.TaskClusterAttempt)3 TaskAttemptDescriptor (org.apache.hyracks.control.common.job.TaskAttemptDescriptor)3 BitSet (java.util.BitSet)2 TaskAttemptId (org.apache.hyracks.api.dataflow.TaskAttemptId)2 HyracksException (org.apache.hyracks.api.exceptions.HyracksException)2 PartitionId (org.apache.hyracks.api.partitions.PartitionId)2 INodeManager (org.apache.hyracks.control.cc.cluster.INodeManager)2 UnknownHostException (java.net.UnknownHostException)1 HashSet (java.util.HashSet)1