Examples with PartitionState org.apache.hyracks.control.common.job.PartitionState used on opensource projects

Example 1 with PartitionState

use of org.apache.hyracks.control.common.job.PartitionState in project asterixdb by apache.

the class JobExecutor method findDoomedTaskClusters.

private boolean findDoomedTaskClusters(TaskCluster tc, Set<TaskCluster> doomedTaskClusters) {
    if (doomedTaskClusters.contains(tc)) {
        return true;
    }
    TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
    if (lastAttempt != null) {
        switch(lastAttempt.getStatus()) {
            case ABORTED:
            case FAILED:
            case COMPLETED:
                return false;
            default:
                break;
        }
    }
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicyMap = jobRun.getConnectorPolicyMap();
    PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
    boolean doomed = false;
    for (TaskCluster depTC : tc.getDependencyTaskClusters()) {
        if (findDoomedTaskClusters(depTC, doomedTaskClusters)) {
            doomed = true;
        }
    }
    for (PartitionId pid : tc.getRequiredPartitions()) {
        ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
        IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
        PartitionState maxState = pmm.getMaximumAvailableState(pid);
        if ((maxState == null || (cPolicy.consumerWaitsForProducerToFinish() && maxState != PartitionState.COMMITTED)) && findDoomedTaskClusters(partitionProducingTaskClusterMap.get(pid), doomedTaskClusters)) {
            doomed = true;
        }
    }
    if (doomed) {
        doomedTaskClusters.add(tc);
    }
    return doomed;
}

Example 2 with PartitionState

use of org.apache.hyracks.control.common.job.PartitionState in project asterixdb by apache.

the class JobExecutor method assignRunnabilityRank.

/*
     * Runnability rank has the following semantics
     * Runnability(Runnable TaskCluster depending on completed TaskClusters) = {RUNNABLE, 0}
     * Runnability(Runnable TaskCluster) = max(Rank(Dependent TaskClusters)) + 1
     * Runnability(Non-schedulable TaskCluster) = {NOT_RUNNABLE, _}
     */
private Runnability assignRunnabilityRank(TaskCluster goal, Map<TaskCluster, Runnability> runnabilityMap) {
    if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine("Computing runnability: " + goal);
    }
    if (runnabilityMap.containsKey(goal)) {
        return runnabilityMap.get(goal);
    }
    TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(goal);
    if (lastAttempt != null) {
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Last Attempt Status: " + lastAttempt.getStatus());
        }
        if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
            Runnability runnability = new Runnability(Runnability.Tag.COMPLETED, Integer.MIN_VALUE);
            runnabilityMap.put(goal, runnability);
            return runnability;
        }
        if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.RUNNING) {
            Runnability runnability = new Runnability(Runnability.Tag.RUNNING, Integer.MIN_VALUE);
            runnabilityMap.put(goal, runnability);
            return runnability;
        }
    }
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicyMap = jobRun.getConnectorPolicyMap();
    PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
    Runnability aggregateRunnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
    for (PartitionId pid : goal.getRequiredPartitions()) {
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Inspecting required partition: " + pid);
        }
        Runnability runnability;
        ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
        IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
        PartitionState maxState = pmm.getMaximumAvailableState(pid);
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("Policy: " + cPolicy + " maxState: " + maxState);
        }
        if (PartitionState.COMMITTED.equals(maxState)) {
            runnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
        } else if (PartitionState.STARTED.equals(maxState) && !cPolicy.consumerWaitsForProducerToFinish()) {
            runnability = new Runnability(Runnability.Tag.RUNNABLE, 1);
        } else {
            runnability = assignRunnabilityRank(partitionProducingTaskClusterMap.get(pid), runnabilityMap);
            switch(runnability.getTag()) {
                case RUNNABLE:
                    if (cPolicy.consumerWaitsForProducerToFinish()) {
                        runnability = new Runnability(Runnability.Tag.NOT_RUNNABLE, Integer.MAX_VALUE);
                    } else {
                        runnability = new Runnability(Runnability.Tag.RUNNABLE, runnability.getPriority() + 1);
                    }
                    break;
                case NOT_RUNNABLE:
                    break;
                case RUNNING:
                    if (cPolicy.consumerWaitsForProducerToFinish()) {
                        runnability = new Runnability(Runnability.Tag.NOT_RUNNABLE, Integer.MAX_VALUE);
                    } else {
                        runnability = new Runnability(Runnability.Tag.RUNNABLE, 1);
                    }
                    break;
                default:
                    break;
            }
        }
        aggregateRunnability = Runnability.getWorstCase(aggregateRunnability, runnability);
        if (aggregateRunnability.getTag() == Runnability.Tag.NOT_RUNNABLE) {
            // already not runnable -- cannot get better. bail.
            break;
        }
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.fine("aggregateRunnability: " + aggregateRunnability);
        }
    }
    runnabilityMap.put(goal, aggregateRunnability);
    return aggregateRunnability;
}