Examples with ActivityId org.apache.hyracks.api.dataflow.ActivityId used on opensource projects

Example 16 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterPlanner method assignConnectorPolicy.

private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
    Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap = new HashMap<>();
    Set<ActivityId> activities = ac.getActivityMap().keySet();
    BitSet targetBitmap = new BitSet();
    for (ActivityId a1 : activities) {
        Task[] ac1TaskStates = taskMap.get(a1).getTasks();
        int nProducers = ac1TaskStates.length;
        List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
        if (outputConns == null) {
            continue;
        }
        for (IConnectorDescriptor c : outputConns) {
            ConnectorDescriptorId cdId = c.getConnectorId();
            ActivityId a2 = ac.getConsumerActivity(cdId);
            Task[] ac2TaskStates = taskMap.get(a2).getTasks();
            int nConsumers = ac2TaskStates.length;
            int[] fanouts = new int[nProducers];
            if (c.allProducersToAllConsumers()) {
                for (int i = 0; i < nProducers; ++i) {
                    fanouts[i] = nConsumers;
                }
            } else {
                for (int i = 0; i < nProducers; ++i) {
                    c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
                    fanouts[i] = targetBitmap.cardinality();
                }
            }
            IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
            cPolicyMap.put(cdId, cp);
        }
    }
    executor.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}

Example 17 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterPlanner method buildActivityPlanMap.

private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
    Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<>();
    Set<ActivityId> depAnIds = new HashSet<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        depAnIds.clear();
        getDependencyActivityIds(depAnIds, anId, ac);
        ActivityPartitionDetails apd = pcMap.get(anId);
        Task[] tasks = new Task[apd.getPartitionCount()];
        ActivityPlan activityPlan = new ActivityPlan(apd);
        for (int i = 0; i < tasks.length; ++i) {
            TaskId tid = new TaskId(anId, i);
            tasks[i] = new Task(tid, activityPlan);
            for (ActivityId danId : depAnIds) {
                ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
                ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
                assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for " + "dependency AC: Encountered no plan for ActivityID " + danId;
                Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
                assert dATasks != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for" + " dependency AC: Encountered no plan for ActivityID " + danId;
                assert dATasks.length == tasks.length : "Dependency activity partitioned differently from " + "dependent: " + dATasks.length + " != " + tasks.length;
                Task dTask = dATasks[i];
                TaskId dTaskId = dTask.getTaskId();
                tasks[i].getDependencies().add(dTaskId);
                dTask.getDependents().add(tid);
            }
        }
        activityPlan.setTasks(tasks);
        activityPlanMap.put(anId, activityPlan);
    }
    return activityPlanMap;
}

Example 18 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterPlanner method buildConnectorPolicyUnawareTaskClusters.

private TaskCluster[] buildConnectorPolicyUnawareTaskClusters(ActivityCluster ac, Map<ActivityId, ActivityPlan> activityPlanMap) {
    List<Task> taskStates = new ArrayList<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        ActivityPlan ap = activityPlanMap.get(anId);
        Task[] tasks = ap.getTasks();
        taskStates.addAll(Arrays.asList(tasks));
    }
    TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), 0), ac, taskStates.toArray(new Task[taskStates.size()]));
    for (Task t : tc.getTasks()) {
        t.setTaskCluster(tc);
    }
    return new TaskCluster[] { tc };
}

Example 19 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterPlanner method planActivityCluster.

ActivityClusterPlan planActivityCluster(ActivityCluster ac) throws HyracksException {
    JobRun jobRun = executor.getJobRun();
    Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
    Map<ActivityId, ActivityPlan> activityPlanMap = buildActivityPlanMap(ac, jobRun, pcMap);
    assignConnectorPolicy(ac, activityPlanMap);
    TaskCluster[] taskClusters = computeTaskClusters(ac, jobRun, activityPlanMap);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Plan for " + ac);
        LOGGER.info("Built " + taskClusters.length + " Task Clusters");
        for (TaskCluster tc : taskClusters) {
            LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
        }
    }
    return new ActivityClusterPlan(taskClusters, activityPlanMap);
}

Example 20 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class FramewriterTest method createWriters.

/**
     * @return a list of writers to test. these writers can be of the same type but behave differently based on included mocks
     * @throws HyracksDataException
     * @throws IndexException
     */
public IFrameWriter[] createWriters() throws HyracksDataException {
    ArrayList<BTreeSearchOperatorNodePushable> writers = new ArrayList<>();
    Pair<IIndexDataflowHelperFactory, ISearchOperationCallbackFactory>[] pairs = pairs();
    IRecordDescriptorProvider[] recordDescProviders = mockRecDescProviders();
    int partition = 0;
    IHyracksTaskContext[] ctxs = mockIHyracksTaskContext();
    int[] keys = { 0 };
    boolean lowKeyInclusive = true;
    boolean highKeyInclusive = true;
    for (Pair<IIndexDataflowHelperFactory, ISearchOperationCallbackFactory> pair : pairs) {
        for (IRecordDescriptorProvider recordDescProvider : recordDescProviders) {
            for (IHyracksTaskContext ctx : ctxs) {
                BTreeSearchOperatorNodePushable writer = new BTreeSearchOperatorNodePushable(ctx, partition, recordDescProvider.getInputRecordDescriptor(new ActivityId(new OperatorDescriptorId(0), 0), 0), keys, keys, lowKeyInclusive, highKeyInclusive, keys, keys, pair.getLeft(), false, false, null, pair.getRight(), false);
                writers.add(writer);
            }
        }
    }
    // Create the framewriter using the mocks
    return writers.toArray(new IFrameWriter[writers.size()]);
}