Example 1 with DelayNode
use of org.apache.hudi.integ.testsuite.dag.nodes.DelayNode in project hudi by apache.
the class DagScheduler method execute.
/**
* Method to start executing the nodes in workflow DAGs.
*
* @param service ExecutorService
* @param workflowDag instance of workflow dag that needs to be executed
* @throws Exception will be thrown if ant error occurred
*/
private void execute(ExecutorService service, WorkflowDag workflowDag) throws Exception {
// Nodes at the same level are executed in parallel
log.info("Running workloads");
List<DagNode> nodes = workflowDag.getNodeList();
int curRound = 1;
do {
log.warn("===================================================================");
log.warn("Running workloads for round num " + curRound);
log.warn("===================================================================");
Queue<DagNode> queue = new PriorityQueue<>();
for (DagNode dagNode : nodes) {
queue.add(dagNode.clone());
}
do {
List<Future> futures = new ArrayList<>();
Set<DagNode> childNodes = new HashSet<>();
while (queue.size() > 0) {
DagNode nodeToExecute = queue.poll();
log.warn("Executing node \"" + nodeToExecute.getConfig().getOtherConfigs().get(CONFIG_NAME) + "\" :: " + nodeToExecute.getConfig());
int finalCurRound = curRound;
futures.add(service.submit(() -> executeNode(nodeToExecute, finalCurRound)));
if (nodeToExecute.getChildNodes().size() > 0) {
childNodes.addAll(nodeToExecute.getChildNodes());
}
}
queue.addAll(childNodes);
childNodes.clear();
for (Future future : futures) {
future.get(1, TimeUnit.HOURS);
}
} while (queue.size() > 0);
log.info("Finished workloads for round num " + curRound);
if (curRound < workflowDag.getRounds()) {
new DelayNode(workflowDag.getIntermittentDelayMins()).execute(executionContext, curRound);
}
// After each level, report and flush the metrics
Metrics.flush();
} while (curRound++ < workflowDag.getRounds());
log.info("Finished workloads");
}