use of io.hops.hopsworks.common.util.templates.airflow.AirflowJobSuccessSensor in project hopsworks by logicalclocks.
the class AirflowDagDTO method toAirflowDagTemplate.
public static AirflowDAG toAirflowDagTemplate(AirflowDagDTO dagDefinition, Users owner, Project project) {
AirflowDAG dag = new AirflowDAG(dagDefinition.getName(), owner.getUsername(), project.getName());
dag.setScheduleInterval(dagDefinition.getScheduleInterval());
if (dagDefinition.getApiKey() != null) {
dag.setApiKey(dagDefinition.getApiKey());
}
List<AirflowOperator> operators = new ArrayList<>(dagDefinition.getOperators().size());
for (AirflowOperatorDTO op : dagDefinition.getOperators()) {
AirflowOperator operator = null;
switch(op.getName()) {
case AirflowJobLaunchOperator.NAME:
operator = new AirflowJobLaunchOperator(project.getName(), op.getId(), op.getJobName(), op.getJobArgs());
((AirflowJobLaunchOperator) operator).setWait(op.isWait());
break;
case AirflowJobSuccessSensor.NAME:
operator = new AirflowJobSuccessSensor(project.getName(), op.getId(), op.getJobName());
break;
default:
}
if (operator != null) {
if (op.getDependsOn() != null && !op.getDependsOn().isEmpty()) {
StringBuilder sb = new StringBuilder();
if (op.getDependsOn().size() == 1) {
sb.append(AirflowOperator.sanitizeId(op.getDependsOn().get(0)));
} else {
sb.append("[");
for (String dependency : op.getDependsOn()) {
sb.append(AirflowOperator.sanitizeId(dependency)).append(",");
}
sb.append("]");
}
operator.setUpstream(sb.toString());
}
operators.add(operator);
}
}
dag.setOperators(operators);
return dag;
}
Aggregations