Examples with AirflowOperator io.hops.hopsworks.common.util.templates.airflow.AirflowOperator used on opensource projects

Example 1 with AirflowOperator

use of io.hops.hopsworks.common.util.templates.airflow.AirflowOperator in project hopsworks by logicalclocks.

the class AirflowDagDTO method toAirflowDagTemplate.

public static AirflowDAG toAirflowDagTemplate(AirflowDagDTO dagDefinition, Users owner, Project project) {
    AirflowDAG dag = new AirflowDAG(dagDefinition.getName(), owner.getUsername(), project.getName());
    dag.setScheduleInterval(dagDefinition.getScheduleInterval());
    if (dagDefinition.getApiKey() != null) {
        dag.setApiKey(dagDefinition.getApiKey());
    }
    List<AirflowOperator> operators = new ArrayList<>(dagDefinition.getOperators().size());
    for (AirflowOperatorDTO op : dagDefinition.getOperators()) {
        AirflowOperator operator = null;
        switch(op.getName()) {
            case AirflowJobLaunchOperator.NAME:
                operator = new AirflowJobLaunchOperator(project.getName(), op.getId(), op.getJobName(), op.getJobArgs());
                ((AirflowJobLaunchOperator) operator).setWait(op.isWait());
                break;
            case AirflowJobSuccessSensor.NAME:
                operator = new AirflowJobSuccessSensor(project.getName(), op.getId(), op.getJobName());
                break;
            default:
        }
        if (operator != null) {
            if (op.getDependsOn() != null && !op.getDependsOn().isEmpty()) {
                StringBuilder sb = new StringBuilder();
                if (op.getDependsOn().size() == 1) {
                    sb.append(AirflowOperator.sanitizeId(op.getDependsOn().get(0)));
                } else {
                    sb.append("[");
                    for (String dependency : op.getDependsOn()) {
                        sb.append(AirflowOperator.sanitizeId(dependency)).append(",");
                    }
                    sb.append("]");
                }
                operator.setUpstream(sb.toString());
            }
            operators.add(operator);
        }
    }
    dag.setOperators(operators);
    return dag;
}