Search in sources :

Example 1 with MultiConnectorSource

use of io.cdap.cdap.etl.batch.connector.MultiConnectorSource in project cdap by caskdata.

the class SmartWorkflow method getPhaseSpec.

private BatchPhaseSpec getPhaseSpec(String programName, PipelinePhase phase) {
    Map<String, String> phaseConnectorDatasets = new HashMap<>();
    // if this phase uses connectors, add the local dataset for that connector if we haven't already
    for (StageSpec connectorInfo : phase.getStagesOfType(Constants.Connector.PLUGIN_TYPE)) {
        String connectorName = connectorInfo.getName();
        if (!connectorDatasets.containsKey(connectorName)) {
            String datasetName = "conn-" + connectorNum++;
            connectorDatasets.put(connectorName, datasetName);
            phaseConnectorDatasets.put(connectorName, datasetName);
            // add the local dataset
            ConnectorSource connectorSource = new MultiConnectorSource(datasetName, null);
            connectorSource.configure(getConfigurer());
        } else {
            phaseConnectorDatasets.put(connectorName, connectorDatasets.get(connectorName));
        }
    }
    // published.
    for (StageSpec alertPublisherInfo : phase.getStagesOfType(AlertPublisher.PLUGIN_TYPE)) {
        String stageName = alertPublisherInfo.getName();
        if (!connectorDatasets.containsKey(stageName)) {
            String datasetName = "alerts-" + publisherNum++;
            connectorDatasets.put(alertPublisherInfo.getName(), datasetName);
            phaseConnectorDatasets.put(alertPublisherInfo.getName(), datasetName);
            AlertPublisherSink alertPublisherSink = new AlertPublisherSink(datasetName, null);
            alertPublisherSink.configure(getConfigurer());
        } else {
            phaseConnectorDatasets.put(stageName, connectorDatasets.get(stageName));
        }
    }
    return new BatchPhaseSpec(programName, phase, spec.getResources(), spec.getDriverResources(), spec.getClientResources(), spec.isStageLoggingEnabled(), spec.isProcessTimingEnabled(), phaseConnectorDatasets, spec.getNumOfRecordsPreview(), spec.getProperties(), !plan.getConditionPhaseBranches().isEmpty(), spec.getSqlEngineStageSpec());
}
Also used : MultiConnectorSource(io.cdap.cdap.etl.batch.connector.MultiConnectorSource) ConnectorSource(io.cdap.cdap.etl.batch.connector.ConnectorSource) MultiConnectorSource(io.cdap.cdap.etl.batch.connector.MultiConnectorSource) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) BatchPhaseSpec(io.cdap.cdap.etl.batch.BatchPhaseSpec) AlertPublisherSink(io.cdap.cdap.etl.batch.connector.AlertPublisherSink)

Aggregations

BatchPhaseSpec (io.cdap.cdap.etl.batch.BatchPhaseSpec)1 AlertPublisherSink (io.cdap.cdap.etl.batch.connector.AlertPublisherSink)1 ConnectorSource (io.cdap.cdap.etl.batch.connector.ConnectorSource)1 MultiConnectorSource (io.cdap.cdap.etl.batch.connector.MultiConnectorSource)1 StageSpec (io.cdap.cdap.etl.proto.v2.spec.StageSpec)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1