Search in sources :

Example 76 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicHDPSparkStreamingKafkaAssemblyModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String spark2KafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(spark2KafkaAssemblyMrRequiredRuntimeId);
    DistributionModuleGroup dmgSpark21 = new DistributionModuleGroup(spark2KafkaAssemblyMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
    hs.add(dmgSpark21);
    return hs;
}
Also used : DynamicPluginAdapter(org.talend.hadoop.distribution.dynamic.adapter.DynamicPluginAdapter) SparkStreamingLinkedNodeCondition(org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition) DistributionModuleGroup(org.talend.hadoop.distribution.DistributionModuleGroup) HashSet(java.util.HashSet)

Example 77 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class SqoopModuleGroup method getModuleGroups.

public static Set<DistributionModuleGroup> getModuleGroups(String distributionVersion) {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    hs.add(new DistributionModuleGroup(ModuleGroupName.SQOOP.get(distributionVersion)));
    hs.add(new DistributionModuleGroup(ModuleGroupName.SQOOP_PARQUET.get(distributionVersion), true, parquetOutputCondition));
    hs.add(new DistributionModuleGroup(ModuleGroupName.SQOOP_AVRO.get(distributionVersion), true, avroOutputCondition));
    hs.add(new DistributionModuleGroup(ModuleGroupName.SQOOP_SEQUENCE.get(distributionVersion), true, sequenceOutputCondition));
    hs.add(new DistributionModuleGroup(ModuleGroupName.SQOOP_TEXT.get(distributionVersion), true, textOutputCondition));
    return hs;
}
Also used : DistributionModuleGroup(org.talend.hadoop.distribution.DistributionModuleGroup) HashSet(java.util.HashSet)

Example 78 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkBatchAzureNodeModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkAzureMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_AZURE_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkAzureMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkAzureMrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkAzureMrRequiredRuntimeId, true, new SparkBatchLinkedNodeCondition(distribution, version, SparkBatchConstant.SPARK_BATCH_AZURE_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
        hs.add(dmg);
    }
    return hs;
}
Also used : DynamicPluginAdapter(org.talend.hadoop.distribution.dynamic.adapter.DynamicPluginAdapter) SparkBatchLinkedNodeCondition(org.talend.hadoop.distribution.condition.common.SparkBatchLinkedNodeCondition) DistributionModuleGroup(org.talend.hadoop.distribution.DistributionModuleGroup) HashSet(java.util.HashSet)

Example 79 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkBatchParquetNodeModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkParquetRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_PARQUET_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkParquetRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkParquetRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkParquetRequiredRuntimeId, true, new SparkBatchLinkedNodeCondition(distribution, version).getCondition());
        hs.add(dmg);
    }
    return hs;
}
Also used : DynamicPluginAdapter(org.talend.hadoop.distribution.dynamic.adapter.DynamicPluginAdapter) SparkBatchLinkedNodeCondition(org.talend.hadoop.distribution.condition.common.SparkBatchLinkedNodeCondition) DistributionModuleGroup(org.talend.hadoop.distribution.DistributionModuleGroup) HashSet(java.util.HashSet)

Example 80 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkStreamingKafkaAvroModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkKafkaAvroMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_AVRO_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkKafkaAvroMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkKafkaAvroMrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkKafkaAvroMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
        hs.add(dmg);
    }
    return hs;
}
Also used : DynamicPluginAdapter(org.talend.hadoop.distribution.dynamic.adapter.DynamicPluginAdapter) SparkStreamingLinkedNodeCondition(org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition) DistributionModuleGroup(org.talend.hadoop.distribution.DistributionModuleGroup) HashSet(java.util.HashSet)

Aggregations

DistributionModuleGroup (org.talend.hadoop.distribution.DistributionModuleGroup)148 HashSet (java.util.HashSet)106 DynamicPluginAdapter (org.talend.hadoop.distribution.dynamic.adapter.DynamicPluginAdapter)43 HashMap (java.util.HashMap)36 Test (org.junit.Test)36 SparkStreamingLinkedNodeCondition (org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition)20 SparkBatchLinkedNodeCondition (org.talend.hadoop.distribution.condition.common.SparkBatchLinkedNodeCondition)14 MultiComponentCondition (org.talend.hadoop.distribution.condition.MultiComponentCondition)13 ComponentCondition (org.talend.hadoop.distribution.condition.ComponentCondition)12 SimpleComponentCondition (org.talend.hadoop.distribution.condition.SimpleComponentCondition)12 NodeComponentTypeBean (org.talend.hadoop.distribution.NodeComponentTypeBean)11 BasicExpression (org.talend.hadoop.distribution.condition.BasicExpression)11 Set (java.util.Set)6 NestedComponentCondition (org.talend.hadoop.distribution.condition.NestedComponentCondition)4 HDFSLinkedNodeCondition (org.talend.hadoop.distribution.condition.common.HDFSLinkedNodeCondition)3 Map (java.util.Map)2 ComponentType (org.talend.hadoop.distribution.ComponentType)2 RawExpression (org.talend.hadoop.distribution.condition.RawExpression)2 ShowExpression (org.talend.hadoop.distribution.condition.ShowExpression)2 DynamicSparkBatchKuduNodeModuleGroup (org.talend.hadoop.distribution.dynamic.template.modulegroup.node.sparkbatch.DynamicSparkBatchKuduNodeModuleGroup)2