Examples with DistributionModuleGroup org.talend.hadoop.distribution.DistributionModuleGroup used on opensource projects

Example 16 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkStreamingS3NodeModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkS3MrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_S3_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkS3MrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkS3MrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkS3MrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.S3_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
        hs.add(dmg);
    }
    return hs;
}

Example 17 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicCDHSparkStreamingKafkaAvroModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> moduleGroups = new HashSet<>();
    Set<DistributionModuleGroup> moduleGroupsFromSuper = super.getModuleGroups(distribution, version);
    if (moduleGroupsFromSuper != null && !moduleGroupsFromSuper.isEmpty()) {
        moduleGroups.addAll(moduleGroupsFromSuper);
    }
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    // Add Spark Streaming Kafka dependencies as well
    moduleGroups.addAll(new DynamicCDHSparkStreamingKafkaAssemblyModuleGroup(pluginAdapter).getModuleGroups(distribution, version));
    return moduleGroups;
}

Example 18 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkStreamingKafkaClientModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkKafkaClientMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_CLIENT_MRREQUIRED_MODULE_GROUP.getModuleName());
    String spark2KafkaClientMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK2_KAFKA_CLIENT_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkKafkaClientMrRequiredRuntimeId);
    checkRuntimeId(spark2KafkaClientMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkKafkaClientMrRequiredRuntimeId)) {
        // Spark 1
        DistributionModuleGroup dmgSpark1 = new DistributionModuleGroup(sparkKafkaClientMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark1Condition)));
        hs.add(dmgSpark1);
    }
    if (StringUtils.isNotBlank(spark2KafkaClientMrRequiredRuntimeId)) {
        // Spark 2
        DistributionModuleGroup dmgSpark2 = new DistributionModuleGroup(spark2KafkaClientMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark2Condition)));
        hs.add(dmgSpark2);
    }
    return hs;
}

Example 19 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicCDPSparkStreamingKafkaAvroModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> moduleGroups = new HashSet<>();
    Set<DistributionModuleGroup> moduleGroupsFromSuper = super.getModuleGroups(distribution, version);
    if (moduleGroupsFromSuper != null && !moduleGroupsFromSuper.isEmpty()) {
        moduleGroups.addAll(moduleGroupsFromSuper);
    }
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    // Add Spark Streaming Kafka dependencies as well
    moduleGroups.addAll(new DynamicCDPSparkStreamingKafkaAssemblyModuleGroup(pluginAdapter).getModuleGroups(distribution, version));
    return moduleGroups;
}

Example 20 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicHDPSparkStreamingKafkaAvroModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> moduleGroups = new HashSet<>();
    Set<DistributionModuleGroup> moduleGroupsFromSuper = super.getModuleGroups(distribution, version);
    if (moduleGroupsFromSuper != null && !moduleGroupsFromSuper.isEmpty()) {
        moduleGroups.addAll(moduleGroupsFromSuper);
    }
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    // Add Spark Streaming Kafka dependencies as well
    moduleGroups.addAll(new DynamicHDPSparkStreamingKafkaAssemblyModuleGroup(pluginAdapter).getModuleGroups(distribution, version));
    return moduleGroups;
}