Examples with SparkStreamingLinkedNodeCondition org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition used on opensource projects

Example 11 with SparkStreamingLinkedNodeCondition

use of org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition in project tbd-studio-se by Talend.

the class DynamicHDPSparkStreamingKafkaAssemblyModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String spark2KafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(spark2KafkaAssemblyMrRequiredRuntimeId);
    DistributionModuleGroup dmgSpark21 = new DistributionModuleGroup(spark2KafkaAssemblyMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
    hs.add(dmgSpark21);
    return hs;
}

Example 12 with SparkStreamingLinkedNodeCondition

use of org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition in project tbd-studio-se by Talend.

the class DynamicSparkStreamingKafkaAvroModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkKafkaAvroMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_AVRO_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkKafkaAvroMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkKafkaAvroMrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkKafkaAvroMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition());
        hs.add(dmg);
    }
    return hs;
}

Example 13 with SparkStreamingLinkedNodeCondition

use of org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition in project tbd-studio-se by Talend.

the class DynamicSparkStreamingKafkaAssemblyModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkKafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    String spark2KafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK2_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkKafkaAssemblyMrRequiredRuntimeId);
    checkRuntimeId(spark2KafkaAssemblyMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkKafkaAssemblyMrRequiredRuntimeId)) {
        // Spark 1.6 Kafka assembly
        DistributionModuleGroup dmgSpark16 = new DistributionModuleGroup(sparkKafkaAssemblyMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark1Condition)));
        hs.add(dmgSpark16);
    }
    if (StringUtils.isNotBlank(spark2KafkaAssemblyMrRequiredRuntimeId)) {
        // Spark 2.1 Kafka assembly
        DistributionModuleGroup dmgSpark21 = new DistributionModuleGroup(spark2KafkaAssemblyMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark2Condition)));
        hs.add(dmgSpark21);
    }
    return hs;
}

Example 14 with SparkStreamingLinkedNodeCondition

use of org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition in project tbd-studio-se by Talend.

the class DynamicSparkStreamingParquetNodeModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkParquetMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_PARQUET_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkParquetMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkParquetMrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkParquetMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version).getCondition());
        hs.add(dmg);
    }
    return hs;
}

Example 15 with SparkStreamingLinkedNodeCondition

use of org.talend.hadoop.distribution.condition.common.SparkStreamingLinkedNodeCondition in project tbd-studio-se by Talend.

the class ModuleGroupsUtils method getStreamingModuleGroups.

/**
 * Utility method to create the collection of {@link DistributionModuleGroup} with a condition made of a
 * {@link SparkStreamingLinkedNodeCondition} and an additional raw condition
 *
 * @param distribution the distribution key
 * @param version the version key
 * @param condition a nullable additional condition
 * @param moduleName the module name
 * @param mrRequired if the module group is mrRequired
 * @return a set of {@link DistributionModuleGroup}
 */
public static Set<DistributionModuleGroup> getStreamingModuleGroups(String distribution, String version, ComponentCondition condition, String moduleName, boolean mrRequired) {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    ComponentCondition distribCondition = new SparkStreamingLinkedNodeCondition(distribution, version).getCondition();
    ComponentCondition cc = null;
    if (condition != null) {
        cc = new MultiComponentCondition(condition, BooleanOperator.AND, distribCondition);
    } else {
        cc = distribCondition;
    }
    DistributionModuleGroup dmg = new DistributionModuleGroup(moduleName, mrRequired, cc);
    hs.add(dmg);
    return hs;
}