Examples with DistributionModuleGroup org.talend.hadoop.distribution.DistributionModuleGroup used on opensource projects

Example 81 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicHDPGraphFramesNodeModuleGroup method getModuleGroups.

public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version, String condition) throws Exception {
    Set<DistributionModuleGroup> dmg = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    // $NON-NLS-1$
    ComponentCondition spark1Condition = getComponentCondition(ESparkVersion.SPARK_1_6.getSparkVersion());
    // $NON-NLS-1$
    ComponentCondition spark2Condition = getComponentCondition(ESparkVersion.SPARK_2_1.getSparkVersion());
    if (condition != null) {
        ComponentCondition c = new SimpleComponentCondition(new RawExpression(condition));
        spark1Condition = new MultiComponentCondition(spark1Condition, BooleanOperator.AND, c);
        spark2Condition = new MultiComponentCondition(spark2Condition, BooleanOperator.AND, c);
    }
    String graphFramesMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.GRAPHFRAMES_MRREQUIRED_MODULE_GROUP.getModuleName());
    String graphFramesSpark2MrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.GRAPHFRAMES_SPARK2_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(graphFramesMrRequiredRuntimeId);
    checkRuntimeId(graphFramesSpark2MrRequiredRuntimeId);
    if (StringUtils.isNotBlank(graphFramesMrRequiredRuntimeId)) {
        dmg.addAll(ModuleGroupsUtils.getModuleGroups(distribution, version, spark1Condition, graphFramesMrRequiredRuntimeId, true));
    }
    if (StringUtils.isNotBlank(graphFramesSpark2MrRequiredRuntimeId)) {
        dmg.addAll(ModuleGroupsUtils.getModuleGroups(distribution, version, spark2Condition, graphFramesSpark2MrRequiredRuntimeId, true));
    }
    return dmg;
}

Example 82 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkStreamingKafkaAssemblyModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkKafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    String spark2KafkaAssemblyMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK2_KAFKA_ASSEMBLY_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkKafkaAssemblyMrRequiredRuntimeId);
    checkRuntimeId(spark2KafkaAssemblyMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkKafkaAssemblyMrRequiredRuntimeId)) {
        // Spark 1.6 Kafka assembly
        DistributionModuleGroup dmgSpark16 = new DistributionModuleGroup(sparkKafkaAssemblyMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark1Condition)));
        hs.add(dmgSpark16);
    }
    if (StringUtils.isNotBlank(spark2KafkaAssemblyMrRequiredRuntimeId)) {
        // Spark 2.1 Kafka assembly
        DistributionModuleGroup dmgSpark21 = new DistributionModuleGroup(spark2KafkaAssemblyMrRequiredRuntimeId, true, new NestedComponentCondition(new MultiComponentCondition(new SparkStreamingLinkedNodeCondition(distribution, version, SparkStreamingConstant.KAFKA_SPARKCONFIGURATION_LINKEDPARAMETER).getCondition(), BooleanOperator.AND, spark2Condition)));
        hs.add(dmgSpark21);
    }
    return hs;
}

Example 83 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicSparkStreamingParquetNodeModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups(String distribution, String version) throws Exception {
    Set<DistributionModuleGroup> hs = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String sparkParquetMrRequiredRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_PARQUET_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(sparkParquetMrRequiredRuntimeId);
    if (StringUtils.isNotBlank(sparkParquetMrRequiredRuntimeId)) {
        DistributionModuleGroup dmg = new DistributionModuleGroup(sparkParquetMrRequiredRuntimeId, true, new SparkStreamingLinkedNodeCondition(distribution, version).getCondition());
        hs.add(dmg);
    }
    return hs;
}

Example 84 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicCDPSparkStreamingModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups() throws Exception {
    Set<DistributionModuleGroup> moduleGroups = new HashSet<>();
    Set<DistributionModuleGroup> moduleGroupsFromSuper = super.getModuleGroups();
    if (moduleGroupsFromSuper != null && !moduleGroupsFromSuper.isEmpty()) {
        moduleGroups.addAll(moduleGroupsFromSuper);
    }
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String hdfsSpark2RuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.HDFS_MODULE_GROUP_SPARK2_1.getModuleName());
    String hdfsCommonRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.HDFS_MODULE_GROUP_COMMON.getModuleName());
    String mrRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.MAPREDUCE_MODULE_GROUP.getModuleName());
    checkRuntimeId(hdfsSpark2RuntimeId);
    checkRuntimeId(hdfsCommonRuntimeId);
    checkRuntimeId(mrRuntimeId);
    if (StringUtils.isNotBlank(hdfsSpark2RuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(hdfsSpark2RuntimeId, false, spark2Condition));
    }
    if (StringUtils.isNotBlank(hdfsCommonRuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(hdfsCommonRuntimeId, false, spark2Condition));
    }
    if (StringUtils.isNotBlank(mrRuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(mrRuntimeId, false, spark2Condition));
    }
    return moduleGroups;
}

Example 85 with DistributionModuleGroup

use of org.talend.hadoop.distribution.DistributionModuleGroup in project tbd-studio-se by Talend.

the class DynamicHDPHiveOnSparkModuleGroup method getModuleGroups.

@Override
public Set<DistributionModuleGroup> getModuleGroups() throws Exception {
    Set<DistributionModuleGroup> moduleGroups = new HashSet<>();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    String tezRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.TEZ_MODULE_GROUP.getModuleName());
    String sparkHiveRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK_HIVE_MRREQUIRED_MODULE_GROUP.getModuleName());
    String spark2HiveRuntimeId = pluginAdapter.getRuntimeModuleGroupIdByTemplateId(DynamicModuleGroupConstant.SPARK2_HIVE_MRREQUIRED_MODULE_GROUP.getModuleName());
    checkRuntimeId(tezRuntimeId);
    checkRuntimeId(sparkHiveRuntimeId);
    checkRuntimeId(spark2HiveRuntimeId);
    if (StringUtils.isNotBlank(tezRuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(tezRuntimeId, false));
    }
    if (StringUtils.isNotBlank(sparkHiveRuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(sparkHiveRuntimeId, true, spark1Condition));
    }
    if (StringUtils.isNotBlank(spark2HiveRuntimeId)) {
        moduleGroups.add(new DistributionModuleGroup(spark2HiveRuntimeId, true, spark2Condition));
    }
    return moduleGroups;
}