Examples with SparkStreamingComponent org.talend.hadoop.distribution.component.SparkStreamingComponent used on opensource projects

Example 6 with SparkStreamingComponent

use of org.talend.hadoop.distribution.component.SparkStreamingComponent in project tbd-studio-se by Talend.

the class AbstractDynamicCDPDistributionTemplate method buildModuleGroupsTemplateMap.

@Override
protected Map<ComponentType, IDynamicModuleGroupTemplate> buildModuleGroupsTemplateMap() {
    Map<ComponentType, IDynamicModuleGroupTemplate> groupTemplateMap = super.buildModuleGroupsTemplateMap();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    if (this instanceof HiveOnSparkComponent) {
        groupTemplateMap.put(ComponentType.HIVEONSPARK, new DynamicCDPHiveOnSparkModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SparkBatchComponent) {
        groupTemplateMap.put(ComponentType.SPARKBATCH, new DynamicCDPSparkBatchModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SparkStreamingComponent) {
        groupTemplateMap.put(ComponentType.SPARKSTREAMING, new DynamicCDPSparkStreamingModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SqoopComponent) {
        groupTemplateMap.put(ComponentType.SQOOP, new DynamicCDPSqoopModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof ImpalaComponent) {
        groupTemplateMap.put(ComponentType.IMPALA, new DynamicImpalaModuleGroupTemplate(pluginAdapter));
    }
    return groupTemplateMap;
}

Example 7 with SparkStreamingComponent

use of org.talend.hadoop.distribution.component.SparkStreamingComponent in project tbd-studio-se by Talend.

the class AbstractDynamicHDPDistributionTemplate method buildModuleGroupsTemplateMap.

@Override
protected Map<ComponentType, IDynamicModuleGroupTemplate> buildModuleGroupsTemplateMap() {
    Map<ComponentType, IDynamicModuleGroupTemplate> groupTemplateMap = super.buildModuleGroupsTemplateMap();
    DynamicPluginAdapter pluginAdapter = getPluginAdapter();
    if (this instanceof HDFSComponent) {
        groupTemplateMap.put(ComponentType.HDFS, new DynamicHDPHDFSModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof MRComponent) {
        groupTemplateMap.put(ComponentType.MAPREDUCE, new DynamicHDPMapReduceModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SparkBatchComponent) {
        groupTemplateMap.put(ComponentType.SPARKBATCH, new DynamicHDPSparkBatchModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SparkStreamingComponent) {
        groupTemplateMap.put(ComponentType.SPARKSTREAMING, new DynamicHDPSparkStreamingModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof HiveOnSparkComponent) {
        groupTemplateMap.put(ComponentType.HIVEONSPARK, new DynamicHDPHiveOnSparkModuleGroupTemplate(pluginAdapter));
    }
    if (this instanceof SqoopComponent) {
        groupTemplateMap.put(ComponentType.SQOOP, new DynamicHDPSqoopModuleGroupTemplate(pluginAdapter));
    }
    return groupTemplateMap;
}

Example 8 with SparkStreamingComponent

use of org.talend.hadoop.distribution.component.SparkStreamingComponent in project tbd-studio-se by Talend.

the class DistributionHelperTest method test_doSupportService_SparkComponent.

@Test
public void test_doSupportService_SparkComponent() {
    HadoopComponent hadoopComponent = Mockito.mock(HadoopComponent.class);
    DistributionBean distribution = Mockito.mock(DistributionBean.class);
    Assert.assertFalse(DistributionHelper.doSupportService(new DistributionVersion(hadoopComponent, distribution, "ABC_10", "Abc 1.0"), IHDConstants.SERVICE_SPARK));
    SparkComponent sparkComponent = Mockito.mock(SparkComponent.class);
    Assert.assertTrue(DistributionHelper.doSupportService(new DistributionVersion(sparkComponent, distribution, "ABC_10", "Abc 1.0"), IHDConstants.SERVICE_SPARK));
    // because the SparkStreamingComponent is sub class of SparkComponent
    SparkStreamingComponent sparkStreamingComponent = Mockito.mock(SparkStreamingComponent.class);
    Assert.assertTrue(DistributionHelper.doSupportService(new DistributionVersion(sparkStreamingComponent, distribution, "ABC_10", "Abc 1.0"), IHDConstants.SERVICE_SPARK));
}