Examples with HadoopClusterConnection org.talend.repository.model.hadoopcluster.HadoopClusterConnection used on opensource projects

Example 6 with HadoopClusterConnection

use of org.talend.repository.model.hadoopcluster.HadoopClusterConnection in project tbd-studio-se by Talend.

the class HadoopClusterServiceTest method testUseCustomConfsJarIfNeeded.

@Test
public void testUseCustomConfsJarIfNeeded() throws PersistenceException {
    List<ModuleNeeded> modulesNeeded = new ArrayList<>();
    HadoopClusterConnection hadoopClusterConnection = (HadoopClusterConnection) hadoopClusterItem.getConnection();
    String clusterId = hadoopClusterItem.getProperty().getId();
    String clusterLabel = hadoopClusterItem.getProperty().getLabel();
    service.useCustomConfsJarIfNeeded(modulesNeeded, clusterId);
    // useCustomConfs is false
    assertEquals(0, modulesNeeded.size());
    // useCustomConfs is true and not context mode
    hadoopClusterConnection.setUseCustomConfs(true);
    service.useCustomConfsJarIfNeeded(modulesNeeded, clusterId);
    assertEquals(1, modulesNeeded.size());
    String expectConfsJarName = HadoopParameterUtil.getConfsJarDefaultName(clusterLabel);
    assertEquals(expectConfsJarName, modulesNeeded.get(0).getModuleName());
    assertFalse(Boolean.valueOf(String.valueOf(modulesNeeded.get(0).getExtraAttributes().get(HadoopConstants.IS_DYNAMIC_JAR))));
    // useCustomConfs is true and context mode
    List<ContextType> contexts = new ArrayList<>();
    // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    List<String> contextNames = Arrays.asList("TEST", "DEV", "PROD");
    for (String contextName : contextNames) {
        contexts.add(ClusterTestUtil.createContextType(contextName));
    }
    ContextItem contextItem = ClusterTestUtil.createContextItem(clusterLabel, contexts, true);
    hadoopClusterConnection.setContextMode(true);
    hadoopClusterConnection.setContextId(contextItem.getProperty().getId());
    modulesNeeded.clear();
    service.useCustomConfsJarIfNeeded(modulesNeeded, clusterId);
    assertEquals(3, modulesNeeded.size());
    List<String> moduleNames = new ArrayList<>();
    for (ModuleNeeded module : modulesNeeded) {
        moduleNames.add(module.getModuleName());
        assertTrue(Boolean.valueOf(String.valueOf(modulesNeeded.get(0).getExtraAttributes().get(HadoopConstants.IS_DYNAMIC_JAR))));
    }
    for (String contextName : contextNames) {
        // $NON-NLS-1$
        assertTrue(moduleNames.contains(HadoopParameterUtil.getConfsJarDefaultName(clusterLabel + "_" + contextName)));
    }
    ClusterTestUtil.deleteItem(contextItem);
}

Example 7 with HadoopClusterConnection

use of org.talend.repository.model.hadoopcluster.HadoopClusterConnection in project tbd-studio-se by Talend.

the class HadoopClusterServiceTest method testGetCustomConfsJarName.

@Test
public void testGetCustomConfsJarName() throws PersistenceException {
    // test getCustomConfsJarName() not using context group set in connection but default group in repository context.
    HadoopClusterConnection hadoopClusterConnection = (HadoopClusterConnection) hadoopClusterItem.getConnection();
    hadoopClusterConnection.setUseCustomConfs(true);
    String clusterLabel = hadoopClusterItem.getProperty().getLabel();
    List<ContextType> contexts = new ArrayList<>();
    // $NON-NLS-1$ //$NON-NLS-2$
    List<String> contextNames = Arrays.asList("TEST", "DEV");
    for (String contextName : contextNames) {
        contexts.add(ClusterTestUtil.createContextType(contextName));
    }
    ContextItem contextItem = ClusterTestUtil.createContextItem(clusterLabel, contexts, true);
    // $NON-NLS-1$
    contextItem.setDefaultContext("TEST");
    hadoopClusterConnection.setContextMode(true);
    hadoopClusterConnection.setContextId(contextItem.getProperty().getId());
    // $NON-NLS-1$
    hadoopClusterConnection.setContextName("DEV");
    String jarName = service.getCustomConfsJar(hadoopClusterItem, false, true).map(b -> b.getCustomConfJarName()).orElse(null);
    // $NON-NLS-1$
    assertTrue(jarName.contains(clusterLabel + "_" + contextItem.getDefaultContext()));
    ClusterTestUtil.deleteItem(contextItem);
}

Example 8 with HadoopClusterConnection

use of org.talend.repository.model.hadoopcluster.HadoopClusterConnection in project tbd-studio-se by Talend.

the class SynapseRepositoryUtilTest method testFillDefaultValuesOfHadoopCluster.

@Test
public void testFillDefaultValuesOfHadoopCluster() {
    HadoopClusterConnectionItem synapseItem = ClusterTestUtil.createDefaultHadoopClusterItem();
    HadoopClusterConnection synapseConnection = (HadoopClusterConnection) synapseItem.getConnection();
    synapseConnection.setDistribution(ISynapseDistribution.DISTRIBUTION_NAME);
    synapseConnection.setDfVersion("AZURE_SYNAPSE");
    HCRepositoryUtil.fillDefaultValuesOfHadoopCluster(synapseConnection);
    assertEquals("https://xxxx.dev.azuresynapse.net", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_HOST));
    assertEquals("sparkpools", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_SPARK_POOLS));
    assertEquals("https://xxxx.dfs.core.windows.net", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_FS_HOSTNAME));
    assertEquals("container", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_FS_CONTAINER));
    assertEquals("user", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_FS_USERNAME));
    assertEquals("/tmp", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_SYNAPSE_DEPLOY_BLOB));
    assertEquals("2g", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_DRIVER_MEMORY));
    assertEquals("1", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_DRIVER_CORES));
    assertEquals("2g", synapseConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_EXECUTOR_MEMORY));
}

Example 9 with HadoopClusterConnection

use of org.talend.repository.model.hadoopcluster.HadoopClusterConnection in project tbd-studio-se by Talend.

the class ChangeRepositoryValueFromCdhtoEmrTest method testExecute_cdhToEmr.

// test for TUP-17721 xwen
@Test
public void testExecute_cdhToEmr() {
    Property propertyEMR = PropertiesFactory.eINSTANCE.createProperty();
    IProcess2 processEMR = new Process(propertyEMR);
    IComponent sourceComEMR = ComponentsFactoryProvider.getInstance().get("tSparkConfiguration", ComponentCategory.CATEGORY_4_SPARK.getName());
    HadoopClusterConnection hadoopClusterConnection2 = HadoopClusterFactory.eINSTANCE.createHadoopClusterConnection();
    hadoopClusterConnection2.setAuthMode("USERNAME");
    hadoopClusterConnection2.setDfVersion("EMR_5_0_0");
    hadoopClusterConnection2.setDistribution("AMAZON_EMR");
    hadoopClusterConnection2.setLabel("emr");
    hadoopClusterConnection2.setEnableKerberos(false);
    INode newElemEMR = new Node(sourceComEMR, processEMR);
    newElemEMR.getElementParameter("USE_KRB").setValue(new Boolean(false));
    ChangeValuesFromRepository changeValuesFromCDHtoEMR = new ChangeValuesFromRepository(newElemEMR, hadoopClusterConnection2, "MR_PROPERTY:REPOSITORY_PROPERTY_TYPE", "_y7dnkGIREeegQqu8SjIMUw");
    changeValuesFromCDHtoEMR.execute();
    assertEquals(newElemEMR.getElementParameter("USE_KRB").getValue(), false);
}

Example 10 with HadoopClusterConnection

use of org.talend.repository.model.hadoopcluster.HadoopClusterConnection in project tbd-studio-se by Talend.

the class HCRepositoryUtilTest method testFillDefaultValuesOfHadoopCluster.

@Test
public void testFillDefaultValuesOfHadoopCluster() {
    HadoopClusterConnectionItem hcItem = ClusterTestUtil.createDefaultHadoopClusterItem();
    HadoopClusterConnection hcConnection = (HadoopClusterConnection) hcItem.getConnection();
    hcConnection.setDistribution(IMicrosoftHDInsightDistribution.DISTRIBUTION_NAME);
    hcConnection.setDfVersion("MICROSOFT_HD_INSIGHT_3_4");
    HCRepositoryUtil.fillDefaultValuesOfHadoopCluster(hcConnection);
    assertEquals("clustername.azurehdinsight.net", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_WEB_HCAT_HOSTNAME));
    assertEquals("443", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_WEB_HCAT_PORT));
    assertEquals("admin", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_WEB_HCAT_USERNAME));
    assertEquals("admin", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_HDI_USERNAME));
    assertEquals("storage.blob.core.windows.net", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_AZURE_HOSTNAME));
    assertEquals("clustername", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_AZURE_CONTAINER));
    assertEquals("storage", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_AZURE_USERNAME));
    assertEquals("D:\\blobdir", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_AZURE_DEPLOY_BLOB));
    assertEquals("D:\\hdJobResult", hcConnection.getParameters().get(ConnParameterKeys.CONN_PARA_KEY_WEB_HCAT_JOB_RESULT_FOLDER));
}