Examples with HadoopCluster org.talend.repository.hadoopcluster.configurator.HadoopCluster used on opensource projects

Example 1 with HadoopCluster

use of org.talend.repository.hadoopcluster.configurator.HadoopCluster in project tbd-studio-se by Talend.

the class ClouderaHadoopConfig512Test method testConfigurator_cdh512.

// FIXME: use Mock of Cloudera API or TestContainers framework (https://www.testcontainers.org/) to be able to test it.
@Ignore("ignored in buildme system, comment out this line when run locally")
@Test
public void testConfigurator_cdh512() throws Exception {
    String folder = "/tmp/cm";
    // 512
    String url = "http://163.172.6.25:7180";
    HadoopConfigurator configurator = new HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.CLOUDERA_MANAGER).withBaseURL(new URL(url)).withUsernamePassword("talendwiz", "83RpC5MWcnZPbQyVaNxF").build();
    TestUtil.checkCluster(configurator, "Cluster 1");
    HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
    Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
    TestUtil.checkService(services, HadoopHostedService.HDFS, HadoopHostedService.YARN, HadoopHostedService.HIVE, HadoopHostedService.HBASE);
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HDFS), "hdfs-site.xml", "core-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.YARN), "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HIVE), "hive-site.xml", "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HBASE), "hbase-site.xml", "hdfs-site.xml", "core-site.xml");
}

Example 2 with HadoopCluster

use of org.talend.repository.hadoopcluster.configurator.HadoopCluster in project tbd-studio-se by Talend.

the class ClouderaHadoopConfig611Test method testConfigurator_cdh611.

// FIXME: use Mock of Cloudera API or TestContainers framework (https://www.testcontainers.org/) to be able to test it.
@Ignore("ignored in buildme system, comment out this line when run locally")
@Test
public void testConfigurator_cdh611() throws Exception {
    String folder = "/tmp/cm";
    // 611
    String url = "https://tal-qa300.talend.lan:7183";
    String trustStoreFile = "tal-qa300.jks";
    String trustStorePwd = "ROv7PbRuQpB4o3KkyY0FUPfn8NAClUCraPF0APHawb8";
    String trustStoreType = "jks";
    HadoopConfigurator configurator = TestUtil.getConfigurator(url, trustStoreFile, trustStoreType, trustStorePwd);
    TestUtil.checkCluster(configurator, "Cluster 1");
    HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
    Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
    TestUtil.checkService(services, HadoopHostedService.HDFS, HadoopHostedService.YARN, HadoopHostedService.HIVE, HadoopHostedService.HBASE);
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HDFS), "hdfs-site.xml", "core-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.YARN), "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HIVE), "hive-site.xml", "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HBASE), "hbase-site.xml", "hdfs-site.xml", "core-site.xml");
}

Example 3 with HadoopCluster

use of org.talend.repository.hadoopcluster.configurator.HadoopCluster in project tbd-studio-se by Talend.

the class HadoopImportRemoteOptionPage method addListener.

private void addListener() {
    selectClusterButton.addSelectionListener(new SelectionAdapter() {

        @Override
        public void widgetSelected(SelectionEvent e) {
            try {
                IRetrieveConfsJobService retrieveJobServer = connectionForm.getRetrieveJobService();
                if (retrieveJobServer != null) {
                    retrieveJobServer.setDistributionVersion(distriVersion);
                    retrieveJobServer.setSelectedCluster(getSelectedCluster());
                    confsService = (IRetrieveConfsService) retrieveJobServer;
                } else if (configurator != null) {
                    HadoopCluster cluster = configurator.getCluster(getSelectedCluster());
                    confsService = new RetrieveRemoteConfsService(cluster);
                }
                if (confsService != null) {
                    servicesTableComp.setServices(confsService.getAllServices());
                }
            } catch (Exception ex) {
                ExceptionHandler.process(ex);
            }
        }
    });
}

Example 4 with HadoopCluster

use of org.talend.repository.hadoopcluster.configurator.HadoopCluster in project tbd-studio-se by Talend.

the class TalendHadoopConfiguratorApiDemo method main.

public static void main(String[] args) {
    try {
        String folder = "/tmp/cm";
        // HadoopConfigurator configurator = new HadoopConfiguratorBuilder()
        // .withVendor(HadoopConfigurationManager.CLOUDERA_MANAGER).withBaseURL(new
        // URL("http://192.168.32.35:7180"))
        // .withUsernamePassword("admin", "admin").build();
        HadoopConfigurator configurator = new HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.AMBARI).withBaseURL(new URL("http://192.168.33.12:8080")).withUsernamePassword("admin", "talend").build();
        // HadoopConfigurator configurator = new
        // HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.AMBARI)
        // .withBaseURL(new URL("http://192.168.33.74:8080")).withUsernamePassword("admin", "admin").build();
        System.out.println(configurator.getAllClusters());
        HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
        Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
        for (HadoopHostedService serviceName : services.keySet()) {
            HadoopClusterService service = services.get(serviceName);
            System.out.println("---------------------------------");
            System.out.println("Service Name:" + serviceName);
            Map<String, String> configuration = service.getConfiguration();
            for (String key : configuration.keySet()) {
                System.out.println(key + ":" + configuration.get(key));
            }
            System.out.println("---------------------------------");
            service.exportConfigurationToXml(folder + "/" + serviceName);
        // service.exportConfigurationToXml(folder);
        }
    } catch (MalformedURLException e) {
        e.printStackTrace();
    }
}

Example 5 with HadoopCluster

use of org.talend.repository.hadoopcluster.configurator.HadoopCluster in project tbd-studio-se by Talend.

the class ClouderaHadoopConfig513Test method testConfigurator_cdh513.

/**
 * there is a cluster at Bonn
 *
 * 192.168.150.98 tal-qa143.talend.lan tal-qa143
 *
 * 192.168.150.99 tal-qa144.talend.lan tal-qa144
 *
 * 192.168.150.77 tal-qa146.talend.lan tal-qa146
 *
 * 192.168.150.78 tal-qa147.talend.lan tal-qa147
 */
// FIXME: use Mock of Cloudera API or TestContainers framework (https://www.testcontainers.org/) to be able to test it.
@Ignore("ignored in buildme system, comment out this line when run locally")
@Test
public void testConfigurator_cdh513() throws Exception {
    String folder = "/tmp/cm";
    // 513
    String url = "https://tal-qa14.talend.lan:7183";
    String trustStoreFile = "cdhcn.truststore";
    String trustStorePwd = "talend";
    String trustStoreType = "jks";
    HadoopConfigurator configurator = TestUtil.getConfigurator(url, trustStoreFile, trustStoreType, trustStorePwd);
    TestUtil.checkCluster(configurator, "Cluster 1");
    HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
    Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
    TestUtil.checkService(services, HadoopHostedService.HDFS, HadoopHostedService.YARN, HadoopHostedService.HIVE, HadoopHostedService.HBASE);
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HDFS), "hdfs-site.xml", "core-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.YARN), "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HIVE), "hive-site.xml", "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HBASE), "hbase-site.xml", "hdfs-site.xml", "core-site.xml");
}