Examples with HadoopConfigurator org.talend.repository.hadoopcluster.configurator.HadoopConfigurator used on opensource projects

Example 1 with HadoopConfigurator

use of org.talend.repository.hadoopcluster.configurator.HadoopConfigurator in project tbd-studio-se by Talend.

the class ClouderaHadoopConfig512Test method testConfigurator_cdh512.

// FIXME: use Mock of Cloudera API or TestContainers framework (https://www.testcontainers.org/) to be able to test it.
@Ignore("ignored in buildme system, comment out this line when run locally")
@Test
public void testConfigurator_cdh512() throws Exception {
    String folder = "/tmp/cm";
    // 512
    String url = "http://163.172.6.25:7180";
    HadoopConfigurator configurator = new HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.CLOUDERA_MANAGER).withBaseURL(new URL(url)).withUsernamePassword("talendwiz", "83RpC5MWcnZPbQyVaNxF").build();
    TestUtil.checkCluster(configurator, "Cluster 1");
    HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
    Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
    TestUtil.checkService(services, HadoopHostedService.HDFS, HadoopHostedService.YARN, HadoopHostedService.HIVE, HadoopHostedService.HBASE);
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HDFS), "hdfs-site.xml", "core-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.YARN), "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HIVE), "hive-site.xml", "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HBASE), "hbase-site.xml", "hdfs-site.xml", "core-site.xml");
}

Example 2 with HadoopConfigurator

use of org.talend.repository.hadoopcluster.configurator.HadoopConfigurator in project tbd-studio-se by Talend.

the class ClouderaHadoopConfig611Test method testConfigurator_cdh611.

// FIXME: use Mock of Cloudera API or TestContainers framework (https://www.testcontainers.org/) to be able to test it.
@Ignore("ignored in buildme system, comment out this line when run locally")
@Test
public void testConfigurator_cdh611() throws Exception {
    String folder = "/tmp/cm";
    // 611
    String url = "https://tal-qa300.talend.lan:7183";
    String trustStoreFile = "tal-qa300.jks";
    String trustStorePwd = "ROv7PbRuQpB4o3KkyY0FUPfn8NAClUCraPF0APHawb8";
    String trustStoreType = "jks";
    HadoopConfigurator configurator = TestUtil.getConfigurator(url, trustStoreFile, trustStoreType, trustStorePwd);
    TestUtil.checkCluster(configurator, "Cluster 1");
    HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
    Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
    TestUtil.checkService(services, HadoopHostedService.HDFS, HadoopHostedService.YARN, HadoopHostedService.HIVE, HadoopHostedService.HBASE);
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HDFS), "hdfs-site.xml", "core-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.YARN), "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HIVE), "hive-site.xml", "yarn-site.xml", "hdfs-site.xml", "core-site.xml", "mapred-site.xml");
    TestUtil.checkServiceConf(services.get(HadoopHostedService.HBASE), "hbase-site.xml", "hdfs-site.xml", "core-site.xml");
}

Example 3 with HadoopConfigurator

use of org.talend.repository.hadoopcluster.configurator.HadoopConfigurator in project tbd-studio-se by Talend.

the class HadoopImportRemoteOptionPage method propertyChange.

@Override
public void propertyChange(PropertyChangeEvent evt) {
    setErrorMessage(null);
    Exception exception = null;
    String propertyName = evt.getPropertyName();
    Object newValue = evt.getNewValue();
    if (IPropertyConstants.PROPERTY_CONNECT.equals(propertyName)) {
        if (newValue instanceof HadoopConfigurator) {
            configurator = (HadoopConfigurator) newValue;
            try {
                List<String> clusters = configurator.getAllClusters();
                bindCombo(clusters);
            } catch (Exception e) {
                exception = e;
            }
        } else if (newValue instanceof Exception) {
            exception = (Exception) newValue;
        }
    }
    if (IPropertyConstants.PROPERTY_RETRIEVE_JOB.equals(propertyName)) {
        if (newValue instanceof IRetrieveConfsJobService) {
            IRetrieveConfsJobService retrieveConfigJobService = (IRetrieveConfsJobService) newValue;
            List<String> clusters = retrieveConfigJobService.getAllClusterName();
            bindCombo(clusters);
        } else if (newValue instanceof Exception) {
            exception = (Exception) newValue;
        }
    }
    if (exception != null) {
        // $NON-NLS-1$
        setErrorMessage(Messages.getString("HadoopImportRemoteOptionPage.connectionFailed"));
        ExceptionHandler.process(exception);
    }
}

Example 4 with HadoopConfigurator

use of org.talend.repository.hadoopcluster.configurator.HadoopConfigurator in project tbd-studio-se by Talend.

the class TalendHadoopConfiguratorApiDemo method main.

public static void main(String[] args) {
    try {
        String folder = "/tmp/cm";
        // HadoopConfigurator configurator = new HadoopConfiguratorBuilder()
        // .withVendor(HadoopConfigurationManager.CLOUDERA_MANAGER).withBaseURL(new
        // URL("http://192.168.32.35:7180"))
        // .withUsernamePassword("admin", "admin").build();
        HadoopConfigurator configurator = new HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.AMBARI).withBaseURL(new URL("http://192.168.33.12:8080")).withUsernamePassword("admin", "talend").build();
        // HadoopConfigurator configurator = new
        // HadoopConfiguratorBuilder().withVendor(HadoopConfigurationManager.AMBARI)
        // .withBaseURL(new URL("http://192.168.33.74:8080")).withUsernamePassword("admin", "admin").build();
        System.out.println(configurator.getAllClusters());
        HadoopCluster cluster = configurator.getCluster(configurator.getAllClusters().get(0));
        Map<HadoopHostedService, HadoopClusterService> services = cluster.getHostedServices();
        for (HadoopHostedService serviceName : services.keySet()) {
            HadoopClusterService service = services.get(serviceName);
            System.out.println("---------------------------------");
            System.out.println("Service Name:" + serviceName);
            Map<String, String> configuration = service.getConfiguration();
            for (String key : configuration.keySet()) {
                System.out.println(key + ":" + configuration.get(key));
            }
            System.out.println("---------------------------------");
            service.exportConfigurationToXml(folder + "/" + serviceName);
        // service.exportConfigurationToXml(folder);
        }
    } catch (MalformedURLException e) {
        e.printStackTrace();
    }
}

Example 5 with HadoopConfigurator

use of org.talend.repository.hadoopcluster.configurator.HadoopConfigurator in project tbd-studio-se by Talend.

the class HadoopConfsUtils method getHadoopConfigurator.

public static HadoopConfigurator getHadoopConfigurator(HadoopConfigurationManager configurationManager, HadoopConfsConnection confsConnection) throws MalformedURLException {
    HadoopConfiguratorBuilder configuratorBuilder = new HadoopConfiguratorBuilder().withVendor(configurationManager).withBaseURL(new URL(confsConnection.getConnURL())).withUsernamePassword(confsConnection.getUsername(), confsConnection.getPassword());
    if (confsConnection.isUseAuth()) {
        configuratorBuilder = configuratorBuilder.withTrustManagers(confsConnection.getTrustStoreFile(), confsConnection.getTrustStoreType(), confsConnection.getTrustStorePassword());
    }
    HadoopConfigurator configurator = configuratorBuilder.build();
    return configurator;
}