Examples with EHadoopVersion4Drivers org.talend.core.hadoop.version.EHadoopVersion4Drivers used on opensource projects

Example 1 with EHadoopVersion4Drivers

use of org.talend.core.hadoop.version.EHadoopVersion4Drivers in project tbd-studio-se by Talend.

the class AbstractHDFSBrowseController method getHDFSConnectionBean.

protected HDFSConnectionBean getHDFSConnectionBean() {
    INode node = (INode) elem;
    IProcess process = node.getProcess();
    IContext context = null;
    IContextManager cm = process.getContextManager();
    if (cm != null) {
        context = cm.getDefaultContext();
    }
    boolean isMr = false;
    HDFSConnectionBean connectionBean = new HDFSConnectionBean();
    // $NON-NLS-1$
    String useExistingConnection = ElementParameterParser.getValue(elem, "__USE_EXISTING_CONNECTION__");
    if ("true".equalsIgnoreCase(useExistingConnection)) {
        // $NON-NLS-1$
        // $NON-NLS-1$
        String connectionName = ElementParameterParser.getValue(node, "__CONNECTION__");
        List<? extends INode> nodes = node.getProcess().getGeneratingNodes();
        for (INode iNode : nodes) {
            if (iNode.getUniqueName().equals(connectionName)) {
                node = iNode;
                break;
            }
        }
    } else if (node != null && node.getComponent() != null && node.getComponent().getPaletteType() != null && node.getComponent().getPaletteType().equals("MR")) {
        // $NON-NLS-1$
        List<? extends INode> nodes = node.getProcess().getGeneratingNodes();
        for (INode iNode : nodes) {
            IComponent component = iNode.getComponent();
            if (component != null && component.getName() != null && component.getName().equals("tMRConfiguration")) {
                // $NON-NLS-1$
                isMr = true;
                node = iNode;
                if (node instanceof DataNode || node instanceof BigDataNode) {
                    String versionParameter = (String) getParameterValue(node, EHadoopParameter.VERSION.getName());
                    if (versionParameter != null) {
                        connectionBean.setDfVersion(versionParameter);
                    }
                    String nameNodeParameter = getParameterValueWithContext(node, context, EHadoopParameter.NAMENODE_URI.getName());
                    if (nameNodeParameter != null) {
                        connectionBean.setNameNodeURI(nameNodeParameter);
                    }
                }
            }
        }
        String distribution = (String) getParameterValue(node, EHadoopParameter.DISTRIBUTION.getName());
        if (IGoogleDataprocDistribution.DISTRIBUTION_NAME.equals(distribution)) {
            displayUnsupportedOperationDialog();
            return null;
        }
    } else if (node != null && node.getComponent() != null && node.getComponent().getPaletteType() != null && node.getComponent().getPaletteType().startsWith("SPARK")) {
        // $NON-NLS-1$
        // SPARK and SPARKSTREAMING
        boolean browseIsSupported = false;
        // $NON-NLS-1$
        boolean defineStorageConfiguration = (Boolean) getParameterValue(node, "DEFINE_STORAGE_CONFIGURATION");
        if (defineStorageConfiguration) {
            // $NON-NLS-1$
            String configurationComponent = (String) getParameterValue(node, "STORAGE_CONFIGURATION");
            if (configurationComponent.startsWith("tHDFSConfiguration")) {
                // $NON-NLS-1$
                List<? extends INode> nodes = node.getProcess().getGeneratingNodes();
                for (INode iNode : nodes) {
                    if (configurationComponent.equals(iNode.getUniqueName())) {
                        browseIsSupported = true;
                        isMr = true;
                        node = iNode;
                        String versionParameter = (String) getParameterValue(node, EHadoopParameter.VERSION.getName());
                        if (versionParameter != null) {
                            connectionBean.setDfVersion(versionParameter);
                        }
                        String nameNodeParameter = getParameterValueWithContext(node, context, EHadoopParameter.NAMENODE_URI.getName());
                        if (nameNodeParameter != null) {
                            connectionBean.setNameNodeURI(nameNodeParameter);
                        }
                    }
                }
            }
        }
        if (!browseIsSupported) {
            displayUnsupportedOperationDialog();
            return null;
        }
    }
    String distribution = (String) getParameterValue(node, EHadoopParameter.DISTRIBUTION.getName());
    if (!isMr) {
        String version = (String) getParameterValue(node, EHadoopParameter.VERSION.getName());
        String nameNodeUri = getParameterValueWithContext(node, context, EHadoopParameter.NAMENODE_URI.getName());
        connectionBean.setDfVersion(version);
        connectionBean.setNameNodeURI(nameNodeUri);
    }
    // check support the group or not
    EHadoopVersion4Drivers version4Drivers = EHadoopVersion4Drivers.indexOfByVersion(connectionBean.getDfVersion());
    if (version4Drivers != null && version4Drivers.isSupportGroup()) {
        String group = getParameterValueWithContext(node, context, EHadoopParameter.GROUP.getName());
        connectionBean.setGroup(group);
    }
    String userName = getParameterValueWithContext(node, context, EHadoopParameter.USERNAME.getName());
    Boolean useKrb = (Boolean) getParameterValue(node, EHadoopParameter.USE_KRB.getName());
    Boolean useMaprTicket = (Boolean) getParameterValue(node, EHadoopParameter.USE_MAPRTICKET.getName());
    String nnPrincipal = getParameterValueWithContext(node, context, EHadoopParameter.NAMENODE_PRINCIPAL.getName());
    Boolean useKeytab = (Boolean) getParameterValue(node, EHadoopParameter.USE_KEYTAB.getName());
    String ktPrincipal = getParameterValueWithContext(node, context, EHadoopParameter.PRINCIPAL.getName());
    String ktPath = getParameterValueWithContext(node, context, EHadoopParameter.KEYTAB_PATH.getName());
    Boolean isUseCustom = EHadoopDistributions.CUSTOM.getName().equals(distribution);
    if (isUseCustom) {
        Object authMode = getParameterValue(node, EHadoopParameter.AUTHENTICATION_MODE.getName());
        if ("KRB".equals(authMode)) {
            // $NON-NLS-1$
            useKrb = true;
        }
    }
    if (!isMr && node != null && node.getComponent() != null && node.getComponent().getName().equals("tHDFSInput")) {
        if ("true".equalsIgnoreCase(useExistingConnection) || useKrb || (distribution.equals("MAPR") && !useMaprTicket)) {
            userName = "";
        }
    }
    String customJars = (String) getParameterValue(node, EHadoopParameter.HADOOP_CUSTOM_JARS.getName());
    connectionBean.setDistribution(distribution);
    connectionBean.setUserName(userName);
    connectionBean.setEnableKerberos(useKrb != null ? useKrb : false);
    connectionBean.setPrincipal(nnPrincipal);
    connectionBean.setUseKeytab(useKeytab != null ? useKeytab : false);
    connectionBean.setKeytabPrincipal(ktPrincipal);
    connectionBean.setKeytab(ktPath);
    connectionBean.setUseCustomVersion(isUseCustom != null ? isUseCustom : false);
    if (StringUtils.isNotBlank(customJars)) {
        connectionBean.getAdditionalProperties().put(ECustomVersionGroup.COMMON.getName(), customJars);
    }
    if (process instanceof IProcess2) {
        IProcess2 pro = (IProcess2) process;
        connectionBean.setRelativeHadoopClusterId(pro.getProperty().getId());
        IElementParameter propertyParam = node.getElementParameterFromField(EParameterFieldType.PROPERTY_TYPE);
        if (propertyParam != null) {
            IElementParameter repositoryType = propertyParam.getChildParameters().get(EParameterName.PROPERTY_TYPE.getName());
            if (repositoryType != null && EmfComponent.REPOSITORY.equals(repositoryType.getValue())) {
                IHadoopClusterService hadoopClusterService = HadoopRepositoryUtil.getHadoopClusterService();
                String relativeHadoopClusterId = null;
                IElementParameter repositoryId = propertyParam.getChildParameters().get((EParameterName.REPOSITORY_PROPERTY_TYPE.getName()));
                if (hadoopClusterService != null) {
                    Item item = hadoopClusterService.getHadoopClusterBySubitemId(((String) repositoryId.getValue()));
                    if (item != null) {
                        relativeHadoopClusterId = item.getProperty().getId();
                        if (relativeHadoopClusterId != null) {
                            connectionBean.setRelativeHadoopClusterId(relativeHadoopClusterId);
                        }
                    }
                }
            }
        }
    }
    connectionBean.getConfigurations().putAll(getHadoopProperties(node, context));
    return connectionBean;
}

Example 2 with EHadoopVersion4Drivers

use of org.talend.core.hadoop.version.EHadoopVersion4Drivers in project tbd-studio-se by Talend.

the class PushHCatalogIntoHadoopClusterMigrationTask method getClusterName.

/*
     * (non-Javadoc)
     *
     * @see
     * org.talend.repository.hadoopcluster.model.migration.AbstractHadoopClusterMigrationTask#getClusterName(org.talend
     * .repository.model.hadoopcluster.HadoopConnection)
     */
@Override
protected String getClusterName(HadoopSubConnection hadoopSubConnection) {
    StringBuffer cnBuffer = new StringBuffer();
    HCatalogConnection connection = (HCatalogConnection) hadoopSubConnection;
    EHadoopVersion4Drivers version = EHadoopVersion4Drivers.indexOfByVersion(connection.getHcatVersion());
    String hostName = connection.getHostName();
    if (version != null) {
        cnBuffer.append(version.getVersionValue());
        cnBuffer.append(UNDER_LINE);
    }
    if (hostName != null) {
        cnBuffer.append(hostName);
    }
    String clusterName = cnBuffer.toString();
    // $NON-NLS-1$
    clusterName = clusterName.replaceAll("[^a-zA-Z0-9_]", UNDER_LINE);
    return clusterName;
}

Example 3 with EHadoopVersion4Drivers

use of org.talend.core.hadoop.version.EHadoopVersion4Drivers in project tbd-studio-se by Talend.

the class PushHDFSIntoHadoopClusterMigrationTask method getClusterName.

/*
     * (non-Javadoc)
     *
     * @see
     * org.talend.repository.hadoopcluster.model.migration.AbstractHadoopClusterMigrationTask#getClusterName(org.talend
     * .repository.model.hadoopcluster.HadoopConnection)
     */
@Override
protected String getClusterName(HadoopSubConnection hadoopConnection) {
    StringBuffer cnBuffer = new StringBuffer();
    HDFSConnection connection = (HDFSConnection) hadoopConnection;
    EHadoopVersion4Drivers version = EHadoopVersion4Drivers.indexOfByVersion(connection.getDfVersion());
    if (version != null) {
        cnBuffer.append(version.getVersionValue());
        cnBuffer.append(UNDER_LINE);
    }
    String nameNodeURI = connection.getNameNodeURI();
    if (nameNodeURI != null) {
        cnBuffer.append(HadoopParameterUtil.getHostNameFromNameNodeURI(nameNodeURI));
    }
    String clusterName = cnBuffer.toString();
    // $NON-NLS-1$
    clusterName = clusterName.replaceAll("[^a-zA-Z0-9_]", UNDER_LINE);
    return clusterName;
}