Examples with HDFSConnection org.talend.repository.model.hdfs.HDFSConnection used on opensource projects

Example 6 with HDFSConnection

use of org.talend.repository.model.hdfs.HDFSConnection in project tbd-studio-se by Talend.

the class ExtractHDFSSchemaManager method extractColumns.

public List<MetadataColumn> extractColumns(HDFSConnection connection, ClassLoader classLoader, MetadataTable metadataTable) throws Exception {
    List<MetadataColumn> columns = new ArrayList<MetadataColumn>();
    if (connection == null || metadataTable == null) {
        return columns;
    }
    EMap<String, String> additionalProperties = metadataTable.getAdditionalProperties();
    String hdfsPath = additionalProperties.get(HDFSConstants.HDFS_PATH);
    if (StringUtils.isEmpty(hdfsPath)) {
        return columns;
    }
    HDFSConnectionBean connectionBean = HDFSModelUtil.convert2HDFSConnectionBean(connection);
    Object filePath = getHDFSFilePath(connectionBean, classLoader, hdfsPath);
    IExtractSchemaService<HDFSConnection> service = ExtractHDFSMetaServiceFactory.getService(connectionBean, classLoader, filePath);
    return service.extractColumns(connection, metadataTable);
}

Example 7 with HDFSConnection

use of org.talend.repository.model.hdfs.HDFSConnection in project tbd-studio-se by Talend.

the class HdfsContextHandler method createContextParameters.

@Override
public List<IContextParameter> createContextParameters(String prefixName, Connection connection, Set<IConnParamName> paramSet) {
    List<IContextParameter> varList = new ArrayList<IContextParameter>();
    if (connection instanceof HDFSConnection) {
        HDFSConnection conn = (HDFSConnection) connection;
        String paramPrefix = prefixName + ConnectionContextHelper.LINE;
        String paramName = null;
        for (IConnParamName param : paramSet) {
            if (param instanceof EHadoopParamName) {
                EHadoopParamName hdfsParam = (EHadoopParamName) param;
                paramName = paramPrefix + hdfsParam;
                switch(hdfsParam) {
                    case HdfsUser:
                        ConnectionContextHelper.createParameters(varList, paramName, conn.getUserName());
                        break;
                    case HdfsRowSeparator:
                        ConnectionContextHelper.createParameters(varList, paramName, conn.getRowSeparator());
                        break;
                    case HdfsFileSeparator:
                        ConnectionContextHelper.createParameters(varList, paramName, conn.getFieldSeparator());
                        break;
                    case HdfsRowHeader:
                        ConnectionContextHelper.createParameters(varList, paramName, conn.getHeaderValue(), JavaTypesManager.INTEGER);
                        break;
                    default:
                }
            }
        }
        createHadoopPropertiesContextVariable(prefixName, varList, conn.getHadoopProperties());
    }
    return varList;
}

Example 8 with HDFSConnection

use of org.talend.repository.model.hdfs.HDFSConnection in project tbd-studio-se by Talend.

the class HdfsContextHandler method setPropertiesForExistContextMode.

@Override
public void setPropertiesForExistContextMode(Connection connection, Set<IConnParamName> paramSet, Map<ContextItem, List<ConectionAdaptContextVariableModel>> adaptMap) {
    if (connection == null) {
        return;
    }
    if (connection instanceof HDFSConnection) {
        HDFSConnection hcatalogConn = (HDFSConnection) connection;
        ContextItem currentContext = null;
        for (IConnParamName param : paramSet) {
            if (param instanceof EHadoopParamName) {
                String hdfsVariableName = null;
                EHadoopParamName hdfsConnectionParam = (EHadoopParamName) param;
                if (adaptMap != null && adaptMap.size() > 0) {
                    for (Map.Entry<ContextItem, List<ConectionAdaptContextVariableModel>> entry : adaptMap.entrySet()) {
                        currentContext = entry.getKey();
                        List<ConectionAdaptContextVariableModel> modelList = entry.getValue();
                        for (ConectionAdaptContextVariableModel model : modelList) {
                            if (model.getValue().equals(hdfsConnectionParam.name())) {
                                hdfsVariableName = model.getName();
                                break;
                            }
                        }
                    }
                }
                if (hdfsVariableName != null) {
                    hdfsVariableName = getCorrectVariableName(currentContext, hdfsVariableName, hdfsConnectionParam);
                    matchContextForAttribues(hcatalogConn, hdfsConnectionParam, hdfsVariableName);
                }
            }
        }
        matchAdditionProperties(hcatalogConn, adaptMap);
    }
}

Example 9 with HDFSConnection

use of org.talend.repository.model.hdfs.HDFSConnection in project tbd-studio-se by Talend.

the class HdfsContextHandler method revertPropertiesForContextMode.

@Override
public void revertPropertiesForContextMode(Connection connection, ContextType contextType) {
    if (connection instanceof HDFSConnection) {
        HDFSConnection conn = (HDFSConnection) connection;
        String hdfsUser = TalendQuoteUtils.removeQuotes(ContextParameterUtils.getOriginalValue(contextType, conn.getUserName()));
        String hdfsRowSeparator = TalendQuoteUtils.removeQuotes(ContextParameterUtils.getOriginalValue(contextType, conn.getRowSeparator()));
        String hdfsFiledSeparator = TalendQuoteUtils.removeQuotes(ContextParameterUtils.getOriginalValue(contextType, conn.getFieldSeparator()));
        String hdfsRowHeader = TalendQuoteUtils.removeQuotes(ContextParameterUtils.getOriginalValue(contextType, conn.getHeaderValue()));
        String hadoopProperties = conn.getHadoopProperties();
        List<Map<String, Object>> propertiesAfterRevert = transformContextModeToHadoopProperties(HadoopRepositoryUtil.getHadoopPropertiesList(hadoopProperties), contextType);
        conn.setHadoopProperties(HadoopRepositoryUtil.getHadoopPropertiesJsonStr(propertiesAfterRevert));
        conn.setUserName(hdfsUser);
        conn.setRowSeparator(hdfsRowSeparator);
        conn.setFieldSeparator(hdfsFiledSeparator);
        conn.setHeaderValue(hdfsRowHeader);
    }
}

Example 10 with HDFSConnection

use of org.talend.repository.model.hdfs.HDFSConnection in project tbd-studio-se by Talend.

the class HdfsContextHandler method matchAdditionProperties.

@Override
protected void matchAdditionProperties(Connection conn, Map<ContextItem, List<ConectionAdaptContextVariableModel>> adaptMap) {
    if (conn instanceof HDFSConnection) {
        HDFSConnection hadoopConn = (HDFSConnection) conn;
        if (adaptMap != null && !adaptMap.isEmpty()) {
            List<Map<String, Object>> hadoopListProperties = HadoopRepositoryUtil.getHadoopPropertiesList(hadoopConn.getHadoopProperties());
            Set<String> keys = getConAdditionPropertiesForContextMode(conn);
            for (Map.Entry<ContextItem, List<ConectionAdaptContextVariableModel>> entry : adaptMap.entrySet()) {
                List<ConectionAdaptContextVariableModel> modelList = entry.getValue();
                for (ConectionAdaptContextVariableModel model : modelList) {
                    String propertyKey = model.getValue();
                    if (keys.contains(propertyKey)) {
                        List<Map<String, Object>> propertiesAfterContext = transformHadoopPropertiesForExistContextMode(hadoopListProperties, propertyKey, model.getName());
                        hadoopConn.setHadoopProperties(HadoopRepositoryUtil.getHadoopPropertiesJsonStr(propertiesAfterContext));
                    }
                }
            }
        }
    }
}