Examples with DelimitedFileConnection org.talend.core.model.metadata.builder.connection.DelimitedFileConnection used on opensource projects

Example 1 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdi-studio-se by Talend.

the class DynamicComposite method updateRepositoryListExtra.

/**
     * for job settings extra (feature 2710).
     * 
     */
private void updateRepositoryListExtra(IElementParameter param, List<String> repositoryConnectionNameList, List<String> repositoryConnectionValueList, boolean extra) {
    String repositoryValue = param.getParentParameter().getRepositoryValue();
    if (repositoryValue != null) {
        List<String> connectionNamesList = new ArrayList<String>();
        List<String> connectionValuesList = new ArrayList<String>();
        for (String key : repositoryConnectionItemMap.keySet()) {
            ConnectionItem connectionItem = repositoryConnectionItemMap.get(key);
            Connection connection = connectionItem.getConnection();
            String name = //$NON-NLS-1$
            getRepositoryAliasName(connectionItem) + ":" + connectionItem.getProperty().getLabel();
            if ((connection instanceof DelimitedFileConnection) && (repositoryValue.equals(ERepositoryCategoryType.DELIMITED.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof PositionalFileConnection) && (repositoryValue.equals(ERepositoryCategoryType.POSITIONAL.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof RegexpFileConnection) && (repositoryValue.equals(ERepositoryCategoryType.REGEX.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof XmlFileConnection) && (repositoryValue.equals(ERepositoryCategoryType.XML.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof FileExcelConnection) && (repositoryValue.equals(ERepositoryCategoryType.EXCEL.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof GenericSchemaConnection) && (repositoryValue.equals(ERepositoryCategoryType.GENERIC.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof LDAPSchemaConnection) && (repositoryValue.equals(ERepositoryCategoryType.LDAP.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof SalesforceSchemaConnection) && (repositoryValue.equals(ERepositoryCategoryType.SALESFORCE.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof WSDLSchemaConnection) && (repositoryValue.equals(ERepositoryCategoryType.WSDL.getName()))) {
                addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
            }
            if ((connection instanceof DatabaseConnection) && (repositoryValue.startsWith(ERepositoryCategoryType.DATABASE.getName()))) {
                //$NON-NLS-1$
                String currentDbType = (String) RepositoryToComponentProperty.getValue(connection, "TYPE", null);
                if (repositoryValue.contains(":")) {
                    // database //$NON-NLS-1$
                    // is
                    // specified
                    // //$NON-NLS-1$
                    //$NON-NLS-1$
                    String neededDbType = repositoryValue.substring(repositoryValue.indexOf(":") + 1);
                    if (neededDbType.equals(currentDbType)) {
                        addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
                    }
                } else {
                    addOrderDisplayNames(connectionValuesList, connectionNamesList, key, name);
                }
            }
        }
        repositoryConnectionNameList.addAll(connectionNamesList);
        repositoryConnectionValueList.addAll(connectionValuesList);
    } else {
        List<String> connectionValuesList = new ArrayList<String>();
        List<String> connectionStringList = new ArrayList<String>();
        for (String key : repositoryConnectionItemMap.keySet()) {
            ConnectionItem connectionItem = repositoryConnectionItemMap.get(key);
            String name = connectionItem.getProperty().getLabel();
            addOrderDisplayNames(connectionValuesList, connectionStringList, key, name);
        }
        repositoryConnectionNameList.addAll(connectionStringList);
        repositoryConnectionValueList.addAll(connectionValuesList);
    }
    param.setListItemsDisplayName(repositoryConnectionNameList.toArray(new String[0]));
    param.setListItemsValue(repositoryConnectionValueList.toArray(new String[0]));
    if (!repositoryConnectionItemMap.keySet().contains(param.getValue())) {
        if (repositoryConnectionNameList.size() > 0) {
            param.setValue(repositoryConnectionValueList.get(0));
        }
    }
}

Example 2 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class TOPRepositoryServiceTest method testRemoveAliasInSQLExplorer.

/**
 * Test method for
 * {@link org.talend.dataprofiler.core.service.TOPRepositoryService#removeAliasInSQLExplorer(org.talend.repository.model.IRepositoryNode)}
 * .
 */
public void testRemoveAliasInSQLExplorer() {
    IRepositoryNode child = mock(IRepositoryNode.class);
    IRepositoryViewObject obj = mock(IRepositoryViewObject.class);
    when(child.getObject()).thenReturn(obj);
    Property pro = mock(Property.class);
    when(obj.getProperty()).thenReturn(pro);
    ConnectionItem item = mock(ConnectionItem.class);
    when(pro.getItem()).thenReturn(item);
    MDMConnection mdm = mock(MDMConnection.class);
    DelimitedFileConnection dfile = mock(DelimitedFileConnection.class);
    DatabaseConnection dbc = mock(DatabaseConnection.class);
    when(item.getConnection()).thenReturn(dbc).thenReturn(dfile).thenReturn(mdm);
    List<ModelElement> dependencyClients = new ArrayList<ModelElement>();
    ModelElement m1 = mock(ModelElement.class);
    dependencyClients.add(m1);
    PowerMockito.mockStatic(EObjectHelper.class);
    when(EObjectHelper.getDependencyClients(dbc)).thenReturn(dependencyClients);
    when(EObjectHelper.getDependencyClients(mdm)).thenReturn(dependencyClients);
    when(EObjectHelper.getDependencyClients(dfile)).thenReturn(dependencyClients);
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
}

Example 3 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class WorkbenchUtilsTest method testUpdateDependAnalysisOfDelimitedFile_3.

// test for no column remained
@Test
public void testUpdateDependAnalysisOfDelimitedFile_3() throws IOException, URISyntaxException, PartInitException, BusinessException {
    // create a file connection
    DelimitedFileConnection fileConnection = ConnectionPackage.eINSTANCE.getConnectionFactory().createDelimitedFileConnection();
    // $NON-NLS-1$
    URL fileUrl = this.getClass().getResource("file1");
    MetadataTable metadataTable = UnitTestBuildHelper.initFileConnection(fileUrl, fileConnection);
    UnitTestBuildHelper.initColumns(metadataTable);
    IFile file = WorkspaceUtils.fileToIFile(new File(FileLocator.toFileURL(fileUrl).toURI()));
    IPath itemPath = file.getFullPath();
    Property connectionProperty = PropertiesFactory.eINSTANCE.createProperty();
    connectionProperty.setAuthor(((RepositoryContext) CoreRuntimePlugin.getInstance().getContext().getProperty(Context.REPOSITORY_CONTEXT_KEY)).getUser());
    connectionProperty.setVersion(VersionUtils.DEFAULT_VERSION);
    // $NON-NLS-1$
    connectionProperty.setStatusCode("");
    connectionProperty.setLabel("file3");
    DelimitedFileConnectionItem connectionItem = PropertiesFactory.eINSTANCE.createDelimitedFileConnectionItem();
    connectionItem.setProperty(connectionProperty);
    connectionItem.setConnection(fileConnection);
    try {
        ProxyRepositoryFactory.getInstance().create(connectionItem, itemPath.removeFirstSegments(3).removeLastSegments(1));
    } catch (PersistenceException e) {
        Assert.fail(e.getMessage());
    }
    // create an analysis which use the columns in the file connection
    Analysis analysis = UnitTestBuildHelper.createRealAnalysis("Ana01", null, false);
    AnalysisContext context = AnalysisPackage.eINSTANCE.getAnalysisFactory().createAnalysisContext();
    context.setConnection(fileConnection);
    analysis.setContext(context);
    context.getAnalysedElements().addAll(metadataTable.getColumns());
    DependenciesHandler.getInstance().setDependencyOn(analysis, fileConnection);
    ElementWriterFactory.getInstance().createAnalysisWrite().save(analysis);
    // change the file's schema
    List<MetadataColumn> tempNewColumns = new ArrayList<MetadataColumn>();
    MetadataColumn name = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    name.setName("name2");
    name.setLabel("name2");
    tempNewColumns.add(name);
    MetadataColumn country = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    country.setName("country2");
    country.setLabel("country2");
    tempNewColumns.add(country);
    MetadataColumn country1 = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    country1.setName("country1");
    country1.setLabel("country1");
    tempNewColumns.add(country1);
    metadataTable.getFeature().clear();
    metadataTable.getFeature().addAll(tempNewColumns);
    // before compare, the analysis has 5 analyzed elements
    Assert.assertEquals(5, analysis.getContext().getAnalysedElements().size());
    // call the tested method
    WorkbenchUtils.reloadMetadataOfDelimitedFile(metadataTable);
    List<MetadataColumn> afterCompareColumns = metadataTable.getColumns();
    // check the columns
    Assert.assertEquals(3, afterCompareColumns.size());
    Assert.assertEquals("name2", afterCompareColumns.get(0).getLabel());
    Assert.assertEquals("country2", afterCompareColumns.get(1).getLabel());
    Assert.assertEquals("country1", afterCompareColumns.get(2).getLabel());
    metadataTable.getColumns().clear();
    metadataTable.getColumns().addAll(afterCompareColumns);
    WorkbenchUtils.impactExistingAnalyses(fileConnection);
    // check the depended analysis
    EList<Dependency> clientDependencies = fileConnection.getSupplierDependency();
    for (Dependency dep : clientDependencies) {
        for (ModelElement mod : dep.getClient()) {
            if (!(mod instanceof Analysis)) {
                continue;
            }
            Analysis ana = (Analysis) mod;
            // assert the column with same name still in the analysis
            Assert.assertNotNull(ana.getContext().getAnalysedElements());
            // should be: only 2 with same name remained
            Assert.assertEquals(0, ana.getContext().getAnalysedElements().size());
        }
    }
}

Example 4 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class RepositoryNodeHelper method getInWhichProject.

/**
 * the modelelement can belong to current project or referenced project.
 *
 * @param analysis
 * @return
 */
private static org.talend.core.model.general.Project getInWhichProject(ModelElement modelElement) {
    if (modelElement instanceof DatabaseConnection || modelElement instanceof DelimitedFileConnection) {
        if (modelElement.eIsProxy()) {
            modelElement = (ModelElement) EObjectHelper.resolveObject(modelElement);
        }
        // TDQ-12245: fix a NPE when the modelElement is ref project model
        String projectName = EObjectHelper.getURI(modelElement).segment(1);
        java.util.Set<Project> allProjects = ProxyRepositoryManager.getInstance().getAllProjects();
        for (Project project : allProjects) {
            if (project.getTechnicalLabel().equals(projectName)) {
                return project;
            }
        }
    }
    Property property = PropertyHelper.getProperty(modelElement);
    org.talend.core.model.properties.Project project = ProjectManager.getInstance().getProject(property);
    return new org.talend.core.model.general.Project(project);
}

Example 5 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class ColumnSetIndicatorEvaluator method evaluateByDelimitedFile.

/**
 * orgnize EList 'objectLs' for DelimitedFile connection.
 *
 * @param sqlStatement
 * @param returnCode
 * @return
 */
private ReturnCode evaluateByDelimitedFile(String sqlStatement, ReturnCode returnCode) {
    DelimitedFileConnection fileConnection = (DelimitedFileConnection) analysis.getContext().getConnection();
    String path = JavaSqlFactory.getURL(fileConnection);
    String rowSeparator = JavaSqlFactory.getRowSeparatorValue(fileConnection);
    IPath iPath = new Path(path);
    File file = iPath.toFile();
    if (!file.exists()) {
        // $NON-NLS-1$
        returnCode.setReturnCode(Messages.getString("ColumnSetIndicatorEvaluator.FileNotFound", file.getName()), false);
        return returnCode;
    }
    CSVReader csvReader = null;
    try {
        List<ModelElement> analysisElementList = this.analysis.getContext().getAnalysedElements();
        EMap<Indicator, AnalyzedDataSet> indicToRowMap = analysis.getResults().getIndicToRowMap();
        indicToRowMap.clear();
        if (Escape.CSV.equals(fileConnection.getEscapeType())) {
            // use CsvReader to parse.
            csvReader = FileUtils.createCsvReader(file, fileConnection);
            this.useCsvReader(csvReader, file, fileConnection, analysisElementList);
        } else {
            // use TOSDelimitedReader in FileInputDelimited to parse.
            FileInputDelimited fileInputDelimited = AnalysisExecutorHelper.createFileInputDelimited(fileConnection);
            long currentRow = JavaSqlFactory.getHeadValue(fileConnection);
            int columsCount = 0;
            while (fileInputDelimited.nextRecord()) {
                if (!continueRun()) {
                    break;
                }
                currentRow++;
                if (columsCount == 0) {
                    columsCount = fileInputDelimited.getColumnsCountOfCurrentRow();
                }
                String[] rowValues = new String[columsCount];
                for (int i = 0; i < columsCount; i++) {
                    rowValues[i] = fileInputDelimited.get(i);
                }
                orgnizeObjectsToHandel(path, rowValues, currentRow, analysisElementList, rowSeparator);
            }
            // TDQ-5851~
            fileInputDelimited.close();
        }
    } catch (Exception e) {
        log.error(e, e);
        returnCode.setReturnCode(e.getMessage(), false);
    } finally {
        if (csvReader != null) {
            try {
                csvReader.close();
            } catch (IOException e) {
                log.error(e, e);
            }
        }
    }
    return returnCode;
}