Examples with DelimitedFileConnection org.talend.core.model.metadata.builder.connection.DelimitedFileConnection used on opensource projects

Example 6 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class ColumnSetIndicatorEvaluator method evaluateByDelimitedFile.

/**
 * orgnize EList 'objectLs' for DelimitedFile connection.
 *
 * @param sqlStatement
 * @param returnCode
 * @return
 */
private ReturnCode evaluateByDelimitedFile(String sqlStatement, ReturnCode returnCode) {
    DelimitedFileConnection fileConnection = (DelimitedFileConnection) analysis.getContext().getConnection();
    String path = JavaSqlFactory.getURL(fileConnection);
    String rowSeparator = JavaSqlFactory.getRowSeparatorValue(fileConnection);
    IPath iPath = new Path(path);
    File file = iPath.toFile();
    if (!file.exists()) {
        // $NON-NLS-1$
        returnCode.setReturnCode(Messages.getString("ColumnSetIndicatorEvaluator.FileNotFound", file.getName()), false);
        return returnCode;
    }
    CSVReader csvReader = null;
    try {
        List<ModelElement> analysisElementList = this.analysis.getContext().getAnalysedElements();
        EMap<Indicator, AnalyzedDataSet> indicToRowMap = analysis.getResults().getIndicToRowMap();
        indicToRowMap.clear();
        if (Escape.CSV.equals(fileConnection.getEscapeType())) {
            // use CsvReader to parse.
            csvReader = FileUtils.createCsvReader(file, fileConnection);
            this.useCsvReader(csvReader, file, fileConnection, analysisElementList);
        } else {
            // use TOSDelimitedReader in FileInputDelimited to parse.
            FileInputDelimited fileInputDelimited = AnalysisExecutorHelper.createFileInputDelimited(fileConnection);
            long currentRow = JavaSqlFactory.getHeadValue(fileConnection);
            int columsCount = 0;
            while (fileInputDelimited.nextRecord()) {
                if (!continueRun()) {
                    break;
                }
                currentRow++;
                if (columsCount == 0) {
                    columsCount = fileInputDelimited.getColumnsCountOfCurrentRow();
                }
                String[] rowValues = new String[columsCount];
                for (int i = 0; i < columsCount; i++) {
                    rowValues[i] = fileInputDelimited.get(i);
                }
                orgnizeObjectsToHandel(path, rowValues, currentRow, analysisElementList, rowSeparator);
            }
            // TDQ-5851~
            fileInputDelimited.close();
        }
    } catch (Exception e) {
        log.error(e, e);
        returnCode.setReturnCode(e.getMessage(), false);
    } finally {
        if (csvReader != null) {
            try {
                csvReader.close();
            } catch (IOException e) {
                log.error(e, e);
            }
        }
    }
    return returnCode;
}

Example 7 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class TreeMessageInfoDialog method getLabelProvider.

protected ILabelProvider getLabelProvider() {
    if (labelProvider == null) {
        labelProvider = new LabelProvider() {

            @Override
            public String getText(Object obj) {
                if (obj == null) {
                    // $NON-NLS-1$
                    return "";
                }
                if (obj instanceof ImpactNode) {
                    return ((ImpactNode) obj).toString();
                } else if (obj instanceof IFile) {
                    IFile file = (IFile) obj;
                    ModelElement modelElement = ModelElementFileFactory.getModelElement(file);
                    // MOD msjian TDQ-5909: modify to displayName
                    return modelElement != null ? PropertyHelper.getProperty(modelElement).getDisplayName() : file.getName();
                } else if (obj instanceof RepositoryViewObject) {
                    // Added 20130226 TDQ-6899 show the name for Jrxml object (which has no related ModelElement)
                    return ((IRepositoryViewObject) obj).getLabel();
                } else if (obj instanceof MetadataTable) {
                    // ADD TDQ-7146: for all type's connection table/view node
                    return ((ModelElement) obj).getName();
                }
                Property property = PropertyHelper.getProperty((ModelElement) obj);
                return property == null ? ((ModelElement) obj).getName() : property.getDisplayName();
            }

            @Override
            public Image getImage(Object obj) {
                ModelElement modelElement = null;
                if (obj instanceof ModelElement) {
                    modelElement = (ModelElement) obj;
                } else if (obj instanceof ImpactNode) {
                    modelElement = ((ImpactNode) obj).getNodeElement();
                } else if (obj instanceof IFile) {
                    modelElement = ModelElementFileFactory.getModelElement((IFile) obj);
                } else if (obj instanceof RepositoryViewObject) {
                    // Added 20130226 TDQ-6899 show the name for Jrxml object (which has no related ModelElement)
                    return ImageLib.getImage(ImageLib.JRXML_ICON);
                }
                // ~
                if (modelElement == null) {
                    if (((ImpactNode) obj).getNode() != null) {
                        return ImageLib.getImage(ImageLib.JRXML_ICON);
                    }
                    return super.getImage(obj);
                }
                Image modelElementImage = null;
                String imgName = null;
                if (modelElement instanceof Analysis) {
                    imgName = ImageLib.ANALYSIS_OBJECT;
                } else if (modelElement instanceof TdReport) {
                    imgName = ImageLib.REPORT_OBJECT;
                } else if (modelElement instanceof DatabaseConnection) {
                    imgName = ImageLib.CONNECTION;
                } else if (modelElement instanceof DelimitedFileConnection) {
                    imgName = ImageLib.FILE_DELIMITED;
                } else if (modelElement instanceof Pattern) {
                    imgName = ImageLib.PATTERN_REG;
                } else if (modelElement instanceof IndicatorDefinition) {
                    // IndicatorDefinition
                    if (modelElement instanceof DQRule) {
                        imgName = ImageLib.DQ_RULE;
                    } else if (modelElement instanceof MatchRuleDefinition) {
                        imgName = ImageLib.MATCH_RULE_ICON;
                    } else {
                        imgName = ImageLib.IND_DEFINITION;
                    }
                // TDQ-8550~
                } else if (modelElement instanceof MetadataTable) {
                    imgName = ImageLib.TABLE;
                }
                if (imgName != null) {
                    modelElementImage = ImageLib.getImage(imgName);
                }
                // add lock icon on the image
                if (modelElementImage != null) {
                    if (modelElement != null) {
                        Property property = PropertyHelper.getProperty(modelElement);
                        if (property != null) {
                            Item item = property.getItem();
                            if (item != null) {
                                if (ProxyRepositoryManager.getInstance().isLockByUserOwn(item)) {
                                    modelElementImage = ImageLib.createLockedByOwnIcon(imgName);
                                } else if (ProxyRepositoryManager.getInstance().isLockByOthers(item)) {
                                    modelElementImage = ImageLib.createLockedByOtherIcon(imgName);
                                }
                            }
                        }
                    }
                    return modelElementImage;
                }
                return super.getImage(obj);
            }
        };
    }
    return labelProvider;
}

Example 8 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class FileSamplingDataSourceTest method testGetRecord.

@Test
public void testGetRecord() throws Exception {
    fileDataSource = new FileSamplingDataSource();
    DelimitedFileConnection fileConnection = ConnectionPackage.eINSTANCE.getConnectionFactory().createDelimitedFileConnection();
    // $NON-NLS-1$
    URL fileUrl = this.getClass().getResource("simple_data.csv");
    UnitTestBuildHelper.getDefault().initFileConnection(fileUrl, fileConnection);
    fileDataSource.setDataSource(fileConnection);
    int[] pos = { 0, 1, 2, 3 };
    ((FileSamplingDataSource) fileDataSource).setColumnPositions(pos);
    // CSVReader csvReader = createCsvReader(new File(getClass().getClassLoader()
    // .getResource("org/talend/cwm/db/connection/datasource/simple_data.csv").getFile()));
    int idx = 0;
    while (fileDataSource.hasNext()) {
        String value = getString(fileDataSource.getRecord());
        System.out.println(value);
        if (0 == idx) {
            // $NON-NLS-1$
            assertEquals("-24,male,4000,2010-10-23,", value);
        } else if (1 == idx) {
            // $NON-NLS-1$
            assertEquals("-50.0,male,2000,2011-02-02 12:10:00,", value);
        } else if (5 == idx) {
            // $NON-NLS-1$
            assertEquals(",female,4000,02/01/2008,", value);
        } else if (9 == idx) {
            // $NON-NLS-1$
            assertEquals("a str,male,30000,2004-12-20 00:00:00,", value);
        }
        idx++;
    }
}

Example 9 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class TOPRepositoryServiceTest method testRemoveAliasInSQLExplorer.

/**
 * Test method for
 * {@link org.talend.dataprofiler.core.service.TOPRepositoryService#removeAliasInSQLExplorer(org.talend.repository.model.IRepositoryNode)}
 * .
 */
public void testRemoveAliasInSQLExplorer() {
    IRepositoryNode child = mock(IRepositoryNode.class);
    IRepositoryViewObject obj = mock(IRepositoryViewObject.class);
    when(child.getObject()).thenReturn(obj);
    Property pro = mock(Property.class);
    when(obj.getProperty()).thenReturn(pro);
    ConnectionItem item = mock(ConnectionItem.class);
    when(pro.getItem()).thenReturn(item);
    MDMConnection mdm = mock(MDMConnection.class);
    DelimitedFileConnection dfile = mock(DelimitedFileConnection.class);
    DatabaseConnection dbc = mock(DatabaseConnection.class);
    when(item.getConnection()).thenReturn(dbc).thenReturn(dfile).thenReturn(mdm);
    List<ModelElement> dependencyClients = new ArrayList<ModelElement>();
    ModelElement m1 = mock(ModelElement.class);
    dependencyClients.add(m1);
    PowerMockito.mockStatic(EObjectHelper.class);
    when(EObjectHelper.getDependencyClients(dbc)).thenReturn(dependencyClients);
    when(EObjectHelper.getDependencyClients(mdm)).thenReturn(dependencyClients);
    when(EObjectHelper.getDependencyClients(dfile)).thenReturn(dependencyClients);
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
    Assert.assertFalse(tdqRepService.removeAliasInSQLExplorer(child));
}

Example 10 with DelimitedFileConnection

use of org.talend.core.model.metadata.builder.connection.DelimitedFileConnection in project tdq-studio-se by Talend.

the class WorkbenchUtilsTest method testUpdateDependAnalysisOfDelimitedFile_3.

// test for no column remained
@Test
public void testUpdateDependAnalysisOfDelimitedFile_3() throws IOException, URISyntaxException, PartInitException, BusinessException {
    // create a file connection
    DelimitedFileConnection fileConnection = ConnectionPackage.eINSTANCE.getConnectionFactory().createDelimitedFileConnection();
    // $NON-NLS-1$
    URL fileUrl = this.getClass().getResource("file1");
    MetadataTable metadataTable = UnitTestBuildHelper.initFileConnection(fileUrl, fileConnection);
    UnitTestBuildHelper.initColumns(metadataTable);
    IFile file = WorkspaceUtils.fileToIFile(new File(FileLocator.toFileURL(fileUrl).toURI()));
    IPath itemPath = file.getFullPath();
    Property connectionProperty = PropertiesFactory.eINSTANCE.createProperty();
    connectionProperty.setAuthor(((RepositoryContext) CoreRuntimePlugin.getInstance().getContext().getProperty(Context.REPOSITORY_CONTEXT_KEY)).getUser());
    connectionProperty.setVersion(VersionUtils.DEFAULT_VERSION);
    // $NON-NLS-1$
    connectionProperty.setStatusCode("");
    connectionProperty.setLabel("file3");
    DelimitedFileConnectionItem connectionItem = PropertiesFactory.eINSTANCE.createDelimitedFileConnectionItem();
    connectionItem.setProperty(connectionProperty);
    connectionItem.setConnection(fileConnection);
    try {
        ProxyRepositoryFactory.getInstance().create(connectionItem, itemPath.removeFirstSegments(3).removeLastSegments(1));
    } catch (PersistenceException e) {
        Assert.fail(e.getMessage());
    }
    // create an analysis which use the columns in the file connection
    Analysis analysis = UnitTestBuildHelper.createRealAnalysis("Ana01", null, false);
    AnalysisContext context = AnalysisPackage.eINSTANCE.getAnalysisFactory().createAnalysisContext();
    context.setConnection(fileConnection);
    analysis.setContext(context);
    context.getAnalysedElements().addAll(metadataTable.getColumns());
    DependenciesHandler.getInstance().setDependencyOn(analysis, fileConnection);
    ElementWriterFactory.getInstance().createAnalysisWrite().save(analysis);
    // change the file's schema
    List<MetadataColumn> tempNewColumns = new ArrayList<MetadataColumn>();
    MetadataColumn name = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    name.setName("name2");
    name.setLabel("name2");
    tempNewColumns.add(name);
    MetadataColumn country = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    country.setName("country2");
    country.setLabel("country2");
    tempNewColumns.add(country);
    MetadataColumn country1 = ConnectionPackage.eINSTANCE.getConnectionFactory().createMetadataColumn();
    country1.setName("country1");
    country1.setLabel("country1");
    tempNewColumns.add(country1);
    metadataTable.getFeature().clear();
    metadataTable.getFeature().addAll(tempNewColumns);
    // before compare, the analysis has 5 analyzed elements
    Assert.assertEquals(5, analysis.getContext().getAnalysedElements().size());
    // call the tested method
    WorkbenchUtils.reloadMetadataOfDelimitedFile(metadataTable);
    List<MetadataColumn> afterCompareColumns = metadataTable.getColumns();
    // check the columns
    Assert.assertEquals(3, afterCompareColumns.size());
    Assert.assertEquals("name2", afterCompareColumns.get(0).getLabel());
    Assert.assertEquals("country2", afterCompareColumns.get(1).getLabel());
    Assert.assertEquals("country1", afterCompareColumns.get(2).getLabel());
    metadataTable.getColumns().clear();
    metadataTable.getColumns().addAll(afterCompareColumns);
    WorkbenchUtils.impactExistingAnalyses(fileConnection);
    // check the depended analysis
    EList<Dependency> clientDependencies = fileConnection.getSupplierDependency();
    for (Dependency dep : clientDependencies) {
        for (ModelElement mod : dep.getClient()) {
            if (!(mod instanceof Analysis)) {
                continue;
            }
            Analysis ana = (Analysis) mod;
            // assert the column with same name still in the analysis
            Assert.assertNotNull(ana.getContext().getAnalysedElements());
            // should be: only 2 with same name remained
            Assert.assertEquals(0, ana.getContext().getAnalysedElements().size());
        }
    }
}