Examples with AnalyzedDataSet org.talend.dataquality.analysis.AnalyzedDataSet used on opensource projects

Example 11 with AnalyzedDataSet

use of org.talend.dataquality.analysis.AnalyzedDataSet in project tdq-studio-se by Talend.

the class ColumnSetIndicatorEvaluator method handleObjects.

/**
 * handle Objects and store data for delimited file .
 *
 * @param objectLs
 * @param rowValues
 * @param metadataColumn is one of analysedElements.it is used to get its Table then get the table's columns.
 */
private void handleObjects(EList<Object> objectLs, String[] rowValues, List<MetadataColumn> columnList) {
    if (objectLs.size() == 0) {
        return;
    }
    EList<Indicator> indicators = analysis.getResults().getIndicators();
    EMap<Indicator, AnalyzedDataSet> indicToRowMap = analysis.getResults().getIndicToRowMap();
    int recordIncrement = 0;
    if (indicators != null) {
        for (Indicator indicator : indicators) {
            if (!this.continueRun()) {
                break;
            }
            if (ColumnsetPackage.eINSTANCE.getColumnSetMultiValueIndicator().isSuperTypeOf(indicator.eClass())) {
                indicator.handle(objectLs);
                // feature 19192,store all rows value for RowCountIndicator
                if (indicator instanceof SimpleStatIndicator) {
                    SimpleStatIndicator simpIndi = (SimpleStatIndicator) indicator;
                    for (Indicator leafIndicator : simpIndi.getLeafIndicators()) {
                        if (!this.continueRun()) {
                            break;
                        }
                        // (distinct count, unique count, duplicate count, etc).
                        if (!analysis.getParameters().isStoreData()) {
                            // ~
                            continue;
                        }
                        List<Object[]> valueObjectList = initDataSet(leafIndicator, indicToRowMap);
                        recordIncrement = valueObjectList.size();
                        Object[] valueObject = new Object[columnList.size()];
                        if (recordIncrement < analysis.getParameters().getMaxNumberRows()) {
                            for (int j = 0; j < columnList.size(); j++) {
                                if (!this.continueRun()) {
                                    break;
                                }
                                Object newobject = PluginConstant.EMPTY_STRING;
                                // if (recordIncrement < analysis.getParameters().getMaxNumberRows()) {
                                if (j < rowValues.length) {
                                    newobject = rowValues[j];
                                }
                                if (recordIncrement < valueObjectList.size()) {
                                    valueObjectList.get(recordIncrement)[j] = newobject;
                                } else {
                                    valueObject[j] = newobject;
                                    valueObjectList.add(valueObject);
                                }
                            // }
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 12 with AnalyzedDataSet

use of org.talend.dataquality.analysis.AnalyzedDataSet in project tdq-studio-se by Talend.

the class ColumnSetIndicatorEvaluatorTest method testExecuteSqlQuery_file.

/**
 * Test method for {@link org.talend.dq.indicators.ColumnSetIndicatorEvaluator#executeSqlQuery(String)} .
 */
@Test
public void testExecuteSqlQuery_file() throws Exception {
    Analysis analysis = mock(Analysis.class);
    // $NON-NLS-1$
    stub(method(DelimitedFileIndicatorEvaluator.class, "handleByARow"));
    // $NON-NLS-1$
    stub(method(DelimitedFileIndicatorEvaluator.class, "addResultToIndicatorToRowMap", Indicator.class, EMap.class));
    AnalysisContext context = mock(AnalysisContext.class);
    when(analysis.getContext()).thenReturn(context);
    DelimitedFileConnection deliFileConn = mock(DelimitedFileConnection.class);
    when(context.getConnection()).thenReturn(deliFileConn);
    when(deliFileConn.isContextMode()).thenReturn(true);
    when(deliFileConn.getFilePath()).thenReturn(context_fd1_File);
    when(deliFileConn.getFieldSeparatorValue()).thenReturn(context_fd1_FieldSeparator);
    when(deliFileConn.getEncoding()).thenReturn(context_fd1_Encoding);
    IPath iPath = mock(IPath.class);
    File file = new File(realFile);
    BufferedWriter output = new BufferedWriter(new FileWriter(file));
    String str = "id;Cocust(Tests);owner_id\n" + "1;yellow;3301\n" + "2;blue;3302\n" + "4;red;3307\n" + "5;white;4563\n" + "6;pink2;457883\n" + "7;blank;231233\n";
    output.write(str);
    output.close();
    when(iPath.toFile()).thenReturn(file);
    assertTrue(file.exists());
    PowerMockito.mockStatic(JavaSqlFactory.class);
    when(JavaSqlFactory.getURL(deliFileConn)).thenReturn(realFile);
    when(JavaSqlFactory.getFieldSeparatorValue(deliFileConn)).thenReturn(realFieldSeparator);
    when(JavaSqlFactory.getEncoding(deliFileConn)).thenReturn(realEncoding);
    PowerMockito.mockStatic(ResourceBundle.class);
    ResourceBundle bundle = mock(ResourceBundle.class);
    // $NON-NLS-1$
    when(ResourceBundle.getBundle("Messages")).thenReturn(bundle);
    PowerMockito.mockStatic(Messages.class);
    // $NON-NLS-1$
    when(Messages.getString(anyString())).thenReturn("testString");
    AnalysisResult results = mock(AnalysisResult.class);
    when(analysis.getResults()).thenReturn(results);
    EMap<Indicator, AnalyzedDataSet> indicToRowMap = mock(EMap.class);
    when(results.getIndicToRowMap()).thenReturn(indicToRowMap);
    when(deliFileConn.getHeaderValue()).thenReturn(context_fd1_Header);
    when(deliFileConn.getFooterValue()).thenReturn(zero);
    when(deliFileConn.getLimitValue()).thenReturn(zero);
    when(deliFileConn.getEscapeType()).thenReturn(Escape.DELIMITED);
    when(deliFileConn.getRowSeparatorValue()).thenReturn(context_fd1_RowSeparator);
    when(deliFileConn.isSplitRecord()).thenReturn(false);
    when(deliFileConn.isRemoveEmptyRow()).thenReturn(false);
    when(JavaSqlFactory.getHeadValue(deliFileConn)).thenReturn(realHeading);
    when(JavaSqlFactory.getRowSeparatorValue(deliFileConn)).thenReturn(realRowSeparator);
    PowerMockito.mockStatic(LanguageManager.class);
    when(LanguageManager.getCurrentLanguage()).thenReturn(ECodeLanguage.JAVA);
    PowerMockito.mockStatic(ParameterUtil.class);
    when(ParameterUtil.trimParameter(realFile)).thenReturn(realFile);
    when(ParameterUtil.trimParameter(realEncoding)).thenReturn(realEncoding);
    PowerMockito.mockStatic(StringUtils.class);
    when(StringUtils.loadConvert(realFieldSeparator, ECodeLanguage.JAVA.getName())).thenReturn(realFieldSeparator);
    when(ParameterUtil.trimParameter(realFieldSeparator)).thenReturn(realFieldSeparator);
    when(StringUtils.loadConvert(realRowSeparator, ECodeLanguage.JAVA.getName())).thenReturn(realRowSeparator);
    when(ParameterUtil.trimParameter(realRowSeparator)).thenReturn(realRowSeparator);
    List<ModelElement> columnElementList = new BasicEList<ModelElement>();
    List<MetadataColumn> columnElementList2 = new BasicEList<MetadataColumn>();
    MetadataColumn mc0 = mock(MetadataColumn.class);
    MetadataColumn mc1 = mock(MetadataColumn.class);
    MetadataColumn mc2 = mock(MetadataColumn.class);
    columnElementList.add(mc0);
    columnElementList.add(mc1);
    columnElementList.add(mc2);
    columnElementList2.add(mc0);
    columnElementList2.add(mc1);
    columnElementList2.add(mc2);
    EList<ModelElement> eLs = (EList<ModelElement>) columnElementList;
    when(context.getAnalysedElements()).thenReturn(eLs);
    PowerMockito.mockStatic(ColumnHelper.class);
    MetadataTable mTable = mock(MetadataTable.class);
    when(mTable.getColumns()).thenReturn((EList<MetadataColumn>) columnElementList2);
    when(ColumnHelper.getColumnOwnerAsMetadataTable(mc0)).thenReturn(mTable);
    when(ColumnHelper.getColumnOwnerAsMetadataTable(mc1)).thenReturn(mTable);
    when(ColumnHelper.getColumnOwnerAsMetadataTable(mc2)).thenReturn(mTable);
    ColumnSetIndicatorEvaluator evaluator = new ColumnSetIndicatorEvaluator(analysis);
    ColumnSetIndicatorEvaluator spyEvaluator = Mockito.spy(evaluator);
    Mockito.doReturn(true).when(spyEvaluator).continueRun();
    ReturnCode rc = spyEvaluator.executeSqlQuery(empty);
    assertTrue(rc.isOk());
}

Example 13 with AnalyzedDataSet

use of org.talend.dataquality.analysis.AnalyzedDataSet in project tdq-studio-se by Talend.

the class DelimitedFileIndicatorEvaluatorTest method testExecuteSqlQuery_delimetd.

@Ignore
@Test
public void testExecuteSqlQuery_delimetd() throws Exception {
    // $NON-NLS-1$
    String empty = "";
    Analysis analysis = mock(Analysis.class);
    DelimitedFileIndicatorEvaluator delFileIndiEvaluator = new DelimitedFileIndicatorEvaluator(analysis);
    DelimitedFileIndicatorEvaluator spyEvaluator = Mockito.spy(delFileIndiEvaluator);
    // $NON-NLS-1$
    stub(method(DelimitedFileIndicatorEvaluator.class, "handleByARow"));
    // $NON-NLS-1$
    stub(method(DelimitedFileIndicatorEvaluator.class, "addResultToIndicatorToRowMap", Indicator.class, EMap.class));
    AnalysisContext context = mock(AnalysisContext.class);
    when(analysis.getContext()).thenReturn(context);
    DelimitedFileConnection deliFileConn = mock(DelimitedFileConnection.class);
    when(context.getConnection()).thenReturn(deliFileConn);
    when(deliFileConn.isContextMode()).thenReturn(false);
    // $NON-NLS-1$
    String path = "test.txt";
    PowerMockito.mockStatic(JavaSqlFactory.class);
    when(JavaSqlFactory.getURL(deliFileConn)).thenReturn(path);
    IPath iPath = mock(IPath.class);
    File file = new File(path);
    BufferedWriter output = new BufferedWriter(new FileWriter(file));
    String str = // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
    "id;Cocust(Tests);owner_id\n" + "1;yellow;3301\n" + "2;blue;3302\n" + " 4;red;3307\n" + "5;white;4563\n" + "6;pink2;457883\n" + // $NON-NLS-1$ //$NON-NLS-2$
    "7;blank;231233\n";
    output.write(str);
    output.close();
    when(iPath.toFile()).thenReturn(file);
    // $NON-NLS-1$
    when(deliFileConn.getFieldSeparatorValue()).thenReturn(";");
    // $NON-NLS-1$
    when(deliFileConn.getEncoding()).thenReturn("US-ASCII");
    AnalysisResult results = mock(AnalysisResult.class);
    when(analysis.getResults()).thenReturn(results);
    EMap<Indicator, AnalyzedDataSet> indicToRowMap = mock(EMap.class);
    when(results.getIndicToRowMap()).thenReturn(indicToRowMap);
    List<ModelElement> columnElementList = new BasicEList<ModelElement>();
    List<MetadataColumn> columnElementList2 = new BasicEList<MetadataColumn>();
    MetadataColumn mc0 = mock(MetadataColumn.class);
    MetadataColumn mc1 = mock(MetadataColumn.class);
    MetadataColumn mc2 = mock(MetadataColumn.class);
    columnElementList.add(mc0);
    columnElementList.add(mc1);
    columnElementList.add(mc2);
    columnElementList2.add(mc0);
    columnElementList2.add(mc1);
    columnElementList2.add(mc2);
    EList<ModelElement> eLs = (EList<ModelElement>) columnElementList;
    when(context.getAnalysedElements()).thenReturn(eLs);
    PowerMockito.mockStatic(ColumnHelper.class);
    MetadataTable mTable = mock(MetadataTable.class);
    when(mTable.getColumns()).thenReturn((EList<MetadataColumn>) columnElementList2);
    when(ColumnHelper.getColumnOwnerAsMetadataTable(mc0)).thenReturn(mTable);
    when(ColumnHelper.getColumnOwnerAsMetadataTable(mc1)).thenReturn(mTable);
    when(deliFileConn.getHeaderValue()).thenReturn(empty);
    when(deliFileConn.getFooterValue()).thenReturn(empty);
    when(deliFileConn.getLimitValue()).thenReturn(empty);
    when(deliFileConn.getEscapeType()).thenReturn(Escape.DELIMITED);
    // $NON-NLS-1$
    when(deliFileConn.getRowSeparatorValue()).thenReturn("\\n");
    when(deliFileConn.isRemoveEmptyRow()).thenReturn(false);
    when(deliFileConn.isSplitRecord()).thenReturn(false);
    PowerMockito.mockStatic(LanguageManager.class);
    when(LanguageManager.getCurrentLanguage()).thenReturn(ECodeLanguage.JAVA);
    Mockito.doReturn(true).when(spyEvaluator).continueRun();
    spyEvaluator.executeSqlQuery(empty);
}

Example 14 with AnalyzedDataSet

use of org.talend.dataquality.analysis.AnalyzedDataSet in project tdq-studio-se by Talend.

the class DrillDownEditorInput method filterAdaptDataList.

/**
 * @return get the data which will be displayed on the drill down editor.
 */
public List<Object[]> filterAdaptDataList() {
    // get columnValue
    List<Object[]> newColumnElementList = new ArrayList<Object[]>();
    AnalyzedDataSet analysisDataSet = this.getAnalysis().getResults().getIndicToRowMap().get(currIndicator);
    // TDQ-10545 : when restart studio and without run analysis, analysisDataSet is null
    if (analysisDataSet == null) {
        return newColumnElementList;
    }
    // TDQ-10545~
    if (analysisDataSet.getData() != null && analysisDataSet.getData().size() > 0) {
        List<Object[]> dataList = analysisDataSet.getData();
        newColumnElementList.addAll(getDesignatedData(dataList));
    } else if (analysisDataSet.getFrequencyData() != null && analysisDataSet.getFrequencyData().size() > 0) {
        String selectValue = this.getSelectValue();
        if (currIndicator instanceof LengthIndicator) {
            selectValue = ((LengthIndicator) currIndicator).getLength().toString();
        }
        // MOD yyi 2011-12-14 TDQ-4166:View rows for Date Pattern Frequency Indicator.
        if (currIndicator instanceof DatePatternFreqIndicator) {
            for (Object expression : analysisDataSet.getFrequencyData().keySet()) {
                if (Pattern.matches(((DatePatternFreqIndicator) currIndicator).getRegex(selectValue), expression.toString())) {
                    newColumnElementList.addAll(analysisDataSet.getFrequencyData().get(expression));
                }
            }
        } else {
            // MOD msjian TDQ-4617 2012-2-8: fixed a NPE
            List<Object[]> list = analysisDataSet.getFrequencyData().get(selectValue);
            if (list != null && list.size() > 0) {
                newColumnElementList.addAll(list);
            }
        // TDQ-4617 ~
        }
    } else if (analysisDataSet.getPatternData() != null && analysisDataSet.getPatternData().size() > 0) {
        if (DrillDownUtils.judgeMenuType(getMenuType(), DrillDownUtils.MENU_INVALID_TYPE)) {
            newColumnElementList.addAll(getDesignatedData((List<Object[]>) analysisDataSet.getPatternData().get(AnalyzedDataSetImpl.INVALID_VALUE)));
        } else if (DrillDownUtils.judgeMenuType(getMenuType(), DrillDownUtils.MENU_VALID_TYPE)) {
            newColumnElementList.addAll(getDesignatedData((List<Object[]>) analysisDataSet.getPatternData().get(AnalyzedDataSetImpl.VALID_VALUE)));
        }
    }
    return newColumnElementList;
}

Example 15 with AnalyzedDataSet

use of org.talend.dataquality.analysis.AnalyzedDataSet in project tdq-studio-se by Talend.

the class IndicToRowsMapImpl method setValue.

/**
 * <!-- begin-user-doc -->
 * <!-- end-user-doc -->
 * @generated
 */
public AnalyzedDataSet setValue(AnalyzedDataSet value) {
    AnalyzedDataSet oldValue = getValue();
    setTypedValue(value);
    return oldValue;
}