Examples with UDIndicatorDefinition org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition used on opensource projects

Example 21 with UDIndicatorDefinition

use of org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition in project tdq-studio-se by Talend.

the class SimpleStatisticsExplorerTest method testGetQueryMap.

/**
 * Test method for {@link org.talend.dq.analysis.explore.SimpleStatisticsExplorer#getQueryMap()}.
 */
@Test
public void testGetQueryMap() {
    Analysis ana = UnitTestBuildHelper.createAndInitAnalysis();
    TdTable table = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdTable();
    // $NON-NLS-1$
    table.setName("TDQ_CALENDAR");
    TdColumn column = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdColumn();
    // $NON-NLS-1$
    column.setName("CAL_DATE");
    TdSqlDataType tdsql = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdSqlDataType();
    // $NON-NLS-1$
    tdsql.setName("DATE");
    tdsql.setJavaDataType(Types.DATE);
    column.setSqlDataType(tdsql);
    table.getOwnedElement().add(column);
    column.setOwner(table);
    // create user define indicator
    UserDefIndicator userDefIndicator = IndicatorSqlFactory.eINSTANCE.createUserDefIndicator();
    UDIndicatorDefinition indicatorDefinition = UserdefineFactory.eINSTANCE.createUDIndicatorDefinition();
    // $NON-NLS-1$
    indicatorDefinition.setName("user define");
    userDefIndicator.setName(indicatorDefinition.getName());
    userDefIndicator.setIndicatorDefinition(indicatorDefinition);
    // $NON-NLS-1$
    TdExpression newTdExp = BooleanExpressionHelper.createTdExpression("SQL", body, null);
    newTdExp.setModificationDate(DateUtils.getCurrentDate(DateUtils.PATTERN_5));
    indicatorDefinition.getViewRowsExpression().add(newTdExp);
    // $NON-NLS-1$  //$NON-NLS-2$
    ChartDataEntity chartDataEntity = new ChartDataEntity(userDefIndicator, "2012-06-05", "1");
    chartDataEntity.setLabelNull(false);
    // $NON-NLS-1$
    chartDataEntity.setKey("2012-06-05");
    userDefIndicator.setAnalyzedElement(column);
    AnalysisResult createAnalysisResult = AnalysisFactory.eINSTANCE.createAnalysisResult();
    ExecutionInformations createExecutionInformations = AnalysisFactory.eINSTANCE.createExecutionInformations();
    createAnalysisResult.setResultMetadata(createExecutionInformations);
    createAnalysisResult.getIndicators().add(userDefIndicator);
    userDefIndicator.setAnalyzedElement(column);
    ana.setResults(createAnalysisResult);
    SimpleStatisticsExplorer simpleStatisticsExplorer = new SimpleStatisticsExplorer();
    Assert.assertTrue(simpleStatisticsExplorer.setAnalysis(ana));
    simpleStatisticsExplorer.setEnitty(chartDataEntity);
    Map<String, String> queryMap = simpleStatisticsExplorer.getQueryMap();
    assertFalse(queryMap.isEmpty());
    assertEquals(1, queryMap.size());
    System.err.println(queryMap.get("View rows"));
    assertEquals(// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
    "-- Analysis: anaA ;\n" + "-- Type of Analysis: Column Analysis ;\n" + "-- Purpose:  ;\n" + "-- Description:  ;\n" + "-- AnalyzedElement: CAL_DATE ;\n" + "-- Indicator: user define ;\n" + // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    "-- Showing: View rows ;\n" + "SELECT * FROM TDQ_CALENDAR ", // $NON-NLS-1$ //$NON-NLS-2$
    queryMap.get("View rows"));
    // test when is not sql engine
    ana.getParameters().setExecutionLanguage(ExecutionLanguage.JAVA);
    Map<String, String> queryMap_java = simpleStatisticsExplorer.getQueryMap();
    assertFalse(queryMap_java.isEmpty());
    assertEquals(1, queryMap_java.size());
    // $NON-NLS-1$
    assertEquals(null, queryMap_java.get("View rows"));
}

Example 22 with UDIndicatorDefinition

use of org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition in project tdq-studio-se by Talend.

the class FileSystemImportWriter method write.

/*
     * After check the conflicts of the imported object, calling this method (from ImportWizard) replace the conflicts
     * object in the records if the record is valid; then call the finish to do migrate. OR: merge the conflict system
     * indicators if valid.(overwrite)
     * 
     * @see
     * org.talend.dataprofiler.core.ui.imex.model.IImexWriter#write(org.talend.dataprofiler.core.ui.imex.model.ItemRecord
     * [], org.eclipse.core.runtime.IProgressMonitor)
     */
public void write(ItemRecord[] records, IProgressMonitor monitor) {
    if (monitor == null) {
        monitor = new NullProgressMonitor();
    }
    final ItemRecord[] fRecords = records;
    final IProgressMonitor fMonitor = monitor;
    need2MergeModelElementMap.clear();
    allImportItems.clear();
    RepositoryWorkUnit<Object> workUnit = new // $NON-NLS-1$
    RepositoryWorkUnit<Object>(// $NON-NLS-1$
    "Import TDQ Element") {

        @Override
        protected void run() {
            try {
                for (ItemRecord record : fRecords) {
                    if (fMonitor.isCanceled()) {
                        break;
                    }
                    Map<IPath, IPath> toImportMap = mapping(record);
                    // $NON-NLS-1$
                    fMonitor.subTask("Importing " + record.getName());
                    if (record.isValid()) {
                        boolean isDeleted = false;
                        // $NON-NLS-1$
                        log.info("Importing " + record.getFile().getAbsolutePath());
                        // Delete the conflict node before import.
                        IRepositoryViewObject object = record.getConflictObject();
                        boolean isDelete = true;
                        ModelElement modEle = record.getElement();
                        if (object != null) {
                            // is a system indicator definition, (using its UUid to find this SI not label)
                            if (isIndicatorDefinition(modEle)) {
                                if (isDQRule(modEle)) {
                                    if (isParserRule(modEle)) {
                                        mergeParserRule(record, (TDQBusinessRuleItem) object.getProperty().getItem());
                                        isDelete = false;
                                    } else if (isWhereRule(modEle)) {
                                    // do nothing here now
                                    }
                                } else if (isMatchRuleDefinition(modEle)) {
                                // do nothing here now
                                } else {
                                    // System Indicator and UDI need merge
                                    TDQIndicatorDefinitionItem indItem = (TDQIndicatorDefinitionItem) object.getProperty().getItem();
                                    mergeSystemIndicator(record, indItem);
                                    // only add it when it is UDIndicatorDefinition
                                    if (record.getElement() instanceof UDIndicatorDefinition) {
                                        need2MergeModelElementMap.put(indItem, record.getElement());
                                    }
                                    isDelete = false;
                                }
                            } else if (isPattern(modEle)) {
                                TDQPatternItem patternItem = (TDQPatternItem) object.getProperty().getItem();
                                mergePattern(record, patternItem);
                                need2MergeModelElementMap.put(patternItem, record.getElement());
                                isDelete = false;
                            } else {
                                // remove the dependency of the object
                                EObjectHelper.removeDependencys(PropertyHelper.getModelElement(object.getProperty()));
                                isDeleted = true;
                                // delete the object
                                ProxyRepositoryFactory.getInstance().deleteObjectPhysical(object);
                            }
                        }
                        if (isDelete) {
                            updateFiles.clear();
                            updateFilesCoverd.clear();
                            for (IPath resPath : toImportMap.keySet()) {
                                IPath desPath = toImportMap.get(resPath);
                                ResourceSet resourceSet = ProxyRepositoryFactory.getInstance().getRepositoryFactoryFromProvider().getResourceManager().resourceSet;
                                synchronized (resourceSet) {
                                    write(resPath, desPath);
                                    allCopiedFiles.add(desPath.toFile());
                                }
                                allImportItems.add(desPath);
                                // TDQ-12180
                                if (isDeleted) {
                                    AbstractSvnRepositoryService svnReposService = GlobalServiceRegister.getDefault().getSvnRepositoryService(AbstractSvnRepositoryService.class);
                                    if (svnReposService != null) {
                                        svnReposService.addIfImportOverride(desPath);
                                    }
                                }
                            }
                            for (File file : updateFiles) {
                                update(file, false);
                            }
                            for (File file : updateFilesCoverd) {
                                update(file, true);
                            }
                        }
                    } else {
                        for (String error : record.getErrors()) {
                            log.error(error);
                        }
                    }
                    fMonitor.worked(1);
                }
                finish(fRecords, fMonitor);
            } catch (Exception e) {
                log.error(e, e);
            }
        }
    };
    workUnit.setAvoidUnloadResources(Boolean.TRUE);
    ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(workUnit);
    // after above workUnit executed, the imported items will worked, than can do merge/update about UDI and Pattern
    RepositoryWorkUnit<Object> workUnitFinish = new // $NON-NLS-1$
    RepositoryWorkUnit<Object>(// $NON-NLS-1$
    "Finish Import TDQ Element") {

        @Override
        protected void run() throws LoginException, PersistenceException {
            try {
                postFinish();
            } catch (IOException e) {
                log.error(e, e);
            }
        }
    };
    workUnitFinish.setAvoidUnloadResources(Boolean.TRUE);
    ProxyRepositoryFactory.getInstance().executeRepositoryWorkUnit(workUnitFinish);
}

Example 23 with UDIndicatorDefinition

use of org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition in project tdq-studio-se by Talend.

the class FrequencyStatisticsExplorer method getQueryForUDIndicator.

/**
 * get Query For User define indicator.
 *
 * @param indicatorDefinition
 * @return
 */
private String getQueryForUDIndicator(IndicatorDefinition indicatorDefinition) {
    String sql = PluginConstant.EMPTY_STRING;
    EList<TdExpression> list = ((UDIndicatorDefinition) indicatorDefinition).getViewRowsExpression();
    TdExpression tdExp = DbmsLanguage.getSqlExpression(indicatorDefinition, dbmsLanguage.getDbmsName(), list, dbmsLanguage.getDbVersion());
    sql = tdExp.getBody();
    String dataFilterClause = getDataFilterClause();
    if (!dataFilterClause.equals(PluginConstant.EMPTY_STRING)) {
        // $NON-NLS-1$ //$NON-NLS-2$
        sql = sql.replace(GenericSQLHandler.WHERE_CLAUSE, dbmsLanguage.where() + "(" + dataFilterClause + ")");
        // $NON-NLS-1$ //$NON-NLS-2$
        sql = sql.replace(GenericSQLHandler.AND_WHERE_CLAUSE, dbmsLanguage.and() + "(" + dataFilterClause + ")");
    } else {
        sql = sql.replace(GenericSQLHandler.WHERE_CLAUSE, PluginConstant.EMPTY_STRING);
        sql = sql.replace(GenericSQLHandler.AND_WHERE_CLAUSE, PluginConstant.EMPTY_STRING);
    }
    ModelElement analyzedElement = this.indicator.getAnalyzedElement();
    String tableName = getFullyQualifiedTableName(analyzedElement);
    sql = sql.replace(GenericSQLHandler.TABLE_NAME, tableName);
    sql = sql.replace(GenericSQLHandler.COLUMN_NAMES, dbmsLanguage.quote(analyzedElement.getName()));
    if (sql.indexOf(GenericSQLHandler.UDI_INDICATOR_VALUE) != -1) {
        Object key = entity.getKey();
        if (key == null) {
            // $NON-NLS-1$ //$NON-NLS-2$
            sql = sql.replace("= " + GenericSQLHandler.UDI_INDICATOR_VALUE, "IS NULL");
        } else {
            // $NON-NLS-1$ //$NON-NLS-2$
            sql = sql.replace(GenericSQLHandler.UDI_INDICATOR_VALUE, "'" + key + "'");
        }
    }
    if (sql.indexOf(GenericSQLHandler.GROUP_BY_ALIAS) != -1) {
        sql = sql.replace(GenericSQLHandler.GROUP_BY_ALIAS, dbmsLanguage.quote(analyzedElement.getName()));
    }
    return sql;
}

Example 24 with UDIndicatorDefinition

use of org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition in project tdq-studio-se by Talend.

the class FrequencyStatisticsExplorer method getFreqRowsStatement.

protected String getFreqRowsStatement() {
    IndicatorDefinition indicatorDefinition = this.indicator.getIndicatorDefinition();
    if (indicatorDefinition instanceof UDIndicatorDefinition) {
        return getQueryForUDIndicator(indicatorDefinition);
    }
    String clause = PluginConstant.EMPTY_STRING;
    TdColumn column = (TdColumn) indicator.getAnalyzedElement();
    int javaType = column.getSqlDataType().getJavaDataType();
    if (Java2SqlType.isTextInSQL(javaType)) {
        clause = getInstantiatedClause();
    } else if (Java2SqlType.isDateInSQL(javaType)) {
        // MOD scorreia 2009-09-22 first check whether the value is null
        if (entity.isLabelNull()) {
            clause = getInstantiatedClause();
        } else {
            IndicatorParameters parameters = indicator.getParameters();
            if (parameters != null) {
                DateParameters dateParameters = parameters.getDateParameters();
                if (dateParameters != null) {
                    DateGrain dateGrain = dateParameters.getDateAggregationType();
                    clause = entity.isLabelNull() ? getInstantiatedClause() : getClauseWithDate(dateGrain, clause);
                } else {
                    clause = getInstantiatedClause();
                }
            } else {
                clause = getInstantiatedClause();
            }
        }
    } else if (Java2SqlType.isNumbericInSQL(javaType)) {
        IndicatorParameters parameters = indicator.getParameters();
        if (parameters != null) {
            // handle bins
            Domain bins = parameters.getBins();
            if (bins != null) {
                // rangeStrings = getBinsAsGenericString(bins.getRanges());
                final EList<RangeRestriction> ranges = bins.getRanges();
                for (RangeRestriction rangeRestriction : ranges) {
                    // find the rangeLabel
                    if (entity.getLabel() != null && entity.getLabel().equals(rangeRestriction.getName())) {
                        clause = createWhereClause(rangeRestriction);
                        break;
                    }
                }
            } else {
                // MOD hcheng 2009-05-18.Bug 7377,Frequency indicator,when bins is null,handle as textual data
                clause = getInstantiatedClause();
            }
        } else {
            // MOD scorreia 2009-05-13. Bug 7235
            // no parameter set: handle as textual data
            clause = getInstantiatedClause();
        }
    } else {
        // no quote here
        clause = getDefaultQuotedStatement(PluginConstant.EMPTY_STRING);
    }
    return // $NON-NLS-1$
    "SELECT * FROM " + getFullyQualifiedTableName(column) + dbmsLanguage.where() + inBrackets(clause) + andDataFilterClause();
}

Example 25 with UDIndicatorDefinition

use of org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition in project tdq-studio-se by Talend.

the class PatternExplorer method getValidValuesStatement.

/**
 * get the Valid Values Statement.
 *
 * @return SELECT statement for the valid Value of select column
 */
public String getValidValuesStatement() {
    // when the indicator is the use define match
    IndicatorDefinition indicatorDefinition = this.indicator.getIndicatorDefinition();
    if (indicatorDefinition instanceof UDIndicatorDefinition) {
        EList<TdExpression> list = ((UDIndicatorDefinition) indicatorDefinition).getViewValidValuesExpression();
        return getQueryAfterReplaced(indicatorDefinition, list);
    }
    String regexPatternString = dbmsLanguage.getRegexPatternString(this.indicator);
    String regexCmp = getRegexLike(regexPatternString);
    return getValuesStatement(columnName, regexCmp);
}