Examples with DefinitionHandler org.talend.dq.indicators.definitions.DefinitionHandler used on opensource projects

Example 1 with DefinitionHandler

use of org.talend.dq.indicators.definitions.DefinitionHandler in project tdq-studio-se by Talend.

the class UpdateIndicatorsForVertica method doExecute.

/**
 * migrate these indicators for Vertica database:Pattern Frequency Table,Pattern Low Frequency Table,Regular
 * Expression Matching.
 */
@Override
protected boolean doExecute() throws Exception {
    DefinitionHandler definitionHandler = DefinitionHandler.getInstance();
    IndicatorDefinition patternMatchIndDef = definitionHandler.getDefinitionById(patternMatchIndiDefId);
    IndicatorDefinition patternFreqIndDef = definitionHandler.getDefinitionById(patternFreqIndiDefId);
    IndicatorDefinition patternLowPatternFreqIndDef = definitionHandler.getDefinitionById(lowPatternFreqIndiDefId);
    IndicatorDefinition benFordIndDef = definitionHandler.getDefinitionById(benfordId);
    boolean isNeedReload = false;
    if (patternMatchIndDef != null && !IndicatorDefinitionFileHelper.isExistSqlExprWithLanguage(patternMatchIndDef, language)) {
        IndicatorDefinitionFileHelper.addSqlExpression(patternMatchIndDef, language, patternMatchSQL);
        IndicatorDefinitionFileHelper.save(patternMatchIndDef);
        isNeedReload = true;
    }
    if (patternFreqIndDef != null && !IndicatorDefinitionFileHelper.isExistSqlExprWithLanguage(patternFreqIndDef, language)) {
        IndicatorDefinitionFileHelper.addSqlExpression(patternFreqIndDef, language, patternFreqSQL);
        IndicatorDefinitionFileHelper.addCharacterMapping(patternFreqIndDef, language, name, charToReplace, newReplaceChar);
        IndicatorDefinitionFileHelper.save(patternFreqIndDef);
        if (!isNeedReload) {
            isNeedReload = true;
        }
    }
    if (patternLowPatternFreqIndDef != null && !IndicatorDefinitionFileHelper.isExistSqlExprWithLanguage(patternLowPatternFreqIndDef, language)) {
        IndicatorDefinitionFileHelper.addSqlExpression(patternLowPatternFreqIndDef, language, lowPatternFreqSQL);
        IndicatorDefinitionFileHelper.addCharacterMapping(patternLowPatternFreqIndDef, language, name, charToReplace, newReplaceChar);
        IndicatorDefinitionFileHelper.save(patternLowPatternFreqIndDef);
        if (!isNeedReload) {
            isNeedReload = true;
        }
    }
    if (benFordIndDef != null && !IndicatorDefinitionFileHelper.isExistSqlExprWithLanguage(benFordIndDef, language)) {
        IndicatorDefinitionFileHelper.addSqlExpression(benFordIndDef, language, benFordSQL);
        IndicatorDefinitionFileHelper.save(benFordIndDef);
        if (!isNeedReload) {
            isNeedReload = true;
        }
    }
    if (isNeedReload) {
        definitionHandler.reloadIndicatorsDefinitions();
    }
    return true;
}

Example 2 with DefinitionHandler

use of org.talend.dq.indicators.definitions.DefinitionHandler in project tdq-studio-se by Talend.

the class AddHiveCharMapInPatternFrenquencyTask method doExecute.

/*
     * (non-Javadoc)
     * 
     * @see org.talend.dataprofiler.core.migration.AMigrationTask#doExecute()
     */
@Override
protected boolean doExecute() throws Exception {
    DefinitionHandler definitionHandler = DefinitionHandler.getInstance();
    boolean isFrequencyAdded = true;
    boolean isLowFrequencyAdded = true;
    // Add CharacterMapping of hive to pattern frenquency indicator
    IndicatorDefinition hiveDefinition = definitionHandler.getDefinitionById(PATTERN_FREQUENCY_UUID);
    if (hiveDefinition != null && !IndicatorDefinitionFileHelper.isExistCharacterMappingWithLanguage(hiveDefinition, hive)) {
        IndicatorDefinitionFileHelper.addCharacterMapping(hiveDefinition, hive, CHAR_NAME, CHAR_TOREPLACE, CHAR_REPLACE);
        isFrequencyAdded = IndicatorDefinitionFileHelper.save(hiveDefinition);
    }
    // Update CharacterMapping of hive to pattern low frenquency indicator
    IndicatorDefinition lowDefinition = definitionHandler.getDefinitionById(PATTERN_LOW_FREQUENCY_UUID);
    if (hiveDefinition != null && !IndicatorDefinitionFileHelper.isExistCharacterMappingWithLanguage(lowDefinition, hive)) {
        IndicatorDefinitionFileHelper.addCharacterMapping(lowDefinition, hive, CHAR_NAME, CHAR_TOREPLACE, CHAR_REPLACE);
        isLowFrequencyAdded = IndicatorDefinitionFileHelper.save(lowDefinition);
    }
    DefinitionHandler.getInstance().reloadIndicatorsDefinitions();
    if (!isFrequencyAdded || !isLowFrequencyAdded) {
        return false;
    }
    return true;
}

Example 3 with DefinitionHandler

use of org.talend.dq.indicators.definitions.DefinitionHandler in project tdq-studio-se by Talend.

the class DQStructureManager method createDQStructureUnit.

/**
 * DOC zhao Comment method "createDQStructureUnit".
 */
private void createDQStructureUnit() {
    RepositoryNodeBuilder instance = RepositoryNodeBuilder.getInstance();
    Plugin plugin = CorePlugin.getDefault();
    try {
        IProject project = ResourceManager.getRootProject();
        if (!project.exists()) {
            project = createNewProject(ResourceManager.getRootProjectName());
        }
        if (!project.getFolder(EResourceConstant.DB_CONNECTIONS.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.METADATA, Path.EMPTY, EResourceConstant.DB_CONNECTIONS.getName());
        }
        if (!project.getFolder(EResourceConstant.FILEDELIMITED.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.METADATA, Path.EMPTY, EResourceConstant.FILEDELIMITED.getName());
        }
        if (!project.getFolder(EResourceConstant.ANALYSIS.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_DATA_PROFILING, Path.EMPTY, EResourceConstant.ANALYSIS.getName());
        }
        if (PluginChecker.isTDQLoaded()) {
            if (!project.getFolder(EResourceConstant.REPORTS.getPath()).exists()) {
                ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_DATA_PROFILING, Path.EMPTY, EResourceConstant.REPORTS.getName());
            }
            // Added 20150421 TDQ-9605
            if (!project.getFolder(EResourceConstant.HADOOP_CLUSTER.getPath()).exists() && HadoopClusterUtils.getDefault().isServiceInstalled()) {
                ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.METADATA, Path.EMPTY, EResourceConstant.HADOOP_CLUSTER.getName());
            }
        }
        if (!project.getFolder(EResourceConstant.PATTERNS.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.PATTERNS.getName());
        }
        if (!project.getFolder(EResourceConstant.RULES.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.RULES.getName());
        }
        Folder rulesSQLFoler = null;
        if (!project.getFolder(EResourceConstant.RULES_SQL.getPath()).exists()) {
            rulesSQLFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_RULES, Path.EMPTY, EResourceConstant.RULES_SQL.getName());
        } else {
            rulesSQLFoler = instance.getObjectFolder(EResourceConstant.RULES_SQL);
        }
        rulesSQLFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_RULES_SQL.getFolder());
        Folder rulesMatchLFoler = null;
        if (!project.getFolder(EResourceConstant.RULES_MATCHER.getPath()).exists()) {
            rulesMatchLFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_RULES, Path.EMPTY, EResourceConstant.RULES_MATCHER.getName());
        } else {
            rulesMatchLFoler = instance.getObjectFolder(EResourceConstant.RULES_MATCHER);
        }
        rulesMatchLFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_RULES_MATCHER.getFolder());
        Folder rulesParserFoler = null;
        if (!project.getFolder(EResourceConstant.RULES_PARSER.getPath()).exists()) {
            if (PluginChecker.isTDQLoaded()) {
                rulesParserFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_RULES, Path.EMPTY, EResourceConstant.RULES_PARSER.getName());
            }
        } else {
            rulesParserFoler = instance.getObjectFolder(EResourceConstant.RULES_PARSER);
        }
        if (rulesParserFoler != null) {
            rulesParserFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_RULES_PARSER.getFolder());
        }
        if (!project.getFolder(EResourceConstant.EXCHANGE.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.EXCHANGE.getName());
        }
        if (!project.getFolder(EResourceConstant.INDICATORS.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.INDICATORS.getName());
        }
        Folder systemIndicatorFoler = null;
        if (!project.getFolder(EResourceConstant.SYSTEM_INDICATORS.getPath()).exists()) {
            systemIndicatorFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_INDICATOR_ELEMENT, Path.EMPTY, EResourceConstant.SYSTEM_INDICATORS.getName());
        } else {
            systemIndicatorFoler = instance.getObjectFolder(EResourceConstant.SYSTEM_INDICATORS);
        }
        systemIndicatorFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_SYSTEM_INDICATORS.getFolder());
        if (!project.getFolder(EResourceConstant.USER_DEFINED_INDICATORS.getPath()).exists()) {
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_INDICATOR_ELEMENT, Path.EMPTY, EResourceConstant.USER_DEFINED_INDICATORS.getName());
        }
        if (!project.getFolder(EResourceConstant.USER_DEFINED_INDICATORS_LIB.getPath()).exists()) {
            // MOD zhsne 18724: Java UDI enhancements add lib folder under UDI folder.
            ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_USERDEFINE_INDICATORS, Path.EMPTY, EResourceConstant.USER_DEFINED_INDICATORS_LIB.getName());
        }
        if (PluginChecker.isTDQLoaded()) {
            if (!project.getFolder(EResourceConstant.JRXML_TEMPLATE.getPath()).exists()) {
                ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.JRXML_TEMPLATE.getName());
            }
            if (!project.getFolder(EResourceConstant.CONTEXT.getPath()).exists()) {
                ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.CONTEXT, Path.EMPTY, EResourceConstant.CONTEXT.getName());
            }
        }
        Folder patternRegexFoler = null;
        if (!project.getFolder(EResourceConstant.PATTERN_REGEX.getPath()).exists()) {
            patternRegexFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_PATTERN_ELEMENT, Path.EMPTY, EResourceConstant.PATTERN_REGEX.getName());
        } else {
            patternRegexFoler = instance.getObjectFolder(EResourceConstant.PATTERN_REGEX);
        }
        patternRegexFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_PATTERN_REGEX.getFolder());
        Folder patternSQLFoler = null;
        if (!project.getFolder(EResourceConstant.PATTERN_SQL.getPath()).exists()) {
            patternSQLFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_PATTERN_ELEMENT, Path.EMPTY, EResourceConstant.PATTERN_SQL.getName());
        } else {
            patternSQLFoler = instance.getObjectFolder(EResourceConstant.PATTERN_SQL);
        }
        patternSQLFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_PATTERN_SQL.getFolder());
        Folder sourceFileFoler = null;
        if (!project.getFolder(EResourceConstant.SOURCE_FILES.getPath()).exists()) {
            sourceFileFoler = ProxyRepositoryFactory.getInstance().createFolder(ERepositoryObjectType.TDQ_LIBRARIES, Path.EMPTY, EResourceConstant.SOURCE_FILES.getName());
        } else {
            sourceFileFoler = instance.getObjectFolder(EResourceConstant.SOURCE_FILES);
        }
        sourceFileFoler.getProperty().getItem().getState().setPath(ERepositoryObjectType.TDQ_SOURCE_FILE_ELEMENT.getFolder());
        // use the tos create folder API
        if (systemIndicatorFoler != null && patternRegexFoler != null && patternSQLFoler != null && sourceFileFoler != null && rulesSQLFoler != null) {
            copyFilesToFolder(plugin, SYSTEM_INDICATOR_PATH, true, systemIndicatorFoler, null, ERepositoryObjectType.TDQ_SYSTEM_INDICATORS);
            copyFilesToFolder(plugin, PATTERN_PATH, true, patternRegexFoler, null, ERepositoryObjectType.TDQ_PATTERN_REGEX);
            copyFilesToFolder(plugin, SQL_LIKE_PATH, true, patternSQLFoler, null, ERepositoryObjectType.TDQ_PATTERN_SQL);
            copyFilesToFolder(plugin, DEMO_PATH, true, sourceFileFoler, null, ERepositoryObjectType.TDQ_SOURCE_FILE_ELEMENT);
            copyFilesToFolder(plugin, RULES_PATH, true, rulesSQLFoler, null, ERepositoryObjectType.TDQ_RULES_SQL);
        }
        if (rulesParserFoler != null) {
            copyFilesToFolder(plugin, RULES_PARSER, true, rulesParserFoler, null, ERepositoryObjectType.TDQ_RULES_PARSER);
        }
        WorkspaceVersionHelper.storeVersion();
        // MOD qiongli 2011-12-6 TDQ-4095,make the definition file commit to svn with the DQ struct files.move
        // this code from DQRespositoryView.initWorkspace()
        IFile defFile = ResourceManager.getLibrariesFolder().getFile(DefinitionHandler.FILENAME);
        if (!defFile.exists()) {
            DefinitionHandler handler = DefinitionHandler.getInstance();
            // TDQ-4714 update the aggregateDefinitions tag for some indicator definition.
            handler.updateAggregates();
        }
        ResourceService.refreshStructure();
    } catch (Exception ex) {
        ExceptionHandler.process(ex);
        ProxyRepositoryManager.getInstance().save();
    }
}

Example 4 with DefinitionHandler

use of org.talend.dq.indicators.definitions.DefinitionHandler in project tdq-studio-se by Talend.

the class RemoveMDMFromIndicatorsTask method doExecute.

@Override
protected boolean doExecute() throws Exception {
    DefinitionHandler definitionHandler = DefinitionHandler.getInstance();
    // ROW COUNT
    IndicatorDefinition definition = definitionHandler.getDefinitionById(ROW_COUNT_UUID);
    if (definition != null) {
        IndicatorDefinitionFileHelper.removeSqlExpression(definition, LANGUAGE);
        IndicatorDefinitionFileHelper.save(definition);
    }
    definition = definitionHandler.getDefinitionById(NULL_COUNT_UUID);
    if (definition != null) {
        IndicatorDefinitionFileHelper.removeSqlExpression(definition, LANGUAGE);
        IndicatorDefinitionFileHelper.save(definition);
    }
    definition = definitionHandler.getDefinitionById(UNIQUE_COUNT_UUID);
    if (definition != null) {
        IndicatorDefinitionFileHelper.removeSqlExpression(definition, LANGUAGE);
        IndicatorDefinitionFileHelper.save(definition);
    }
    definition = definitionHandler.getDefinitionById(DISTINCT_COUNT_UUID);
    if (definition != null) {
        IndicatorDefinitionFileHelper.removeSqlExpression(definition, LANGUAGE);
        IndicatorDefinitionFileHelper.save(definition);
    }
    definition = definitionHandler.getDefinitionById(DUPLICATE_COUNT_UUID);
    if (definition != null) {
        IndicatorDefinitionFileHelper.removeSqlExpression(definition, LANGUAGE);
        IndicatorDefinitionFileHelper.save(definition);
    }
    return true;
}

Example 5 with DefinitionHandler

use of org.talend.dq.indicators.definitions.DefinitionHandler in project tdq-studio-se by Talend.

the class UpdateMeanMedianForDB2Task method doExecute.

/**
 * update Mean/Median indicators definition by uuid.
 */
@Override
protected boolean doExecute() throws Exception {
    DefinitionHandler definitionHandler = DefinitionHandler.getInstance();
    boolean isMeanAdded = true;
    boolean isMedianUpdated = true;
    // Add sql of DB2 to Mean indicator
    IndicatorDefinition meanDefinition = definitionHandler.getDefinitionById(MEAN_UUID);
    if (meanDefinition != null && !IndicatorDefinitionFileHelper.isExistSqlExprWithLanguage(meanDefinition, DB2)) {
        IndicatorDefinitionFileHelper.addSqlExpression(meanDefinition, DB2, // $NON-NLS-1$
        "SELECT SUM(double(<%=__COLUMN_NAMES__%>)), COUNT(<%=__COLUMN_NAMES__%>) FROM <%=__TABLE_NAME__%> <%=__WHERE_CLAUSE__%>");
        isMeanAdded = IndicatorDefinitionFileHelper.save(meanDefinition);
    }
    // Update sql of DB2 to Median indicator
    IndicatorDefinition medianDefinition = definitionHandler.getDefinitionById(MEDIAN_UUID);
    if (medianDefinition != null && IndicatorDefinitionFileHelper.removeSqlExpression(medianDefinition, DB2)) {
        List<TdExpression> remainExpLs = new ArrayList<TdExpression>();
        remainExpLs.addAll(medianDefinition.getSqlGenericExpression());
        medianDefinition.getSqlGenericExpression().clear();
        IndicatorDefinitionFileHelper.addSqlExpression(medianDefinition, DB2, // $NON-NLS-1$
        "SELECT AVG(double(<%=__COLUMN_NAMES__%>)) FROM ( SELECT <%=__COLUMN_NAMES__%>, COUNT(*) OVER( ) total, CAST(COUNT(*) OVER( ) AS DECIMAL)/2 mid, CEIL(CAST(COUNT(*) OVER( ) AS DECIMAL)/2) next, ROW_NUMBER() OVER ( ORDER BY <%=__COLUMN_NAMES__%>) rn FROM <%=__TABLE_NAME__%> WHERE <%=__COLUMN_NAMES__%> IS NOT NULL <%=__AND_WHERE_CLAUSE__%>) x WHERE ( MOD(total,2) = 0 AND rn IN ( mid, mid+1 ) ) OR ( MOD(total,2) = 1 AND rn = next )");
        medianDefinition.getSqlGenericExpression().addAll(remainExpLs);
        isMedianUpdated = IndicatorDefinitionFileHelper.save(medianDefinition);
    }
    DefinitionHandler.getInstance().reloadIndicatorsDefinitions();
    if (!isMeanAdded || !isMedianUpdated) {
        return false;
    }
    return true;
}