Search in sources :

Example 21 with Expression

use of orgomg.cwm.objectmodel.core.Expression in project tdq-studio-se by Talend.

the class UDIUtils method createIndicatorUnit.

public static IndicatorUnit[] createIndicatorUnit(IndicatorDefinition udid, ModelElementIndicator meIndicator, Analysis analysis) throws Throwable {
    List<IndicatorUnit> addIndicatorUnits = new ArrayList<IndicatorUnit>();
    // can't add the same user defined indicator
    for (Indicator indicator : meIndicator.getIndicators()) {
        // MOD xwang 2011-08-01 bug TDQ-2730
        if (udid.getName().equals(indicator.getName()) && indicator instanceof UserDefIndicator) {
            // $NON-NLS-1$
            MessageUI.openWarning(DefaultMessagesImpl.getString("UDIUtils.UDISelected", udid.getName()));
            return null;
        }
    }
    Indicator udi = UDIFactory.createUserDefIndicator(udid);
    udi.setIndicatorDefinition(udid);
    // MOD mzhao feature 11128, Handle Java User Defined Indicator.
    Indicator judi = UDIHelper.adaptToJavaUDI(udi);
    if (judi != null) {
        udi = judi;
    }
    IEditorPart activeEditor = CorePlugin.getDefault().getWorkbench().getActiveWorkbenchWindow().getActivePage().getActiveEditor();
    if (activeEditor == null || !(activeEditor instanceof AnalysisEditor)) {
        return null;
    }
    ExecutionLanguage executionLanguage = ((AnalysisEditor) activeEditor).getUIExecuteEngin();
    boolean isJavaEngin = ExecutionLanguage.JAVA.equals(executionLanguage);
    DbmsLanguage dbmsLanguage = DbmsLanguageFactory.createDbmsLanguage(analysis, executionLanguage);
    Expression returnExpression = dbmsLanguage.getExpression(udi);
    String executeType = isJavaEngin ? executionLanguage.getName() : dbmsLanguage.getDbmsName();
    // MOD qiongli 2013.5.22 TDQ-7282.if don't find a valid java expression for JUDI,should also pop this dialog.
    boolean finddExpression = true;
    if (isJavaEngin && judi == null || !isJavaEngin && returnExpression == null) {
        finddExpression = false;
    }
    if (!finddExpression) {
        // open the editor
        boolean openUDI = MessageDialog.openQuestion(PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(), DefaultMessagesImpl.getString("PatternUtilities.Warning"), // $NON-NLS-1$ //$NON-NLS-2$
        DefaultMessagesImpl.getString("UDIUtils.NoExpression", executeType));
        if (openUDI) {
            RepositoryNode node = RepositoryNodeHelper.recursiveFind(udid);
            if (RepositoryNodeHelper.canOpenEditor(node)) {
                new OpenItemEditorAction(new IRepositoryNode[] { node }).run();
            }
        }
        return null;
    }
    // dbmsLanguage
    IndicatorParameters parameters = udi.getParameters();
    if (parameters == null) {
        parameters = IndicatorsFactory.eINSTANCE.createIndicatorParameters();
        udi.setParameters(parameters);
    }
    Domain indicatorValidDomain = parameters.getIndicatorValidDomain();
    if (indicatorValidDomain == null) {
        // $NON-NLS-1$
        indicatorValidDomain = DomainHelper.createDomain("JAVA_UDI_PARAMETERS");
        parameters.setIndicatorValidDomain(indicatorValidDomain);
    }
    List<IndicatorDefinitionParameter> indicatorDefs = udid.getIndicatorDefinitionParameter();
    for (IndicatorDefinitionParameter idp : indicatorDefs) {
        JavaUDIIndicatorParameter judip = DomainHelper.createJavaUDIIndicatorParameter(idp.getKey(), idp.getValue());
        indicatorValidDomain.getJavaUDIIndicatorParameter().add(judip);
    }
    IndicatorEnum indicatorType = IndicatorEnum.findIndicatorEnum(udi.eClass());
    addIndicatorUnits.add(meIndicator.addSpecialIndicator(indicatorType, udi));
    DependenciesHandler.getInstance().setUsageDependencyOn(analysis, udid);
    return addIndicatorUnits.toArray(new IndicatorUnit[addIndicatorUnits.size()]);
}
Also used : IRepositoryNode(org.talend.repository.model.IRepositoryNode) IndicatorEnum(org.talend.dq.nodes.indicator.type.IndicatorEnum) IndicatorParameters(org.talend.dataquality.indicators.IndicatorParameters) IndicatorUnit(org.talend.dataprofiler.core.ui.editor.preview.IndicatorUnit) ArrayList(java.util.ArrayList) JavaUDIIndicatorParameter(org.talend.dataquality.domain.JavaUDIIndicatorParameter) ExecutionLanguage(org.talend.dataquality.analysis.ExecutionLanguage) IEditorPart(org.eclipse.ui.IEditorPart) DQRepositoryNode(org.talend.dq.nodes.DQRepositoryNode) RepositoryNode(org.talend.repository.model.RepositoryNode) IRepositoryNode(org.talend.repository.model.IRepositoryNode) OpenItemEditorAction(org.talend.dataprofiler.core.ui.action.actions.OpenItemEditorAction) ModelElementIndicator(org.talend.dataprofiler.core.model.ModelElementIndicator) UserDefIndicator(org.talend.dataquality.indicators.sql.UserDefIndicator) Indicator(org.talend.dataquality.indicators.Indicator) DbmsLanguage(org.talend.dq.dbms.DbmsLanguage) TdExpression(org.talend.cwm.relational.TdExpression) Expression(orgomg.cwm.objectmodel.core.Expression) IndicatorDefinitionParameter(org.talend.dataquality.indicators.definition.IndicatorDefinitionParameter) AnalysisEditor(org.talend.dataprofiler.core.ui.editor.analysis.AnalysisEditor) Domain(org.talend.dataquality.domain.Domain) UserDefIndicator(org.talend.dataquality.indicators.sql.UserDefIndicator)

Example 22 with Expression

use of orgomg.cwm.objectmodel.core.Expression in project tdq-studio-se by Talend.

the class ColumnAnalysisSqlParallelExecutor method run.

/**
 * run analysis when SqlParallelExecutor.
 *
 * @return true if successfull , false otherwise.
 */
public Boolean run() {
    Expression query = null;
    try {
        if (!continueRun()) {
            return Boolean.FALSE;
        }
        // skip composite indicators that do not require a sql execution
        if (indicator instanceof CompositeIndicator) {
            // options of composite indicators are handled elsewhere
            return Boolean.TRUE;
        }
        synchronized (schemata) {
            // set the connection's catalog
            if (belongToSameSchemata(indicator.getAnalyzedElement())) {
                if (needChangeCatalog(connection)) {
                    String catalogName = getCatalogOrSchemaName(indicator.getAnalyzedElement());
                    if (catalogName != null) {
                        // check whether null argument can be given
                        changeCatalog(catalogName, connection);
                    }
                }
            }
        }
        // add mapping of analyzed elements to their indicators
        synchronized (elementToIndicator) {
            addElements2IndicatorsMapping(elementToIndicator, indicator);
        }
        query = dbms().getInstantiatedExpression(indicator);
        if (query == null) {
            traceError(getErrorMessageForQuery(query));
            return Boolean.FALSE;
        }
        try {
            boolean execStatus = executeQuery(indicator, connection, query.getBody());
            if (!execStatus) {
                traceError(getErrorMessageForQuery(query));
                return Boolean.FALSE;
            }
        } catch (SQLException e) {
            log.error(e, e);
            traceError(getErrorMessageForQuery(query));
            return Boolean.FALSE;
        }
        // set computation done
        indicator.setComputed(true);
    } finally {
        // return the connection after run
        if (POOLED_CONNECTION) {
            TdqAnalysisConnectionPool.returnPooledConnection(cachedAnalysis, connection);
        }
    }
    return Boolean.TRUE;
}
Also used : Expression(orgomg.cwm.objectmodel.core.Expression) SQLException(java.sql.SQLException) CompositeIndicator(org.talend.dataquality.indicators.CompositeIndicator)

Example 23 with Expression

use of orgomg.cwm.objectmodel.core.Expression in project tdq-studio-se by Talend.

the class UpdateCoreSwitch method caseExpression.

@Override
public Boolean caseExpression(Expression object) {
    if (recentElement instanceof Expression) {
        Expression expr = (Expression) recentElement;
        object.setBody(expr.getBody());
        object.setLanguage(expr.getLanguage());
        return true;
    }
    return false;
}
Also used : Expression(orgomg.cwm.objectmodel.core.Expression)

Example 24 with Expression

use of orgomg.cwm.objectmodel.core.Expression in project tdq-studio-se by Talend.

the class MultiColumnAnalysisExecutor method instantiateQuery.

/**
 * DOC scorreia Comment method "instantiateQuery".
 *
 * @param indicator
 */
private void instantiateQuery(Indicator indicator) {
    if (ColumnsetPackage.eINSTANCE.getColumnSetMultiValueIndicator().isSuperTypeOf(indicator.eClass())) {
        ColumnSetMultiValueIndicator colSetMultValIndicator = (ColumnSetMultiValueIndicator) indicator;
        final EList<ModelElement> analyzedColumns = colSetMultValIndicator.getAnalyzedColumns();
        final EList<String> numericFunctions = initializeNumericFunctions(colSetMultValIndicator);
        final EList<String> dateFunctions = initializeDateFunctions(colSetMultValIndicator);
        // ADD msjian 2011-5-30 17479: Excel Odbc connection can not run well on the correlation analysis
        // note: this feature is not supported now, if support, delete this
        // $NON-NLS-1$
        final String caseStr = "SUM(CASE WHEN {0} IS NULL THEN 1 ELSE 0 END)";
        if (// $NON-NLS-1$
        "EXCEL".equals(dbms().getDbmsName()) && (dateFunctions.contains(caseStr) || numericFunctions.contains(caseStr))) {
            // $NON-NLS-1$
            setError(Messages.getString("MultiColumnAnalysisExecutor.errMessage"));
            Display.getDefault().syncExec(new Runnable() {

                public void run() {
                    MessageDialog.openWarning(PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell(), Messages.getString("MultiColumnAnalysisExecutor.errTitle"), // $NON-NLS-1$
                    getErrorMessage());
                    return;
                }
            });
        }
        // ~
        // get indicator definition
        final Expression sqlGenericExpression = dbms().getSqlExpression(indicator.getIndicatorDefinition());
        // separate nominal from numeric columns
        List<String> nominalColumns = new ArrayList<String>();
        for (ModelElement column : colSetMultValIndicator.getNominalColumns()) {
            nominalColumns.add(getQuotedColumnName(column));
        }
        List<String> computedColumns = new ArrayList<String>();
        for (ModelElement column : colSetMultValIndicator.getNumericColumns()) {
            // call functions for each column
            for (String f : numericFunctions) {
                computedColumns.add(replaceVariablesLow(f, getQuotedColumnName(column)));
            }
        }
        for (ModelElement column : colSetMultValIndicator.getDateColumns()) {
            // call functions for each column
            for (String f : dateFunctions) {
                computedColumns.add(replaceVariablesLow(f, getQuotedColumnName(column)));
            }
        }
        // add count(*)
        computedColumns.add(colSetMultValIndicator.getCountAll());
        // MOD msjian TDQ-7254: fix the columnset analysis run get error. the columnset analysis don't need to
        // consider the datamining type.
        List<String> columns = new ArrayList<String>();
        // if the analysis type is columnset, use analyzed columns
        if (AnalysisType.COLUMN_SET == cachedAnalysis.getParameters().getAnalysisType()) {
            for (ModelElement column : analyzedColumns) {
                columns.add(getQuotedColumnName(column));
            }
        } else {
            columns = nominalColumns;
        }
        // TDQ-7254~
        String selectItems = createSelect(columns, computedColumns);
        String grpByClause = createGroupBy(columns);
        // all columns must belong to the same table
        TdColumn firstColumn = SwitchHelpers.COLUMN_SWITCH.doSwitch(analyzedColumns.get(0));
        String tableName = dbms().getQueryColumnSetWithPrefix(firstColumn);
        this.catalogOrSchema = dbms().getCatalogOrSchemaName(firstColumn);
        // definition is SELECT &lt;%=__COLUMN_NAMES__%> FROM &lt;%=__TABLE_NAME__%> GROUP BY
        // &lt;%=__GROUP_BY_ALIAS__%>
        String sqlExpr = dbms().fillGenericQueryWithColumnTableAndAlias(sqlGenericExpression.getBody(), selectItems, tableName, grpByClause);
        // handle data filter
        String stringDataFilter = ContextHelper.getDataFilterWithoutContext(cachedAnalysis);
        if (stringDataFilter == null) {
            stringDataFilter = PluginConstant.EMPTY_STRING;
        }
        sqlExpr = dbms().addWhereToStatement(sqlExpr, stringDataFilter);
        indicator.setInstantiatedExpression(BooleanExpressionHelper.createTdExpression(sqlGenericExpression.getLanguage(), sqlExpr));
        // MOD qiongli 2011-3-30 feature 19192.allow drill down for sql engine.
        if (ColumnsetPackage.eINSTANCE.getSimpleStatIndicator().isSuperTypeOf(indicator.eClass())) {
            SimpleStatIndicator simpleIndicator = (SimpleStatIndicator) indicator;
            // MOD TDQ-7287 lost some columns(type!=norminal) when view values in column set ana. yyin 20130514
            String columnsName = createSelect(columns, new ArrayList<String>());
            for (Indicator leafIndicator : simpleIndicator.getLeafIndicators()) {
                final Expression leafSqlGenericExpression = dbms().getSqlExpression(leafIndicator.getIndicatorDefinition());
                String leafSqlExpr = dbms().fillGenericQueryWithColumnTableAndAlias(leafSqlGenericExpression.getBody(), columnsName, tableName, grpByClause);
                leafSqlExpr = dbms().addWhereToStatement(leafSqlExpr, stringDataFilter);
                leafIndicator.setInstantiatedExpression(BooleanExpressionHelper.createTdExpression(leafSqlGenericExpression.getLanguage(), leafSqlExpr));
            }
        }
    }
}
Also used : ArrayList(java.util.ArrayList) ColumnSetMultiValueIndicator(org.talend.dataquality.indicators.columnset.ColumnSetMultiValueIndicator) ColumnSetMultiValueIndicator(org.talend.dataquality.indicators.columnset.ColumnSetMultiValueIndicator) Indicator(org.talend.dataquality.indicators.Indicator) AllMatchIndicator(org.talend.dataquality.indicators.columnset.AllMatchIndicator) SimpleStatIndicator(org.talend.dataquality.indicators.columnset.SimpleStatIndicator) ModelElement(orgomg.cwm.objectmodel.core.ModelElement) TdColumn(org.talend.cwm.relational.TdColumn) Expression(orgomg.cwm.objectmodel.core.Expression) SimpleStatIndicator(org.talend.dataquality.indicators.columnset.SimpleStatIndicator)

Example 25 with Expression

use of orgomg.cwm.objectmodel.core.Expression in project tdq-studio-se by Talend.

the class TableAnalysisSqlExecutor method executeIndicator.

private boolean executeIndicator(Indicator indicator, Connection connection) {
    // set the connection's catalog
    String catalogName = getCatalogOrSchemaName(indicator.getAnalyzedElement());
    if (catalogName != null && needChangeCatalog(connection)) {
        // check whether null argument can be given
        changeCatalog(catalogName, connection);
    }
    Expression query = dbms().getInstantiatedExpression(indicator);
    if (query == null) {
        traceError(// $NON-NLS-1$//$NON-NLS-2$
        "Query not executed for indicator: \"" + AnalysisExecutorHelper.getIndicatorName(indicator) + "\" " + // $NON-NLS-1$
        "query is null");
        return Boolean.FALSE;
    } else {
        try {
            // Boolean isExecSuccess = executeQuery(indicator, connection, query.getBody());
            List<Object[]> myResultSet = executeQuery(catalogName, connection, query.getBody());
            // give result to indicator so that it handles the results
            Boolean isExecSuccess = indicator.storeSqlResults(myResultSet);
            if (!isExecSuccess) {
                traceError(// $NON-NLS-1$//$NON-NLS-2$
                "Query not executed for indicator: \"" + AnalysisExecutorHelper.getIndicatorName(indicator) + "\" " + "SQL query: " + // $NON-NLS-1$
                query.getBody());
                return Boolean.FALSE;
            }
        } catch (Exception e) {
            traceError(// $NON-NLS-1$//$NON-NLS-2$
            "Query not executed for indicator: \"" + AnalysisExecutorHelper.getIndicatorName(indicator) + "\" " + "SQL query: " + query.getBody() + ". Exception: " + // $NON-NLS-1$ //$NON-NLS-2$
            e.getMessage());
            return Boolean.FALSE;
        }
    }
    indicator.setComputed(true);
    return Boolean.TRUE;
}
Also used : TdExpression(org.talend.cwm.relational.TdExpression) Expression(orgomg.cwm.objectmodel.core.Expression) AnalysisExecutionException(org.talend.cwm.exception.AnalysisExecutionException) SQLException(java.sql.SQLException)

Aggregations

Expression (orgomg.cwm.objectmodel.core.Expression)71 TdExpression (org.talend.cwm.relational.TdExpression)42 RegularExpression (org.talend.dataquality.domain.pattern.RegularExpression)37 Test (org.junit.Test)25 Domain (org.talend.dataquality.domain.Domain)14 ArrayList (java.util.ArrayList)13 TdColumn (org.talend.cwm.relational.TdColumn)11 Pattern (org.talend.dataquality.domain.pattern.Pattern)10 Catalog (orgomg.cwm.resource.relational.Catalog)10 Indicator (org.talend.dataquality.indicators.Indicator)9 IndicatorParameters (org.talend.dataquality.indicators.IndicatorParameters)8 Analysis (org.talend.dataquality.analysis.Analysis)7 PatternMatchingIndicator (org.talend.dataquality.indicators.PatternMatchingIndicator)7 UDIndicatorDefinition (org.talend.dataquality.indicators.definition.userdefine.UDIndicatorDefinition)7 DbmsLanguage (org.talend.dq.dbms.DbmsLanguage)7 Connection (org.talend.core.model.metadata.builder.connection.Connection)6 DatabaseConnection (org.talend.core.model.metadata.builder.connection.DatabaseConnection)6 AnalysisContext (org.talend.dataquality.analysis.AnalysisContext)6 IndicatorDefinition (org.talend.dataquality.indicators.definition.IndicatorDefinition)6 ChartDataEntity (org.talend.dq.indicators.preview.table.ChartDataEntity)6