Examples with ExecutionInformations org.talend.dataquality.analysis.ExecutionInformations used on opensource projects

Example 11 with ExecutionInformations

use of org.talend.dataquality.analysis.ExecutionInformations in project tdq-studio-se by Talend.

the class MatchAnalysisExecutor method execute.

/*
     * (non-Javadoc)
     * 
     * @see org.talend.dq.analysis.IAnalysisExecutor#execute(org.talend.dataquality.analysis.Analysis)
     */
public ReturnCode execute(Analysis analysis) {
    assert analysis != null;
    // --- preconditions
    ReturnCode rc = AnalysisExecutorHelper.check(analysis);
    if (!rc.isOk()) {
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    // --- creation time
    final long startime = AnalysisExecutorHelper.setExecutionDateInAnalysisResult(analysis);
    EList<Indicator> indicators = analysis.getResults().getIndicators();
    RecordMatchingIndicator recordMatchingIndicator = null;
    BlockKeyIndicator blockKeyIndicator = null;
    for (Indicator ind : indicators) {
        if (ind instanceof RecordMatchingIndicator) {
            recordMatchingIndicator = (RecordMatchingIndicator) ind;
        } else if (ind instanceof BlockKeyIndicator) {
            blockKeyIndicator = (BlockKeyIndicator) ind;
        }
    }
    if (recordMatchingIndicator == null || blockKeyIndicator == null) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.noIndicators"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    List<ModelElement> anlayzedElements = analysis.getContext().getAnalysedElements();
    if (anlayzedElements == null || anlayzedElements.size() == 0) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.EmptyAnalyzedElement"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    // TDQ-9664 msjian: check the store on disk path.
    Boolean isStoreOnDisk = TaggedValueHelper.getValueBoolean(SQLExecutor.STORE_ON_DISK_KEY, analysis);
    if (isStoreOnDisk) {
        String tempDataPath = TaggedValueHelper.getValueString(SQLExecutor.TEMP_DATA_DIR, analysis);
        File file = new File(tempDataPath);
        if (!file.exists() || !file.isDirectory()) {
            rc.setOk(Boolean.FALSE);
            // $NON-NLS-1$
            rc.setMessage(Messages.getString("MatchAnalysisExecutor.InvalidPath", file.getPath()));
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
    }
    // TDQ-9664~
    Map<MetadataColumn, String> columnMap = getColumn2IndexMap(anlayzedElements);
    ISQLExecutor sqlExecutor = getSQLExectutor(analysis, recordMatchingIndicator, columnMap);
    if (sqlExecutor == null) {
        rc.setOk(Boolean.FALSE);
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("MatchAnalysisExecutor.noSqlExecutor"));
        AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
        return rc;
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    // Set schema for match key.
    TypedReturnCode<MatchGroupResultConsumer> returnCode = new TypedReturnCode<MatchGroupResultConsumer>();
    MetadataColumn[] completeColumnSchema = AnalysisRecordGroupingUtils.getCompleteColumnSchema(columnMap);
    String[] colSchemaString = new String[completeColumnSchema.length];
    int idx = 0;
    for (MetadataColumn metadataCol : completeColumnSchema) {
        colSchemaString[idx++] = metadataCol.getName();
    }
    recordMatchingIndicator.setMatchRowSchema(colSchemaString);
    recordMatchingIndicator.reset();
    MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(recordMatchingIndicator);
    if (sqlExecutor.isStoreOnDisk()) {
        // need to execute the query
        try {
            sqlExecutor.executeQuery(analysis.getContext().getConnection(), analysis.getContext().getAnalysedElements());
        } catch (SQLException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
        try {
            TypedReturnCode<Object> result = StoreOnDiskUtils.getDefault().executeWithStoreOnDisk(columnMap, recordMatchingIndicator, blockKeyIndicator, sqlExecutor.getStoreOnDiskHandler(), matchResultConsumer);
            if (result != null) {
                returnCode.setObject((MatchGroupResultConsumer) result.getObject());
                returnCode.setOk(result.isOk());
                returnCode.setMessage(result.getMessage());
            }
        } catch (Exception e) {
            log.error(e, e);
            returnCode.setMessage(e.getMessage());
            returnCode.setOk(false);
        }
    } else {
        // Added TDQ-9320 , use the result set iterator to replace the list of result in the memory.
        try {
            Iterator<Record> resultSetIterator = sqlExecutor.getResultSetIterator(analysis.getContext().getConnection(), anlayzedElements);
            BlockAndMatchManager bAndmManager = new BlockAndMatchManager(resultSetIterator, matchResultConsumer, columnMap, recordMatchingIndicator, blockKeyIndicator);
            bAndmManager.run();
        } catch (SQLException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        } catch (BusinessException e) {
            log.error(e, e);
            rc.setOk(Boolean.FALSE);
            rc.setMessage(e.getMessage());
            AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
            return rc;
        }
    }
    if (!returnCode.isOk()) {
        rc.setOk(returnCode.isOk());
        rc.setMessage(returnCode.getMessage());
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    if (isLowMemory) {
        // $NON-NLS-1$
        rc.setMessage(Messages.getString("Evaluator.OutOfMomory", usedMemory));
    }
    // nodify the master page
    refreshTableWithMatchFullResult(analysis);
    // --- set metadata information of analysis
    AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
    // --- compute execution duration
    if (this.continueRun()) {
        long endtime = System.currentTimeMillis();
        final ExecutionInformations resultMetadata = analysis.getResults().getResultMetadata();
        resultMetadata.setExecutionDuration((int) (endtime - startime));
        resultMetadata.setOutThreshold(false);
    }
    if (getMonitor() != null) {
        getMonitor().worked(20);
    }
    return rc;
}

Example 12 with ExecutionInformations

use of org.talend.dataquality.analysis.ExecutionInformations in project tdq-studio-se by Talend.

the class TdReportImpl method basicSetExecInformations.

/**
 * <!-- begin-user-doc --> <!-- end-user-doc -->
 * @generated
 */
public NotificationChain basicSetExecInformations(ExecutionInformations newExecInformations, NotificationChain msgs) {
    ExecutionInformations oldExecInformations = execInformations;
    execInformations = newExecInformations;
    if (eNotificationRequired()) {
        ENotificationImpl notification = new ENotificationImpl(this, Notification.SET, ReportsPackage.TD_REPORT__EXEC_INFORMATIONS, oldExecInformations, newExecInformations);
        if (msgs == null)
            msgs = notification;
        else
            msgs.add(notification);
    }
    return msgs;
}

Example 13 with ExecutionInformations

use of org.talend.dataquality.analysis.ExecutionInformations in project tdq-studio-se by Talend.

the class TdqAnalysisConnectionPoolTest method setUp.

/**
 * DOC yyin Comment method "setUp".
 *
 * @throws java.lang.Exception
 */
@Before
public void setUp() throws Exception {
    analysis = mock(Analysis.class);
    AnalysisContext context = mock(AnalysisContext.class);
    AnalysisResult result = mock(AnalysisResult.class);
    dataManager = mock(org.talend.core.model.metadata.builder.connection.Connection.class);
    ExecutionInformations resultMetadata = mock(ExecutionInformations.class);
    PowerMockito.mockStatic(AnalysisHandler.class);
    AnalysisHandler mockHandler = mock(AnalysisHandler.class);
    when(AnalysisHandler.createHandler(analysis)).thenReturn(mockHandler);
    when(mockHandler.getNumberOfConnectionsPerAnalysis()).thenReturn(5);
    when(context.getConnection()).thenReturn(dataManager);
    when(analysis.getContext()).thenReturn(context);
    when(analysis.getResults()).thenReturn(result);
    when(result.getResultMetadata()).thenReturn(resultMetadata);
    connPool = new TdqAnalysisConnectionPool(analysis, 5);
}

Example 14 with ExecutionInformations

use of org.talend.dataquality.analysis.ExecutionInformations in project tdq-studio-se by Talend.

the class FrequencyStatisticsExplorerTest method testGetFreqRowsStatement_4.

/**
 * Test method for {@link org.talend.dq.analysis.explore.FrequencyStatisticsExplorer#getFreqRowsStatement()} case_3:
 * for UDIndicatorDefinition
 */
@Test
public void testGetFreqRowsStatement_4() {
    // create database construction
    TdTable table = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdTable();
    // $NON-NLS-1$
    table.setName("TDQ_CALENDAR");
    TdColumn column = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdColumn();
    // $NON-NLS-1$
    column.setName("CAL_DATE");
    TdSqlDataType tdsql = org.talend.cwm.relational.RelationalFactory.eINSTANCE.createTdSqlDataType();
    // $NON-NLS-1$
    tdsql.setName("DATE");
    tdsql.setJavaDataType(Types.DATE);
    column.setSqlDataType(tdsql);
    table.getOwnedElement().add(column);
    column.setOwner(table);
    // create indicator
    UserDefIndicator userDefIndicator = IndicatorSqlFactory.eINSTANCE.createUserDefIndicator();
    UDIndicatorDefinition indicatorDefinition = UserdefineFactory.eINSTANCE.createUDIndicatorDefinition();
    // $NON-NLS-1$
    indicatorDefinition.setName("user define count");
    userDefIndicator.setName(indicatorDefinition.getName());
    userDefIndicator.setIndicatorDefinition(indicatorDefinition);
    TdExpression newTdExp = // $NON-NLS-1$
    BooleanExpressionHelper.createTdExpression(// $NON-NLS-1$
    "SQL", "SELECT * FROM <%=__TABLE_NAME__%> <%=__WHERE_CLAUSE__%>", // $NON-NLS-1$
    null);
    newTdExp.setModificationDate(DateUtils.getCurrentDate(DateUtils.PATTERN_5));
    indicatorDefinition.getViewRowsExpression().add(newTdExp);
    // $NON-NLS-1$  //$NON-NLS-2$
    ChartDataEntity chartDataEntity = new ChartDataEntity(userDefIndicator, "2012-06-05", "1");
    chartDataEntity.setLabelNull(false);
    // $NON-NLS-1$
    chartDataEntity.setKey("2012-06-05");
    assertFalse(chartDataEntity.isLabelNull());
    userDefIndicator.setAnalyzedElement(column);
    IndicatorParameters indicatorParameters = IndicatorsFactory.eINSTANCE.createIndicatorParameters();
    indicatorParameters.setDateParameters(null);
    userDefIndicator.setParameters(indicatorParameters);
    assertNull(indicatorParameters.getDateParameters());
    AnalysisResult createAnalysisResult = AnalysisFactory.eINSTANCE.createAnalysisResult();
    ExecutionInformations createExecutionInformations = AnalysisFactory.eINSTANCE.createExecutionInformations();
    createAnalysisResult.setResultMetadata(createExecutionInformations);
    createAnalysisResult.getIndicators().add(userDefIndicator);
    userDefIndicator.setAnalyzedElement(column);
    ana.setResults(createAnalysisResult);
    freqExp = new FrequencyStatisticsExplorer();
    freqExp.setAnalysis(ana);
    freqExp.setEnitty(chartDataEntity);
    String clause = freqExp.getFreqRowsStatement();
    // $NON-NLS-1$
    assertEquals("SELECT * FROM TDQ_CALENDAR ", clause);
}

Example 15 with ExecutionInformations

use of org.talend.dataquality.analysis.ExecutionInformations in project tdq-studio-se by Talend.

the class AnalysisHandlerTest method testGetDefaultLoadedRowCount.

/**
 * Test method for {@link org.talend.dq.analysis.AnalysisHandler#getDefaultLoadedRowCount()}.
 */
@Test
public void testGetDefaultLoadedRowCount() {
    int rowCountValue = 100;
    Analysis createAna = AnalysisFactory.eINSTANCE.createAnalysis();
    AnalysisResult createAnalysisResult = AnalysisFactory.eINSTANCE.createAnalysisResult();
    createAna.setResults(createAnalysisResult);
    ExecutionInformations createExecutionInformations = AnalysisFactory.eINSTANCE.createExecutionInformations();
    createAnalysisResult.setResultMetadata(createExecutionInformations);
    AnalysisHandler createHandler = AnalysisHandler.createHandler(createAna);
    // default case the value should be 50
    String defaultLoadedRowCount = createHandler.getDefaultLoadedRowCount();
    Assert.assertEquals(Integer.valueOf(50), Integer.valueOf(defaultLoadedRowCount));
    // after setting the value the result should be changed
    TaggedValueHelper.setTaggedValue(createAna, TaggedValueHelper.PREVIEW_ROW_NUMBER, String.valueOf(rowCountValue));
    defaultLoadedRowCount = createHandler.getDefaultLoadedRowCount();
    Assert.assertEquals(Integer.valueOf(rowCountValue), Integer.valueOf(defaultLoadedRowCount));
}