use of org.talend.dataquality.indicators.Indicator in project tdq-studio-se by Talend.
the class DelimitedFileAnalysisExecutor method runAnalysis.
/*
* (non-Javadoc)
*
* @see org.talend.dq.analysis.AnalysisExecutor#runAnalysis(org.talend.dataquality.analysis.Analysis,
* java.lang.String)
*/
@Override
protected boolean runAnalysis(Analysis analysis, String sqlStatement) {
DelimitedFileIndicatorEvaluator eval = createIndicatorEvaluator(analysis);
DelimitedFileConnection con = (DelimitedFileConnection) analysis.getContext().getConnection();
EList<Indicator> indicators = analysis.getResults().getIndicators();
eval.setMonitor(getMonitor());
RowCountIndicatorsAdapter.getInstance().clear();
for (Indicator indicator : indicators) {
assert indicator != null;
MetadataColumn mColumn = SwitchHelpers.METADATA_COLUMN_SWITCH.doSwitch(indicator.getAnalyzedElement());
if (mColumn == null) {
continue;
}
String columnName = mColumn.getLabel();
eval.storeIndicator(columnName, indicator);
}
eval.setDelimitedFileconnection(con);
ReturnCode rc = eval.evaluateIndicators(sqlStatement, true);
if (!rc.isOk()) {
log.warn(rc.getMessage());
setError(rc.getMessage());
}
if (getMonitor() != null) {
getMonitor().worked(compIndicatorsWorked);
}
return rc.isOk();
}
use of org.talend.dataquality.indicators.Indicator in project tdq-studio-se by Talend.
the class MatchAnalysisExecutor method execute.
/*
* (non-Javadoc)
*
* @see org.talend.dq.analysis.IAnalysisExecutor#execute(org.talend.dataquality.analysis.Analysis)
*/
public ReturnCode execute(Analysis analysis) {
assert analysis != null;
// --- preconditions
ReturnCode rc = AnalysisExecutorHelper.check(analysis);
if (!rc.isOk()) {
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
// --- creation time
final long startime = AnalysisExecutorHelper.setExecutionDateInAnalysisResult(analysis);
EList<Indicator> indicators = analysis.getResults().getIndicators();
RecordMatchingIndicator recordMatchingIndicator = null;
BlockKeyIndicator blockKeyIndicator = null;
for (Indicator ind : indicators) {
if (ind instanceof RecordMatchingIndicator) {
recordMatchingIndicator = (RecordMatchingIndicator) ind;
} else if (ind instanceof BlockKeyIndicator) {
blockKeyIndicator = (BlockKeyIndicator) ind;
}
}
if (recordMatchingIndicator == null || blockKeyIndicator == null) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.noIndicators"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
List<ModelElement> anlayzedElements = analysis.getContext().getAnalysedElements();
if (anlayzedElements == null || anlayzedElements.size() == 0) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.EmptyAnalyzedElement"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
// TDQ-9664 msjian: check the store on disk path.
Boolean isStoreOnDisk = TaggedValueHelper.getValueBoolean(SQLExecutor.STORE_ON_DISK_KEY, analysis);
if (isStoreOnDisk) {
String tempDataPath = TaggedValueHelper.getValueString(SQLExecutor.TEMP_DATA_DIR, analysis);
File file = new File(tempDataPath);
if (!file.exists() || !file.isDirectory()) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.InvalidPath", file.getPath()));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
}
// TDQ-9664~
Map<MetadataColumn, String> columnMap = getColumn2IndexMap(anlayzedElements);
ISQLExecutor sqlExecutor = getSQLExectutor(analysis, recordMatchingIndicator, columnMap);
if (sqlExecutor == null) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.noSqlExecutor"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
// Set schema for match key.
TypedReturnCode<MatchGroupResultConsumer> returnCode = new TypedReturnCode<MatchGroupResultConsumer>();
MetadataColumn[] completeColumnSchema = AnalysisRecordGroupingUtils.getCompleteColumnSchema(columnMap);
String[] colSchemaString = new String[completeColumnSchema.length];
int idx = 0;
for (MetadataColumn metadataCol : completeColumnSchema) {
colSchemaString[idx++] = metadataCol.getName();
}
recordMatchingIndicator.setMatchRowSchema(colSchemaString);
recordMatchingIndicator.reset();
MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(recordMatchingIndicator);
if (sqlExecutor.isStoreOnDisk()) {
// need to execute the query
try {
sqlExecutor.executeQuery(analysis.getContext().getConnection(), analysis.getContext().getAnalysedElements());
} catch (SQLException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
try {
TypedReturnCode<Object> result = StoreOnDiskUtils.getDefault().executeWithStoreOnDisk(columnMap, recordMatchingIndicator, blockKeyIndicator, sqlExecutor.getStoreOnDiskHandler(), matchResultConsumer);
if (result != null) {
returnCode.setObject((MatchGroupResultConsumer) result.getObject());
returnCode.setOk(result.isOk());
returnCode.setMessage(result.getMessage());
}
} catch (Exception e) {
log.error(e, e);
returnCode.setMessage(e.getMessage());
returnCode.setOk(false);
}
} else {
// Added TDQ-9320 , use the result set iterator to replace the list of result in the memory.
try {
Iterator<Record> resultSetIterator = sqlExecutor.getResultSetIterator(analysis.getContext().getConnection(), anlayzedElements);
BlockAndMatchManager bAndmManager = new BlockAndMatchManager(resultSetIterator, matchResultConsumer, columnMap, recordMatchingIndicator, blockKeyIndicator);
bAndmManager.run();
} catch (SQLException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
} catch (BusinessException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
}
if (!returnCode.isOk()) {
rc.setOk(returnCode.isOk());
rc.setMessage(returnCode.getMessage());
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
if (isLowMemory) {
// $NON-NLS-1$
rc.setMessage(Messages.getString("Evaluator.OutOfMomory", usedMemory));
}
// nodify the master page
refreshTableWithMatchFullResult(analysis);
// --- set metadata information of analysis
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
// --- compute execution duration
if (this.continueRun()) {
long endtime = System.currentTimeMillis();
final ExecutionInformations resultMetadata = analysis.getResults().getResultMetadata();
resultMetadata.setExecutionDuration((int) (endtime - startime));
resultMetadata.setOutThreshold(false);
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
return rc;
}
use of org.talend.dataquality.indicators.Indicator in project tdq-studio-se by Talend.
the class ModelElementAnalysisHandler method getIndicators.
/**
* Method "getIndicators".
*
* @param modelElement
* @return the indicators attached to this column
*/
public Collection<Indicator> getIndicators(ModelElement modelElement) {
ModelElement me = modelElement;
// TDQ-9553, if import an old analysis with MDM Connection,the modelement will be null.
Collection<Indicator> indics = new ArrayList<Indicator>();
if (me == null) {
return indics;
}
if (me.eIsProxy()) {
me = (ModelElement) EObjectHelper.resolveObject(me);
}
EList<Indicator> allIndics = analysis.getResults().getIndicators();
for (Indicator indicator : allIndics) {
if (indicator.getAnalyzedElement() != null && indicator.getAnalyzedElement().equals(me)) {
initializeIndicator(indicator);
indics.add(indicator);
}
}
return indics;
}
use of org.talend.dataquality.indicators.Indicator in project tdq-studio-se by Talend.
the class RowMatchingAnalysisExecutor method runAnalysis.
/*
* (non-Javadoc)
*
* @see org.talend.dq.analysis.AnalysisExecutor#runAnalysis(org.talend.dataquality.analysis.Analysis,
* java.lang.String)
*/
@Override
protected boolean runAnalysis(Analysis analysis, String sqlStatement) {
boolean isSuccess = Boolean.TRUE;
TypedReturnCode<java.sql.Connection> trc = this.getConnectionBeforeRun(analysis);
if (!trc.isOk()) {
log.error(trc.getMessage());
setError(trc.getMessage());
traceError(Messages.getString("FunctionalDependencyExecutor.CANNOTEXECUTEANALYSIS", analysis.getName(), // $NON-NLS-1$
trc.getMessage()));
return Boolean.FALSE;
}
Connection connection = trc.getObject();
try {
// execute the sql statement for each indicator
EList<Indicator> indicators = analysis.getResults().getIndicators();
EList<Indicator> deactivatedIndicators = analysis.getParameters().getDeactivatedIndicators();
for (Indicator indicator : indicators) {
if (deactivatedIndicators.contains(indicator)) {
// do not evaluate this indicator
continue;
}
// set the connection's catalog
if (this.catalogOrSchema != null && needChangeCatalog(connection)) {
// check whether null argument can be given
changeCatalog(this.catalogOrSchema, connection);
}
Expression query = dbms().getInstantiatedExpression(indicator);
if (query == null) {
traceError(// $NON-NLS-1$//$NON-NLS-2$
"Query not executed for indicator: \"" + AnalysisExecutorHelper.getIndicatorName(indicator) + "\" " + // $NON-NLS-1$
"query is null");
isSuccess = Boolean.FALSE;
continue;
}
try {
Boolean isExecSuccess = executeQuery(indicator, connection, query);
indicator.setComputed(true);
if (!isExecSuccess) {
traceError(// $NON-NLS-1$//$NON-NLS-2$
"Query not executed for indicator: \"" + AnalysisExecutorHelper.getIndicatorName(indicator) + "\" " + "SQL query: " + // $NON-NLS-1$
query.getBody());
isSuccess = Boolean.FALSE;
continue;
}
} catch (AnalysisExecutionException e) {
traceError(e.getMessage());
isSuccess = Boolean.FALSE;
continue;
}
}
} finally {
ReturnCode rc = closeConnection(analysis, connection);
if (!rc.isOk()) {
isSuccess = Boolean.FALSE;
}
}
return isSuccess;
}
use of org.talend.dataquality.indicators.Indicator in project tdq-studio-se by Talend.
the class AnalysisExecutor method initializeIndicators.
/**
* 2012-3-14 TDQ-4433,reset indicatorDefinition for indicator if needed(indicatorDefinition is null or proxy).
*/
protected void initializeIndicators(Analysis analysis) {
ModelElementAnalysisHandler modHandler = new ModelElementAnalysisHandler();
for (Indicator ind : analysis.getResults().getIndicators()) {
modHandler.initializeIndicator(ind);
}
// ADD msjian TDQ-8202 2013-11-19: when import a judi indicator, should update JUDIs For Analysis.
// else when the user do generate report directly, will can not load udi jars
UDIHelper.updateJUDIsForAnalysis(analysis);
// TDQ-8202~
}
Aggregations