use of org.talend.dq.analysis.match.BlockAndMatchManager in project tdq-studio-se by Talend.
the class MatchAnalysisExecutor method execute.
/*
* (non-Javadoc)
*
* @see org.talend.dq.analysis.IAnalysisExecutor#execute(org.talend.dataquality.analysis.Analysis)
*/
public ReturnCode execute(Analysis analysis) {
assert analysis != null;
// --- preconditions
ReturnCode rc = AnalysisExecutorHelper.check(analysis);
if (!rc.isOk()) {
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
// --- creation time
final long startime = AnalysisExecutorHelper.setExecutionDateInAnalysisResult(analysis);
EList<Indicator> indicators = analysis.getResults().getIndicators();
RecordMatchingIndicator recordMatchingIndicator = null;
BlockKeyIndicator blockKeyIndicator = null;
for (Indicator ind : indicators) {
if (ind instanceof RecordMatchingIndicator) {
recordMatchingIndicator = (RecordMatchingIndicator) ind;
} else if (ind instanceof BlockKeyIndicator) {
blockKeyIndicator = (BlockKeyIndicator) ind;
}
}
if (recordMatchingIndicator == null || blockKeyIndicator == null) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.noIndicators"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
List<ModelElement> anlayzedElements = analysis.getContext().getAnalysedElements();
if (anlayzedElements == null || anlayzedElements.size() == 0) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.EmptyAnalyzedElement"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
// TDQ-9664 msjian: check the store on disk path.
Boolean isStoreOnDisk = TaggedValueHelper.getValueBoolean(SQLExecutor.STORE_ON_DISK_KEY, analysis);
if (isStoreOnDisk) {
String tempDataPath = TaggedValueHelper.getValueString(SQLExecutor.TEMP_DATA_DIR, analysis);
File file = new File(tempDataPath);
if (!file.exists() || !file.isDirectory()) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.InvalidPath", file.getPath()));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
}
// TDQ-9664~
Map<MetadataColumn, String> columnMap = getColumn2IndexMap(anlayzedElements);
ISQLExecutor sqlExecutor = getSQLExectutor(analysis, recordMatchingIndicator, columnMap);
if (sqlExecutor == null) {
rc.setOk(Boolean.FALSE);
// $NON-NLS-1$
rc.setMessage(Messages.getString("MatchAnalysisExecutor.noSqlExecutor"));
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
// Set schema for match key.
TypedReturnCode<MatchGroupResultConsumer> returnCode = new TypedReturnCode<MatchGroupResultConsumer>();
MetadataColumn[] completeColumnSchema = AnalysisRecordGroupingUtils.getCompleteColumnSchema(columnMap);
String[] colSchemaString = new String[completeColumnSchema.length];
int idx = 0;
for (MetadataColumn metadataCol : completeColumnSchema) {
colSchemaString[idx++] = metadataCol.getName();
}
recordMatchingIndicator.setMatchRowSchema(colSchemaString);
recordMatchingIndicator.reset();
MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(recordMatchingIndicator);
if (sqlExecutor.isStoreOnDisk()) {
// need to execute the query
try {
sqlExecutor.executeQuery(analysis.getContext().getConnection(), analysis.getContext().getAnalysedElements());
} catch (SQLException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
try {
TypedReturnCode<Object> result = StoreOnDiskUtils.getDefault().executeWithStoreOnDisk(columnMap, recordMatchingIndicator, blockKeyIndicator, sqlExecutor.getStoreOnDiskHandler(), matchResultConsumer);
if (result != null) {
returnCode.setObject((MatchGroupResultConsumer) result.getObject());
returnCode.setOk(result.isOk());
returnCode.setMessage(result.getMessage());
}
} catch (Exception e) {
log.error(e, e);
returnCode.setMessage(e.getMessage());
returnCode.setOk(false);
}
} else {
// Added TDQ-9320 , use the result set iterator to replace the list of result in the memory.
try {
Iterator<Record> resultSetIterator = sqlExecutor.getResultSetIterator(analysis.getContext().getConnection(), anlayzedElements);
BlockAndMatchManager bAndmManager = new BlockAndMatchManager(resultSetIterator, matchResultConsumer, columnMap, recordMatchingIndicator, blockKeyIndicator);
bAndmManager.run();
} catch (SQLException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
} catch (BusinessException e) {
log.error(e, e);
rc.setOk(Boolean.FALSE);
rc.setMessage(e.getMessage());
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
return rc;
}
}
if (!returnCode.isOk()) {
rc.setOk(returnCode.isOk());
rc.setMessage(returnCode.getMessage());
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
if (isLowMemory) {
// $NON-NLS-1$
rc.setMessage(Messages.getString("Evaluator.OutOfMomory", usedMemory));
}
// nodify the master page
refreshTableWithMatchFullResult(analysis);
// --- set metadata information of analysis
AnalysisExecutorHelper.setExecutionInfoInAnalysisResult(analysis, rc.isOk(), rc.getMessage());
// --- compute execution duration
if (this.continueRun()) {
long endtime = System.currentTimeMillis();
final ExecutionInformations resultMetadata = analysis.getResults().getResultMetadata();
resultMetadata.setExecutionDuration((int) (endtime - startime));
resultMetadata.setOutThreshold(false);
}
if (getMonitor() != null) {
getMonitor().worked(20);
}
return rc;
}
Aggregations