use of org.talend.dataquality.indicators.DuplicateCountIndicator in project tdq-studio-se by Talend.
the class CreateDuplicatesAnalysisAction method run.
@Override
public void run() {
ReturnCode success = new ReturnCode(true);
if (this.getColumnsMap() == null || this.getColumnsMap().isEmpty() || this.getConnection() == null) {
return;
}
try {
Set<ColumnSet> keySet = this.getColumnsMap().keySet();
for (ColumnSet cs : keySet) {
List<TdColumn> columns = this.getColumnsMap().get(cs);
// create the analysis
Analysis analysis = null;
AnalysisBuilder analysisBuilder = new AnalysisBuilder();
boolean analysisInitialized = analysisBuilder.initializeAnalysis(getAnalysisName(cs), AnalysisType.MULTIPLE_COLUMN);
if (analysisInitialized) {
analysis = analysisBuilder.getAnalysis();
}
fillMetadataToAnalysis(analysis, createDefaultAnalysisParameter(cs));
// add Connection into the Analysis Context
analysis.getContext().setConnection(this.getConnection());
for (TdColumn theColumn : columns) {
// add TdColumn into the Analysis Context
analysis.getContext().getAnalysedElements().add(theColumn);
// create Row Count Indicator
RowCountIndicator rcIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
rcIndicator.setAnalyzedElement(theColumn);
DefinitionHandler.getInstance().setDefaultIndicatorDefinition(rcIndicator);
analysis.getResults().getIndicators().add(rcIndicator);
// create Duplicate Count Indicator
DuplicateCountIndicator dcIndicator = IndicatorsFactory.eINSTANCE.createDuplicateCountIndicator();
dcIndicator.setAnalyzedElement(theColumn);
DefinitionHandler.getInstance().setDefaultIndicatorDefinition(dcIndicator);
analysis.getResults().getIndicators().add(dcIndicator);
}
// create dependencies
DependenciesHandler.getInstance().setDependencyOn(analysis, this.getConnection());
// set the domain
for (Domain domain : analysis.getParameters().getDataFilter()) {
domain.setName(analysis.getName());
}
// set execution language
analysis.getParameters().setExecutionLanguage(ExecutionLanguage.SQL);
// save the analysis
RepositoryNode analysisRepNode = (RepositoryNode) RepositoryNodeHelper.getDataProfilingFolderNode(EResourceConstant.ANALYSIS);
IFolder folder = WorkbenchUtils.getFolder(analysisRepNode);
AnalysisWriter analysisWriter = ElementWriterFactory.getInstance().createAnalysisWrite();
TypedReturnCode<Object> create = analysisWriter.create(analysis, folder);
if (create.isOk()) {
// refresh the RepositoryView
CorePlugin.getDefault().refreshDQView(analysisRepNode);
// open the editor
AnalysisRepNode anaRepNode = RepositoryNodeHelper.recursiveFindAnalysis(analysis);
AnalysisItemEditorInput analysisEditorInput = new AnalysisItemEditorInput(anaRepNode);
IRepositoryNode connectionRepNode = RepositoryNodeHelper.recursiveFind(this.getConnection());
analysisEditorInput.setConnectionNode(connectionRepNode);
CorePlugin.getDefault().openEditor(analysisEditorInput, AnalysisEditor.class.getName());
} else {
success.setOk(false);
success.setMessage(create.getMessage());
}
}
// for
} catch (Exception e) {
success.setOk(false);
success.setMessage(e.getMessage());
}
if (!success.isOk()) {
MessageUI.openError(success.getMessage());
}
}
use of org.talend.dataquality.indicators.DuplicateCountIndicator in project tdq-studio-se by Talend.
the class IndicatorEvaluator method executeSqlQuery.
@Override
protected ReturnCode executeSqlQuery(String sqlStatement) throws SQLException {
ReturnCode ok = new ReturnCode(true);
// check analyzed columns
Set<String> columns = getAnalyzedElements();
// feature 0010630 zshen:Make order unify which columns and columnName in the sqlStatement.mssqlOdbc need do
// this
List<String> columnlist = sortColumnName(columns, sqlStatement);
if (columnlist.isEmpty()) {
// $NON-NLS-1$
ok.setReturnCode(Messages.getString("IndicatorEvaluator.DefineAnalyzedColumns"), false);
return ok;
}
// ADD xqliu 2010-07-27 bug 13826
Map<String, String> columnlistMap = buildColumnListMap(columnlist);
// ~ 13826
// create query statement
// feature 0010630 zshen: Tables are not found when using Excel with ODBC connection
Statement statement = createStatement();
// MOD xqliu 2009-02-09 bug 6237
if (continueRun()) {
if (log.isInfoEnabled()) {
// $NON-NLS-1$
log.info("Executing query: " + sqlStatement);
}
statement.execute(sqlStatement);
}
// get the results
ResultSet resultSet = statement.getResultSet();
if (resultSet == null) {
// $NON-NLS-1$
String mess = Messages.getString("Evaluator.NoResultSet", sqlStatement);
log.warn(mess);
ok.setReturnCode(mess, false);
statement.close();
return ok;
}
// MOD qiongli TDQ-7282,check invalid judi.if there are invalid judis,return false code and show message later.
ok = getMessageForInvalidJUDIs();
int columnCount = resultSet.getMetaData().getColumnCount();
int maxNumberRows = analysis.getParameters().getMaxNumberRows();
// MOD mzhao feature: 12919, add capability to dill down data on Java engine.
AnalysisResult anaResult = analysis.getResults();
EMap<Indicator, AnalyzedDataSet> indicToRowMap = anaResult.getIndicToRowMap();
indicToRowMap.clear();
int recordIncrement = 0;
// --- for each row
int columnListSize = columnlist.size();
label: while (resultSet.next()) {
// feature 0010630 zshen: dislodge the Qualifiers from name of the column
for (int i = 0; i < columnListSize; i++) {
// MOD xqliu 2010-07-27 bug 13826
String col = columnlist.get(i);
List<Indicator> indicators = getIndicators(col);
col = columnlistMap.get(col);
// --- get content of column
Object object = ResultSetUtils.getBigObject(resultSet, col);
// FIXME this will slow down a lot the computation
if (object != null && !(object instanceof String) && object.toString().indexOf("TIMESTAMP") > -1) {
// $NON-NLS-1$
object = resultSet.getTimestamp(col);
}
// TDQ-11299: fix the ClassCastException: java.sql.Date cannot be cast to java.lang.String
if (object instanceof Date) {
if (object instanceof Time) {
object = new TalendFormatTime((Time) object);
} else {
object = new TalendFormatDate((Date) object);
}
}
// --- give row to handle to indicators
for (Indicator indicator : indicators) {
// MOD xqliu 2009-02-09 bug 6237
if (!continueRun()) {
break label;
}
// Added yyin 20120608 TDQ-3589
if (indicator instanceof DuplicateCountIndicator) {
((DuplicateCountIndicator) indicator).handle(object, resultSet, columnCount);
} else {
// ~
indicator.handle(object);
// ~MOD mzhao feature: 12919
}
AnalyzedDataSet analyzedDataSet = indicToRowMap.get(indicator);
if (analyzedDataSet == null) {
analyzedDataSet = AnalysisFactory.eINSTANCE.createAnalyzedDataSet();
indicToRowMap.put(indicator, analyzedDataSet);
analyzedDataSet.setDataCount(maxNumberRows);
analyzedDataSet.setRecordSize(0);
}
// should store data for dirll down
if (analysis.getParameters().isStoreData()) {
// current indicator is need to store the data
if (indicator.mustStoreRow()) {
List<Object[]> valueObjectList = initDataSet(indicator, indicToRowMap, object);
// MOD zshen add another loop to insert all of columnValue on the row into indicator.
recordIncrement = valueObjectList.size();
// MOD klliu 2011-06-30 bug 22523 whichever is Table or View,that finds columns should ues
// columnset
ColumnSet doSwitch = SwitchHelpers.COLUMN_SET_SWITCH.doSwitch(indicator.getAnalyzedElement().eContainer());
List<TdColumn> columnList = ColumnSetHelper.getColumns(doSwitch);
List<Object> inputRowList = new ArrayList<Object>();
for (int j = 0; j < columnCount; j++) {
String newcol = columnList.get(j).getName();
Object newobject = ResultSetUtils.getBigObject(resultSet, newcol);
// same format as result page.
if (newobject instanceof Date) {
if (newobject instanceof Time) {
newobject = new TalendFormatTime((Time) newobject);
} else {
newobject = new TalendFormatDate((Date) newobject);
}
}
if (indicator.isUsedMapDBMode()) {
inputRowList.add(newobject == null ? PluginConstant.NULL_STRING : newobject);
continue;
} else {
if (recordIncrement < maxNumberRows) {
// data.
if (recordIncrement < valueObjectList.size()) {
// decide whether need to
// increase
// current array.
valueObjectList.get(recordIncrement)[j] = newobject;
} else {
Object[] valueObject = new Object[columnCount];
valueObject[j] = newobject;
valueObjectList.add(valueObject);
}
} else {
break;
}
}
}
if (indicator.isUsedMapDBMode()) {
MapDBUtils.handleDrillDownData(object, inputRowList, indicator);
}
// ~
} else if (indicator instanceof UniqueCountIndicator && analysis.getResults().getIndicToRowMap().get(indicator).getData() != null) {
List<Object[]> removeValueObjectList = analysis.getResults().getIndicToRowMap().get(indicator).getData();
// MOD klliu 2011-06-30 bug 22523 whichever is Table or View,that finds columns should ues
// columnset
ColumnSet doSwitch = SwitchHelpers.COLUMN_SET_SWITCH.doSwitch(indicator.getAnalyzedElement().eContainer());
List<TdColumn> columnElementList = ColumnSetHelper.getColumns(doSwitch);
int offsetting = columnElementList.indexOf(indicator.getAnalyzedElement());
for (Object[] dataObject : removeValueObjectList) {
if (dataObject[offsetting].equals(object)) {
removeValueObjectList.remove(dataObject);
break;
}
}
}
}
}
}
}
// --- release resultset
resultSet.close();
// --- release statement
statement.close();
// --- close connection
getConnection().close();
return ok;
}
Aggregations