use of orgomg.cwm.resource.relational.ColumnSet in project tdq-studio-se by Talend.
the class CreateDuplicatesAnalysisAction method run.
@Override
public void run() {
ReturnCode success = new ReturnCode(true);
if (this.getColumnsMap() == null || this.getColumnsMap().isEmpty() || this.getConnection() == null) {
return;
}
try {
Set<ColumnSet> keySet = this.getColumnsMap().keySet();
for (ColumnSet cs : keySet) {
List<TdColumn> columns = this.getColumnsMap().get(cs);
// create the analysis
Analysis analysis = null;
AnalysisBuilder analysisBuilder = new AnalysisBuilder();
boolean analysisInitialized = analysisBuilder.initializeAnalysis(getAnalysisName(cs), AnalysisType.MULTIPLE_COLUMN);
if (analysisInitialized) {
analysis = analysisBuilder.getAnalysis();
}
fillMetadataToAnalysis(analysis, createDefaultAnalysisParameter(cs));
// add Connection into the Analysis Context
analysis.getContext().setConnection(this.getConnection());
for (TdColumn theColumn : columns) {
// add TdColumn into the Analysis Context
analysis.getContext().getAnalysedElements().add(theColumn);
// create Row Count Indicator
RowCountIndicator rcIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
rcIndicator.setAnalyzedElement(theColumn);
DefinitionHandler.getInstance().setDefaultIndicatorDefinition(rcIndicator);
analysis.getResults().getIndicators().add(rcIndicator);
// create Duplicate Count Indicator
DuplicateCountIndicator dcIndicator = IndicatorsFactory.eINSTANCE.createDuplicateCountIndicator();
dcIndicator.setAnalyzedElement(theColumn);
DefinitionHandler.getInstance().setDefaultIndicatorDefinition(dcIndicator);
analysis.getResults().getIndicators().add(dcIndicator);
}
// create dependencies
DependenciesHandler.getInstance().setDependencyOn(analysis, this.getConnection());
// set the domain
for (Domain domain : analysis.getParameters().getDataFilter()) {
domain.setName(analysis.getName());
}
// set execution language
analysis.getParameters().setExecutionLanguage(ExecutionLanguage.SQL);
// save the analysis
RepositoryNode analysisRepNode = (RepositoryNode) RepositoryNodeHelper.getDataProfilingFolderNode(EResourceConstant.ANALYSIS);
IFolder folder = WorkbenchUtils.getFolder(analysisRepNode);
AnalysisWriter analysisWriter = ElementWriterFactory.getInstance().createAnalysisWrite();
TypedReturnCode<Object> create = analysisWriter.create(analysis, folder);
if (create.isOk()) {
// refresh the RepositoryView
CorePlugin.getDefault().refreshDQView(analysisRepNode);
// open the editor
AnalysisRepNode anaRepNode = RepositoryNodeHelper.recursiveFindAnalysis(analysis);
AnalysisItemEditorInput analysisEditorInput = new AnalysisItemEditorInput(anaRepNode);
IRepositoryNode connectionRepNode = RepositoryNodeHelper.recursiveFind(this.getConnection());
analysisEditorInput.setConnectionNode(connectionRepNode);
CorePlugin.getDefault().openEditor(analysisEditorInput, AnalysisEditor.class.getName());
} else {
success.setOk(false);
success.setMessage(create.getMessage());
}
}
// for
} catch (Exception e) {
success.setOk(false);
success.setMessage(e.getMessage());
}
if (!success.isOk()) {
MessageUI.openError(success.getMessage());
}
}
use of orgomg.cwm.resource.relational.ColumnSet in project tdq-studio-se by Talend.
the class ChangeConnectionAction method synAnalyzedElements.
private boolean synAnalyzedElements(AnalyzedElementSynDialog anaEleSynDialog, Analysis synAnalysis, Connection oldDataProvider, Connection newDataProv) {
// Change connection uuid.
Map<ModelElement, ModelElement> synEleMap = anaEleSynDialog == null ? null : anaEleSynDialog.getSynedEleMap();
AnalysisBuilder anaBuilder = new AnalysisBuilder();
anaBuilder.setAnalysis(synAnalysis);
// Remove old dependencies.
boolean isRemovedDependency = DependenciesHandler.getInstance().removeConnDependencyAndSave(analysisItem);
if (!isRemovedDependency) {
return false;
}
synAnalysis.getContext().setConnection(newDataProv);
// Synchronize analysis result.
EList<Indicator> indcList = synAnalysis.getResults().getIndicators();
Indicator[] copiedIndArray = new Indicator[indcList.size()];
System.arraycopy(indcList.toArray(), 0, copiedIndArray, 0, indcList.size());
// MOD qiongli 2010-6-13,bug 12766:To column analysis, which has same
// construct connection with before and maybe have not indicator
boolean isExistSynedElement = false;
AnalysisType analysisType = synAnalysis.getParameters().getAnalysisType();
if (analysisType == AnalysisType.MULTIPLE_COLUMN) {
EList<ModelElement> meLs = synAnalysis.getContext().getAnalysedElements();
ModelElement[] mes = new ModelElement[meLs.size()];
System.arraycopy(meLs.toArray(), 0, mes, 0, meLs.size());
synAnalysis.getContext().getAnalysedElements().clear();
for (ModelElement me : mes) {
if (synEleMap != null && synEleMap.get(me) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(me);
synAnalysis.getContext().getAnalysedElements().add(newColumn);
isExistSynedElement = true;
}
}
}
if (!isExistSynedElement) {
synAnalysis.getContext().getAnalysedElements().clear();
}
// ~
synAnalysis.getResults().getIndicators().clear();
for (Indicator indicator : copiedIndArray) {
// connection.
if (indicator instanceof ColumnSetMultiValueIndicator) {
ColumnSetMultiValueIndicator compositeInd = (ColumnSetMultiValueIndicator) indicator;
ModelElement[] mes = new ModelElement[compositeInd.getAnalyzedColumns().size()];
((ColumnSetMultiValueIndicator) indicator).getAnalyzedColumns().toArray(mes);
compositeInd.getAnalyzedColumns().clear();
for (ModelElement me : mes) {
if (synEleMap != null && synEleMap.get(me) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(me);
DataminingType dataminingType = MetadataHelper.getDataminingType((TdColumn) me);
if (dataminingType == null) {
dataminingType = MetadataHelper.getDefaultDataminingType(((TdColumn) me).getSqlDataType().getJavaDataType());
}
MetadataHelper.setDataminingType(dataminingType, newColumn);
compositeInd.getAnalyzedColumns().add(newColumn);
anaBuilder.addElementToAnalyze(newColumn, indicator);
isExistSynedElement = true;
}
}
} else if (indicator instanceof ColumnsCompareIndicator) {
// Correlation compare
ColumnsCompareIndicator compInd = (ColumnsCompareIndicator) indicator;
if ((compInd.getColumnSetA() == null || compInd.getColumnSetA().size() == 0) || (compInd.getColumnSetB() == null || compInd.getColumnSetB().size() == 0)) {
return false;
}
// Column set(Columns)
ModelElement[] mesA = new ModelElement[compInd.getColumnSetA().size()];
compInd.getColumnSetA().toArray(mesA);
compInd.getColumnSetA().clear();
for (ModelElement me : mesA) {
if (synEleMap != null && synEleMap.get(me) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(me);
compInd.getColumnSetA().add(newColumn);
anaBuilder.addElementToAnalyze(newColumn, indicator);
isExistSynedElement = true;
}
}
ModelElement[] mesB = new ModelElement[compInd.getColumnSetB().size()];
compInd.getColumnSetB().toArray(mesB);
compInd.getColumnSetB().clear();
for (ModelElement me : mesB) {
if (synEleMap != null && synEleMap.get(me) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(me);
compInd.getColumnSetB().add(newColumn);
anaBuilder.addElementToAnalyze(newColumn, indicator);
isExistSynedElement = true;
}
}
// Analyzed element(Table)
ModelElement oldAnaEle = compInd.getAnalyzedElement();
compInd.setAnalyzedElement(null);
ColumnSet oldColSetA = ColumnHelper.getColumnOwnerAsColumnSet(mesA[0]);
ColumnSet oldColSetB = ColumnHelper.getColumnOwnerAsColumnSet(mesB[0]);
if (oldColSetA == oldAnaEle) {
if (synEleMap != null && synEleMap.get(mesA[0]) != null) {
compInd.setAnalyzedElement(ColumnHelper.getColumnOwnerAsColumnSet(synEleMap.get(mesA[0])));
}
}
if (oldColSetB == oldAnaEle) {
if (synEleMap != null && synEleMap.get(mesB[0]) != null) {
compInd.setAnalyzedElement(ColumnHelper.getColumnOwnerAsColumnSet(synEleMap.get(mesB[0])));
}
}
} else if (indicator instanceof ColumnDependencyIndicator) {
// ADD qiongli bug 0012766
// Functional Dependency indicator
ColumnDependencyIndicator funDepInd = (ColumnDependencyIndicator) indicator;
if (funDepInd.getColumnA() == null || funDepInd.getColumnB() == null) {
return false;
}
// Column A
if (synEleMap != null && synEleMap.get(funDepInd.getColumnA()) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(funDepInd.getColumnA());
funDepInd.setColumnA(newColumn);
anaBuilder.addElementToAnalyze(newColumn, indicator);
isExistSynedElement = true;
}
// Column B
if (synEleMap != null && synEleMap.get(funDepInd.getColumnB()) != null) {
TdColumn newColumn = (TdColumn) synEleMap.get(funDepInd.getColumnB());
funDepInd.setColumnB(newColumn);
anaBuilder.addElementToAnalyze(newColumn, indicator);
isExistSynedElement = true;
}
// Analyzed element(Table)
ModelElement oldAnaEle = funDepInd.getAnalyzedElement();
funDepInd.setAnalyzedElement(null);
ColumnSet oldColSetA = ColumnHelper.getColumnOwnerAsColumnSet(funDepInd.getColumnA());
ColumnSet oldColSetB = ColumnHelper.getColumnOwnerAsColumnSet(funDepInd.getColumnB());
if (oldColSetA == oldAnaEle) {
if (synEleMap != null && synEleMap.get(funDepInd.getColumnA()) != null) {
funDepInd.setAnalyzedElement(ColumnHelper.getColumnOwnerAsColumnSet(synEleMap.get(funDepInd.getColumnA())));
}
}
if (oldColSetB == oldAnaEle) {
if (synEleMap != null && synEleMap.get(funDepInd.getColumnB()) != null) {
funDepInd.setAnalyzedElement(ColumnHelper.getColumnOwnerAsColumnSet(synEleMap.get(funDepInd.getColumnB())));
}
}
} else {
ModelElement me = indicator.getAnalyzedElement();
if (synEleMap != null && synEleMap.get(me) != null) {
indicator.setAnalyzedElement(synEleMap.get(me));
if (analysisType == AnalysisType.MULTIPLE_COLUMN) {
synAnalysis.getResults().getIndicators().add(indicator);
} else {
anaBuilder.addElementToAnalyze(synEleMap.get(me), indicator);
}
isExistSynedElement = true;
}
}
}
return isExistSynedElement;
}
use of orgomg.cwm.resource.relational.ColumnSet in project tdq-studio-se by Talend.
the class IndicatorEvaluator method executeSqlQuery.
@Override
protected ReturnCode executeSqlQuery(String sqlStatement) throws SQLException {
ReturnCode ok = new ReturnCode(true);
// check analyzed columns
Set<String> columns = getAnalyzedElements();
// feature 0010630 zshen:Make order unify which columns and columnName in the sqlStatement.mssqlOdbc need do
// this
List<String> columnlist = sortColumnName(columns, sqlStatement);
if (columnlist.isEmpty()) {
// $NON-NLS-1$
ok.setReturnCode(Messages.getString("IndicatorEvaluator.DefineAnalyzedColumns"), false);
return ok;
}
// ADD xqliu 2010-07-27 bug 13826
Map<String, String> columnlistMap = buildColumnListMap(columnlist);
// ~ 13826
// create query statement
// feature 0010630 zshen: Tables are not found when using Excel with ODBC connection
Statement statement = createStatement();
// MOD xqliu 2009-02-09 bug 6237
if (continueRun()) {
if (log.isInfoEnabled()) {
// $NON-NLS-1$
log.info("Executing query: " + sqlStatement);
}
statement.execute(sqlStatement);
}
// get the results
ResultSet resultSet = statement.getResultSet();
if (resultSet == null) {
// $NON-NLS-1$
String mess = Messages.getString("Evaluator.NoResultSet", sqlStatement);
log.warn(mess);
ok.setReturnCode(mess, false);
statement.close();
return ok;
}
// MOD qiongli TDQ-7282,check invalid judi.if there are invalid judis,return false code and show message later.
ok = getMessageForInvalidJUDIs();
int columnCount = resultSet.getMetaData().getColumnCount();
int maxNumberRows = analysis.getParameters().getMaxNumberRows();
// MOD mzhao feature: 12919, add capability to dill down data on Java engine.
AnalysisResult anaResult = analysis.getResults();
EMap<Indicator, AnalyzedDataSet> indicToRowMap = anaResult.getIndicToRowMap();
indicToRowMap.clear();
int recordIncrement = 0;
// --- for each row
int columnListSize = columnlist.size();
label: while (resultSet.next()) {
// feature 0010630 zshen: dislodge the Qualifiers from name of the column
for (int i = 0; i < columnListSize; i++) {
// MOD xqliu 2010-07-27 bug 13826
String col = columnlist.get(i);
List<Indicator> indicators = getIndicators(col);
col = columnlistMap.get(col);
// --- get content of column
Object object = ResultSetUtils.getBigObject(resultSet, col);
// FIXME this will slow down a lot the computation
if (object != null && !(object instanceof String) && object.toString().indexOf("TIMESTAMP") > -1) {
// $NON-NLS-1$
object = resultSet.getTimestamp(col);
}
// TDQ-11299: fix the ClassCastException: java.sql.Date cannot be cast to java.lang.String
if (object instanceof Date) {
if (object instanceof Time) {
object = new TalendFormatTime((Time) object);
} else {
object = new TalendFormatDate((Date) object);
}
}
// --- give row to handle to indicators
for (Indicator indicator : indicators) {
// MOD xqliu 2009-02-09 bug 6237
if (!continueRun()) {
break label;
}
// Added yyin 20120608 TDQ-3589
if (indicator instanceof DuplicateCountIndicator) {
((DuplicateCountIndicator) indicator).handle(object, resultSet, columnCount);
} else {
// ~
indicator.handle(object);
// ~MOD mzhao feature: 12919
}
AnalyzedDataSet analyzedDataSet = indicToRowMap.get(indicator);
if (analyzedDataSet == null) {
analyzedDataSet = AnalysisFactory.eINSTANCE.createAnalyzedDataSet();
indicToRowMap.put(indicator, analyzedDataSet);
analyzedDataSet.setDataCount(maxNumberRows);
analyzedDataSet.setRecordSize(0);
}
// should store data for dirll down
if (analysis.getParameters().isStoreData()) {
// current indicator is need to store the data
if (indicator.mustStoreRow()) {
List<Object[]> valueObjectList = initDataSet(indicator, indicToRowMap, object);
// MOD zshen add another loop to insert all of columnValue on the row into indicator.
recordIncrement = valueObjectList.size();
// MOD klliu 2011-06-30 bug 22523 whichever is Table or View,that finds columns should ues
// columnset
ColumnSet doSwitch = SwitchHelpers.COLUMN_SET_SWITCH.doSwitch(indicator.getAnalyzedElement().eContainer());
List<TdColumn> columnList = ColumnSetHelper.getColumns(doSwitch);
List<Object> inputRowList = new ArrayList<Object>();
for (int j = 0; j < columnCount; j++) {
String newcol = columnList.get(j).getName();
Object newobject = ResultSetUtils.getBigObject(resultSet, newcol);
// same format as result page.
if (newobject instanceof Date) {
if (newobject instanceof Time) {
newobject = new TalendFormatTime((Time) newobject);
} else {
newobject = new TalendFormatDate((Date) newobject);
}
}
if (indicator.isUsedMapDBMode()) {
inputRowList.add(newobject == null ? PluginConstant.NULL_STRING : newobject);
continue;
} else {
if (recordIncrement < maxNumberRows) {
// data.
if (recordIncrement < valueObjectList.size()) {
// decide whether need to
// increase
// current array.
valueObjectList.get(recordIncrement)[j] = newobject;
} else {
Object[] valueObject = new Object[columnCount];
valueObject[j] = newobject;
valueObjectList.add(valueObject);
}
} else {
break;
}
}
}
if (indicator.isUsedMapDBMode()) {
MapDBUtils.handleDrillDownData(object, inputRowList, indicator);
}
// ~
} else if (indicator instanceof UniqueCountIndicator && analysis.getResults().getIndicToRowMap().get(indicator).getData() != null) {
List<Object[]> removeValueObjectList = analysis.getResults().getIndicToRowMap().get(indicator).getData();
// MOD klliu 2011-06-30 bug 22523 whichever is Table or View,that finds columns should ues
// columnset
ColumnSet doSwitch = SwitchHelpers.COLUMN_SET_SWITCH.doSwitch(indicator.getAnalyzedElement().eContainer());
List<TdColumn> columnElementList = ColumnSetHelper.getColumns(doSwitch);
int offsetting = columnElementList.indexOf(indicator.getAnalyzedElement());
for (Object[] dataObject : removeValueObjectList) {
if (dataObject[offsetting].equals(object)) {
removeValueObjectList.remove(dataObject);
break;
}
}
}
}
}
}
}
// --- release resultset
resultSet.close();
// --- release statement
statement.close();
// --- close connection
getConnection().close();
return ok;
}
use of orgomg.cwm.resource.relational.ColumnSet in project tdq-studio-se by Talend.
the class AnalyzedColumnsSynDialog method reloadInputModel.
@Override
public void reloadInputModel() {
TdColumn column = null;
modelInput.clear();
synedEleMap.clear();
for (ModelElement element : analyzedElements) {
try {
column = (TdColumn) element;
synedEleMap.put(column, null);
ColumnSet anaColumnSet = ColumnHelper.getColumnOwnerAsColumnSet(column);
Package anaPackage = ColumnSetHelper.getParentCatalogOrSchema(anaColumnSet);
if (anaPackage == null) {
return;
}
Package connPackage = null;
for (Package pk : newDataProvider.getDataPackage()) {
// MOD mzhao bug 8567. 2009-08-10 (Case of MS SQL Server)
if (pk instanceof Catalog && anaPackage instanceof Schema) {
Catalog catl = CatalogHelper.getParentCatalog(anaPackage);
if (null != catl && pk.getName().equalsIgnoreCase(catl.getName())) {
connPackage = pk;
break;
}
} else if (null != anaPackage && pk.getName().equalsIgnoreCase(anaPackage.getName())) {
connPackage = pk;
break;
}
}
if (connPackage == null) {
SynTreeModel synTreeModel = new SynTreeModel(column);
synTreeModel.setOldDataProvElement(anaPackage);
// synTreeModel.setNewDataProvElement(connPackage);
modelInput.add(synTreeModel);
continue;
}
List<? extends MetadataTable> connColumnSetList = null;
// MOD mzhao bug 8567. 2009-08-10 (Case of MS SQL Server)
if (connPackage instanceof Catalog && anaPackage instanceof Schema) {
for (Schema sche : CatalogHelper.getSchemas((Catalog) connPackage)) {
if (sche.getName().equalsIgnoreCase(anaPackage.getName())) {
// MOD mzhao bug 8567. 2010-03-29.
connPackage = sche;
// }
break;
}
}
}
boolean loadFromDb = connPackage.getOwnedElement().size() == 0;
if (anaColumnSet instanceof TdTable) {
connColumnSetList = DqRepositoryViewService.getTables(newDataProvider, connPackage, null, loadFromDb, false);
// connColumnSetList = PackageHelper.getTables(connPackage);
if (loadFromDb) {
for (MetadataTable table : connColumnSetList) {
PackageHelper.addMetadataTable(table, connPackage);
}
}
// connColumnSetList = PackageHelper.getTables(connPackage);
} else {
connColumnSetList = DqRepositoryViewService.getViews(newDataProvider, connPackage, null, loadFromDb, false);
// connColumnSetList = PackageHelper.getViews(connPackage);
if (loadFromDb) {
for (MetadataTable table : connColumnSetList) {
PackageHelper.addMetadataTable(table, connPackage);
}
}
}
ColumnSet connColumnSet = null;
for (Object colSet : connColumnSetList) {
if (((ColumnSet) colSet).getName().equalsIgnoreCase(anaColumnSet.getName())) {
connColumnSet = (ColumnSet) colSet;
break;
}
}
if (connColumnSet == null) {
SynTreeModel synTreeModel = new SynTreeModel(column);
synTreeModel.setOldDataProvElement(anaColumnSet);
synTreeModel.setNewDataProvElement(connPackage);
modelInput.add(synTreeModel);
continue;
}
TdColumn connColumn = null;
// MOD xqliu 2012-05-04 TDQ-4853 should call getFeature(instead of getOwnedElement) method to get the
// column list
loadFromDb = connColumnSet.getFeature().size() == 0;
// ~ TDQ-4853
List<TdColumn> columns = DqRepositoryViewService.getColumns(newDataProvider, connColumnSet, null, loadFromDb);
ColumnSetHelper.addColumns(connColumnSet, columns);
for (TdColumn loopColumn : columns) {
if (loopColumn.getName().equalsIgnoreCase(column.getName())) {
connColumn = loopColumn;
break;
}
}
if (connColumn == null) {
SynTreeModel synTreeModel = new SynTreeModel(column);
synTreeModel.setOldDataProvElement(column);
synTreeModel.setNewDataProvElement(connColumnSet);
modelInput.add(synTreeModel);
continue;
}
synedEleMap.put(column, connColumn);
if (!connColumn.getSqlDataType().getName().equals(column.getSqlDataType().getName())) {
SynTreeModel synTreeModel = new SynTreeModel(column);
synTreeModel.setOldDataProvElement(column.getSqlDataType());
// synTreeModel.setNewDataProvElement(connColumn
// .getSqlDataType());
modelInput.add(synTreeModel);
continue;
}
} catch (Exception e) {
log.error(e, e);
e.printStackTrace();
}
}
}
use of orgomg.cwm.resource.relational.ColumnSet in project tdq-studio-se by Talend.
the class ColumnsMapSelectionDialog method buildUserMap.
/**
* DOC xqliu Comment method "buildUserMap".
*
* @param allCheckedElements
*/
private void buildUserMap(List<IRepositoryNode> allCheckedElements) {
if (allCheckedElements != null && !allCheckedElements.isEmpty()) {
List<TdColumn> userColumnsAll = new ArrayList<TdColumn>();
for (IRepositoryNode node : allCheckedElements) {
if (node instanceof DBColumnRepNode) {
userColumnsAll.add(((DBColumnRepNode) node).getTdColumn());
}
}
Set<ColumnSet> keySet = this.allMap.keySet();
for (ColumnSet cs : keySet) {
List<TdColumn> userColumns = new ArrayList<TdColumn>();
List<TdColumn> list = this.allMap.get(cs);
for (TdColumn column : list) {
if (userColumnsAll.contains(column)) {
userColumns.add(column);
}
}
if (!userColumns.isEmpty()) {
this.userMap.put(cs, userColumns);
}
}
}
}
Aggregations