use of org.talend.dataquality.rules.MatchRuleDefinition in project tdq-studio-se by Talend.
the class AnalysisRecordGroupingUtilsTest method testCreateSurvivorShipAlgorithmParams.
/**
* Test method for
* {@link org.talend.dq.analysis.AnalysisRecordGroupingUtils#createSurvivorShipAlgorithmParams(org.talend.dataquality.record.linkage.grouping.AnalysisMatchRecordGrouping, org.talend.dataquality.indicators.columnset.RecordMatchingIndicator, Map)
* .
*/
@Test
public void testCreateSurvivorShipAlgorithmParams() {
// Survivorshipkey
RecordMatchingIndicator recordMatchingIndicator = ColumnsetFactory.eINSTANCE.createRecordMatchingIndicator();
MatchRuleDefinition createMatchRuleDefinition = RulesFactory.eINSTANCE.createMatchRuleDefinition();
recordMatchingIndicator.setBuiltInMatchRuleDefinition(createMatchRuleDefinition);
EList<SurvivorshipKeyDefinition> survivorshipKeys = createMatchRuleDefinition.getSurvivorshipKeys();
AlgorithmDefinition createAlgorithmDefinition = RulesFactory.eINSTANCE.createAlgorithmDefinition();
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmParameters("");
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmType("Longest");
// $NON-NLS-1$
survivorshipKeys.add(createKeyDefinition("a1", createAlgorithmDefinition));
createAlgorithmDefinition = RulesFactory.eINSTANCE.createAlgorithmDefinition();
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmParameters("");
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmType("Longest");
// $NON-NLS-1$
survivorshipKeys.add(createKeyDefinition("a2", createAlgorithmDefinition));
// DefaultSurvivorship
EList<DefaultSurvivorshipDefinition> defaultSurvivorshipDefinitions = createMatchRuleDefinition.getDefaultSurvivorshipDefinitions();
createAlgorithmDefinition = RulesFactory.eINSTANCE.createAlgorithmDefinition();
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmParameters("");
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmType("MostCommon");
// $NON-NLS-1$
defaultSurvivorshipDefinitions.add(createDefaultsurvivShip("String", createAlgorithmDefinition));
// DefaultSurvivorship
EList<ParticularDefaultSurvivorshipDefinitions> particularDefaultSurvivorshipDefinitions = createMatchRuleDefinition.getParticularDefaultSurvivorshipDefinitions();
createAlgorithmDefinition = RulesFactory.eINSTANCE.createAlgorithmDefinition();
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmParameters("");
// $NON-NLS-1$
createAlgorithmDefinition.setAlgorithmType("Concatenate");
particularDefaultSurvivorshipDefinitions.add(createParticularDefaultSurvivorshipDefinitions("a2", // $NON-NLS-1$
createAlgorithmDefinition));
// init columnMap
Map<MetadataColumn, String> columnMap = new HashMap<MetadataColumn, String>();
MetadataColumn col0 = ConnectionFactory.eINSTANCE.createMetadataColumn();
// $NON-NLS-1$
col0.setName("a1");
// $NON-NLS-1$
col0.setTalendType("id_String");
// $NON-NLS-1$
columnMap.put(col0, "0");
MetadataColumn col1 = ConnectionFactory.eINSTANCE.createMetadataColumn();
// $NON-NLS-1$
col1.setTalendType("id_String");
// $NON-NLS-1$
col1.setName("a2");
// $NON-NLS-1$
columnMap.put(col1, "1");
MetadataColumn col2 = ConnectionFactory.eINSTANCE.createMetadataColumn();
// $NON-NLS-1$
col2.setTalendType("id_String");
// $NON-NLS-1$
col2.setName("a3");
// $NON-NLS-1$
columnMap.put(col2, "2");
MatchGroupResultConsumer matchGroupResultConsumer = new MatchGroupResultConsumer(true) {
@Override
public void handle(Object row) {
// no need to implement
}
};
AnalysisMatchRecordGrouping analysisMatchRecordGrouping = new AnalysisMatchRecordGrouping(matchGroupResultConsumer);
CombinedRecordMatcher combinedRecordMatcher = analysisMatchRecordGrouping.getCombinedRecordMatcher();
DQMFBRecordMatcher dqmfbRecordMatcher = new DQMFBRecordMatcher(0.9);
combinedRecordMatcher.getMatchers().add(dqmfbRecordMatcher);
List<List<Map<String, String>>> multiMatchRules = analysisMatchRecordGrouping.getMultiMatchRules();
List<Map<String, String>> matchRuleList = new ArrayList<Map<String, String>>();
Map<String, String> matchKeyMap1 = new HashMap<String, String>();
Map<String, String> matchKeyMap2 = new HashMap<String, String>();
Map<String, String> matchKeyMap3 = new HashMap<String, String>();
matchKeyMap1.put(IRecordGrouping.MATCHING_TYPE, AttributeMatcherType.EXACT.name());
// change by
matchKeyMap2.put(IRecordGrouping.MATCHING_TYPE, AttributeMatcherType.DUMMY.name());
// DefaultSurvivorshipDefinitions
// change by
matchKeyMap3.put(IRecordGrouping.MATCHING_TYPE, AttributeMatcherType.DUMMY.name());
// ParticularDefaultSurvivorshipDefinitions
matchRuleList.add(matchKeyMap1);
matchRuleList.add(matchKeyMap2);
matchRuleList.add(matchKeyMap3);
// $NON-NLS-1$
matchKeyMap1.put(IRecordGrouping.MATCH_KEY_NAME, "a1");
// $NON-NLS-1$
matchKeyMap2.put(IRecordGrouping.MATCH_KEY_NAME, "a2");
// $NON-NLS-1$
matchKeyMap3.put(IRecordGrouping.MATCH_KEY_NAME, "a3");
// $NON-NLS-1$
matchKeyMap1.put(IRecordGrouping.COLUMN_IDX, "0");
// $NON-NLS-1$
matchKeyMap2.put(IRecordGrouping.COLUMN_IDX, "1");
// $NON-NLS-1$
matchKeyMap3.put(IRecordGrouping.COLUMN_IDX, "2");
multiMatchRules.add(matchRuleList);
SurvivorShipAlgorithmParams createSurvivorShipAlgorithmParams = AnalysisRecordGroupingUtils.createSurvivorShipAlgorithmParams(analysisMatchRecordGrouping, recordMatchingIndicator, columnMap);
// $NON-NLS-1$
Assert.assertEquals(// $NON-NLS-1$
"The size of SurvivorShipAlgos should be 2", // $NON-NLS-1$
2, createSurvivorShipAlgorithmParams.getSurviorShipAlgos().length);
// the size of default survivorshipRules is come from by (column size * default item size)
Assert.assertEquals("The size of DefaultSurviorshipRules should be 3", 3, // $NON-NLS-1$
createSurvivorShipAlgorithmParams.getDefaultSurviorshipRules().size());
Map<IRecordMatcher, SurvivorshipFunction[]> survivorshipAlgosMap = createSurvivorShipAlgorithmParams.getSurvivorshipAlgosMap();
// $NON-NLS-1$
Assert.assertEquals("The size of survivorshipAlgosMap should be 1", 1, survivorshipAlgosMap.size());
SurvivorshipFunction[] survivorshipFunctions = survivorshipAlgosMap.get(dqmfbRecordMatcher);
// $NON-NLS-1$
Assert.assertEquals("The size of survivorshipFunctions should be 3", 3, survivorshipFunctions.length);
// $NON-NLS-1$
Assert.assertEquals(// $NON-NLS-1$
"The Algorithm of a1 function should be LONGEST", // $NON-NLS-1$
SurvivorShipAlgorithmEnum.LONGEST, survivorshipFunctions[0].getSurvivorShipAlgoEnum());
// $NON-NLS-1$
Assert.assertEquals(// $NON-NLS-1$
"The Algorithm of a2 function should be Concatenate", // $NON-NLS-1$
SurvivorShipAlgorithmEnum.CONCATENATE, survivorshipFunctions[1].getSurvivorShipAlgoEnum());
// $NON-NLS-1$
Assert.assertEquals(// $NON-NLS-1$
"The Algorithm of a3 function should be MostCommon", // $NON-NLS-1$
SurvivorShipAlgorithmEnum.MOST_COMMON, survivorshipFunctions[2].getSurvivorShipAlgoEnum());
}
use of org.talend.dataquality.rules.MatchRuleDefinition in project tdq-studio-se by Talend.
the class ExecuteMatchRuleHandlerTest method testExecute4.
/**
* Test method for
* {@link org.talend.dq.analysis.ExecuteMatchRuleHandler#execute(java.util.Map, org.talend.dataquality.indicators.columnset.RecordMatchingIndicator, java.util.List, org.talend.dataquality.indicators.columnset.BlockKeyIndicator)}
* .
*
* same to case 3 but the match rule order is exchange
*/
@Test
public void testExecute4() {
Map<MetadataColumn, String> columnMap = new HashMap<MetadataColumn, String>();
MetadataColumn col0 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col0.setName(columnName0);
// $NON-NLS-1$
columnMap.put(col0, "0");
MetadataColumn col1 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col1.setName(columnName1);
// $NON-NLS-1$
columnMap.put(col1, "1");
MetadataColumn col2 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col2.setName(columnName2);
// $NON-NLS-1$
columnMap.put(col2, "2");
MetadataColumn col3 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col3.setName(columnName3);
// $NON-NLS-1$
columnMap.put(col3, "3");
RecordMatchingIndicator recordMatchingIndicator = ColumnsetFactory.eINSTANCE.createRecordMatchingIndicator();
MatchRuleDefinition matchRuleDef = RulesPackage.eINSTANCE.getRulesFactory().createMatchRuleDefinition();
recordMatchingIndicator.setBuiltInMatchRuleDefinition(matchRuleDef);
// create match rule
MatchRule matchRule2 = RulesFactory.eINSTANCE.createMatchRule();
MatchKeyDefinition createMatchKeyDefinition2 = RulesFactory.eINSTANCE.createMatchKeyDefinition();
matchRule2.getMatchKeys().add(createMatchKeyDefinition2);
createMatchKeyDefinition2.setColumn(columnName3);
createMatchKeyDefinition2.setConfidenceWeight(1);
// $NON-NLS-1$
createMatchKeyDefinition2.setName("rule1.matchkey1");
createMatchKeyDefinition2.setHandleNull(HandleNullEnum.NULL_MATCH_NULL.getValue());
AlgorithmDefinition createAlgorithmDefinition2 = RulesFactory.eINSTANCE.createAlgorithmDefinition();
createAlgorithmDefinition2.setAlgorithmType(AttributeMatcherType.EXACT.name());
createMatchKeyDefinition2.setAlgorithm(createAlgorithmDefinition2);
matchRuleDef.getMatchRules().add(matchRule2);
// create match rule
MatchRule matchRule1 = RulesFactory.eINSTANCE.createMatchRule();
MatchKeyDefinition createMatchKeyDefinition1 = RulesFactory.eINSTANCE.createMatchKeyDefinition();
matchRule1.getMatchKeys().add(createMatchKeyDefinition1);
createMatchKeyDefinition1.setColumn(columnName2);
createMatchKeyDefinition1.setConfidenceWeight(1);
// $NON-NLS-1$
createMatchKeyDefinition1.setName("rule1.matchkey1");
createMatchKeyDefinition1.setHandleNull(HandleNullEnum.NULL_MATCH_NULL.getValue());
AlgorithmDefinition createAlgorithmDefinition1 = RulesFactory.eINSTANCE.createAlgorithmDefinition();
createAlgorithmDefinition1.setAlgorithmType(AttributeMatcherType.EXACT.name());
createMatchKeyDefinition1.setAlgorithm(createAlgorithmDefinition1);
matchRuleDef.getMatchRules().add(matchRule1);
// create block key
BlockKeyDefinition createBlockKeyDefinition = RulesFactory.eINSTANCE.createBlockKeyDefinition();
createBlockKeyDefinition.setColumn(columnName1);
// $NON-NLS-1$
createBlockKeyDefinition.setName("blockKey1");
// setPreAlgorithm
AlgorithmDefinition blockPreAlgorithm = RulesFactory.eINSTANCE.createAlgorithmDefinition();
blockPreAlgorithm.setAlgorithmType(BlockingKeyPreAlgorithmEnum.NON_ALGO.getValue());
createBlockKeyDefinition.setPreAlgorithm(blockPreAlgorithm);
// setAlgorithm
AlgorithmDefinition blockAlgorithm = RulesFactory.eINSTANCE.createAlgorithmDefinition();
blockAlgorithm.setAlgorithmType(BlockingKeyAlgorithmEnum.EXACT.getValue());
createBlockKeyDefinition.setAlgorithm(blockAlgorithm);
// setPostAlgorithm
AlgorithmDefinition blockPostAlgorithm = RulesFactory.eINSTANCE.createAlgorithmDefinition();
blockPostAlgorithm.setAlgorithmType(BlockingKeyPostAlgorithmEnum.NON_ALGO.getValue());
createBlockKeyDefinition.setPostAlgorithm(blockPostAlgorithm);
matchRuleDef.getBlockKeys().add(createBlockKeyDefinition);
List<Object[]> matchRows = new ArrayList<Object[]>();
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id1", "name1", "number1", "date1" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id2", "name1", "number2", "date1" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id3", "name2", "number2", "date3" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id4", "name2", "number2", "date1" });
BlockKeyIndicator blockKeyIndicator = ColumnsetFactory.eINSTANCE.createBlockKeyIndicator();
ExecuteMatchRuleHandler execHandler = new ExecuteMatchRuleHandler();
MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(columnMap, recordMatchingIndicator);
TypedReturnCode<MatchGroupResultConsumer> executeResult = execHandler.execute(columnMap, recordMatchingIndicator, matchRows, blockKeyIndicator, matchResultConsumer);
Assert.assertTrue(executeResult.isOk());
Assert.assertTrue(executeResult.getMessage() == null);
Assert.assertTrue(executeResult.getObject() != null);
MatchGroupResultConsumer ResultConsumer = executeResult.getObject();
List<Object[]> fullMatchResult = ResultConsumer.getFullMatchResult();
Assert.assertTrue(fullMatchResult.size() == 4);
for (int i = 0; i < fullMatchResult.size(); i++) {
Object[] objectArray = fullMatchResult.get(i);
Object masterValue = objectArray[7];
Object idValue = objectArray[0];
// id2 is because of matchRule1 id4 is because of matchRule2
if ("id2".equals(idValue) || "id4".equals(idValue)) {
// $NON-NLS-1$ //$NON-NLS-2$
Assert.assertFalse(Boolean.parseBoolean(masterValue.toString()));
} else {
Assert.assertTrue(Boolean.parseBoolean(masterValue.toString()));
}
}
}
use of org.talend.dataquality.rules.MatchRuleDefinition in project tdq-studio-se by Talend.
the class ExecuteMatchRuleHandlerTest method testExecute1.
/**
* Test method for
* {@link org.talend.dq.analysis.ExecuteMatchRuleHandler#execute(java.util.Map, org.talend.dataquality.indicators.columnset.RecordMatchingIndicator, java.util.List, org.talend.dataquality.indicators.columnset.BlockKeyIndicator)}
* .
*
* no block key one match key
*/
@Test
public void testExecute1() {
Map<MetadataColumn, String> columnMap = new HashMap<MetadataColumn, String>();
MetadataColumn col0 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col0.setName(columnName0);
// $NON-NLS-1$
columnMap.put(col0, "0");
MetadataColumn col1 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col1.setName(columnName1);
// $NON-NLS-1$
columnMap.put(col1, "1");
MetadataColumn col2 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col2.setName(columnName2);
// $NON-NLS-1$
columnMap.put(col2, "2");
MetadataColumn col3 = ConnectionFactory.eINSTANCE.createMetadataColumn();
col3.setName(columnName3);
// $NON-NLS-1$
columnMap.put(col3, "3");
// create match key
RecordMatchingIndicator recordMatchingIndicator = ColumnsetFactory.eINSTANCE.createRecordMatchingIndicator();
MatchRuleDefinition matchRuleDef = RulesPackage.eINSTANCE.getRulesFactory().createMatchRuleDefinition();
recordMatchingIndicator.setBuiltInMatchRuleDefinition(matchRuleDef);
MatchRule createMatchRule1 = RulesFactory.eINSTANCE.createMatchRule();
MatchKeyDefinition createMatchKeyDefinition1 = RulesFactory.eINSTANCE.createMatchKeyDefinition();
createMatchRule1.getMatchKeys().add(createMatchKeyDefinition1);
createMatchKeyDefinition1.setColumn(columnName0);
createMatchKeyDefinition1.setConfidenceWeight(1);
// $NON-NLS-1$
createMatchKeyDefinition1.setName("rule1.matchkey1");
createMatchKeyDefinition1.setHandleNull(HandleNullEnum.NULL_MATCH_NULL.getValue());
AlgorithmDefinition createAlgorithmDefinition1 = RulesFactory.eINSTANCE.createAlgorithmDefinition();
createAlgorithmDefinition1.setAlgorithmType(AttributeMatcherType.EXACT.name());
createMatchKeyDefinition1.setAlgorithm(createAlgorithmDefinition1);
matchRuleDef.getMatchRules().add(createMatchRule1);
// input data
List<Object[]> matchRows = new ArrayList<Object[]>();
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id1", "name1", "number1", "date1" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id2", "name2", "number2", "date2" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id3", "name1", "number3", "date3" });
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
matchRows.add(new String[] { "id4", "name4", "number2", "date1" });
BlockKeyIndicator blockKeyIndicator = ColumnsetFactory.eINSTANCE.createBlockKeyIndicator();
ExecuteMatchRuleHandler execHandler = new ExecuteMatchRuleHandler();
MatchGroupResultConsumer matchResultConsumer = createMatchGroupResultConsumer(columnMap, recordMatchingIndicator);
TypedReturnCode<MatchGroupResultConsumer> executeResult = execHandler.execute(columnMap, recordMatchingIndicator, matchRows, blockKeyIndicator, matchResultConsumer);
Assert.assertTrue(executeResult.isOk());
Assert.assertTrue(executeResult.getMessage() == null);
Assert.assertTrue(executeResult.getObject() != null);
MatchGroupResultConsumer ResultConsumer = executeResult.getObject();
List<Object[]> fullMatchResult = ResultConsumer.getFullMatchResult();
Assert.assertTrue(fullMatchResult.size() == 4);
// every input data is master data
for (Object[] objectArray : fullMatchResult) {
Object object = objectArray[7];
Assert.assertTrue(Boolean.parseBoolean(object.toString()));
}
}
use of org.talend.dataquality.rules.MatchRuleDefinition in project tdq-studio-se by Talend.
the class FileTreeLabelProviderTest method setUpBeforeClass.
/**
* DOC zshen Comment method "setUpBeforeClass".
*
* @throws java.lang.Exception
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
DQRule createSqlRule = RulesFactory.eINSTANCE.createWhereRule();
ParserRule createParserRule = RulesFactory.eINSTANCE.createParserRule();
MatchRuleDefinition createMatchRuleDefinition = RulesFactory.eINSTANCE.createMatchRuleDefinition();
// $NON-NLS-1$
createSqlRule.setName("sqlRule1");
// $NON-NLS-1$
createParserRule.setName("parserRule1");
// $NON-NLS-1$
createMatchRuleDefinition.setName("matchRule");
createTDQSQLRuleItem = PropertiesFactory.eINSTANCE.createTDQBusinessRuleItem();
createTDQSQLRuleItem.setDqrule(createSqlRule);
createTDQParserRuleItem = PropertiesFactory.eINSTANCE.createTDQBusinessRuleItem();
createTDQParserRuleItem.setDqrule(createParserRule);
createTDQMatchRuleItem = PropertiesFactory.eINSTANCE.createTDQMatchRuleItem();
createTDQMatchRuleItem.setMatchRule(createMatchRuleDefinition);
Property createSQLRuleProperty = org.talend.core.model.properties.PropertiesFactory.eINSTANCE.createProperty();
createSQLRuleProperty.setItem(createTDQSQLRuleItem);
createSQLRuleProperty.setLabel(createSqlRule.getName());
Property createParserRuleProperty = org.talend.core.model.properties.PropertiesFactory.eINSTANCE.createProperty();
createParserRuleProperty.setItem(createTDQParserRuleItem);
createParserRuleProperty.setLabel(createParserRule.getName());
Property createMatchRuleProperty = org.talend.core.model.properties.PropertiesFactory.eINSTANCE.createProperty();
createMatchRuleProperty.setItem(createTDQMatchRuleItem);
createMatchRuleProperty.setLabel(createMatchRuleDefinition.getName());
ProxyRepositoryFactory.getInstance().create(createTDQSQLRuleItem, Path.EMPTY);
ProxyRepositoryFactory.getInstance().create(createTDQParserRuleItem, Path.EMPTY);
ProxyRepositoryFactory.getInstance().create(createTDQMatchRuleItem, Path.EMPTY);
}
use of org.talend.dataquality.rules.MatchRuleDefinition in project tdq-studio-se by Talend.
the class RecordMatchingIndicatorImplTest method setUp.
/**
* DOC zhao Comment method "setUp".
*
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
// Match key: street
MatchRuleDefinition definition = RulesPackage.eINSTANCE.getRulesFactory().createMatchRuleDefinition();
recordMatchingIndicator.setBuiltInMatchRuleDefinition(definition);
recordMatchingIndicator.setMatchRowSchema(COLUMN_SCHEMAS);
}
Aggregations