Search in sources :

Example 21 with Tuple

use of edu.uci.ics.textdb.api.tuple.Tuple in project textdb by TextDB.

the class FuzzyTokenMatcherTest method TestFuzzyTokenMatcher1.

@Test
public void TestFuzzyTokenMatcher1() throws Exception {
    String query = "Twelve Angry Men";
    // The ratio of tokens that need to be matched
    double threshold = 0.5;
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(TestConstants.DESCRIPTION);
    Schema schema = Utils.addAttributeToSchema(TestConstants.SCHEMA_PEOPLE, RESULTS_ATTR);
    List<Span> spanList1 = Arrays.asList(new Span(TestConstants.DESCRIPTION, 5, 10, "angry", "Angry", 1));
    IField[] fields1 = { new StringField("bruce"), new StringField("john Lee"), new IntegerField(46), new DoubleField(5.50), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-14-1970")), new TextField("Tall Angry"), new ListField<Span>(spanList1) };
    List<Span> spanList2 = Arrays.asList(new Span(TestConstants.DESCRIPTION, 6, 11, "angry", "Angry", 1));
    IField[] fields2 = { new StringField("brad lie angelina"), new StringField("pitt"), new IntegerField(44), new DoubleField(6.10), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-12-1972")), new TextField("White Angry"), new ListField<Span>(spanList2) };
    List<Span> spanList3 = Arrays.asList(new Span(TestConstants.DESCRIPTION, 40, 45, "angry", "Angry", 8));
    IField[] fields3 = { new StringField("george lin lin"), new StringField("lin clooney"), new IntegerField(43), new DoubleField(6.06), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1973")), new TextField("Lin Clooney is Short and lin clooney is Angry"), new ListField<Span>(spanList3) };
    List<Span> spanList4 = Arrays.asList(new Span(TestConstants.DESCRIPTION, 6, 11, "angry", "angry", 1));
    IField[] fields4 = { new StringField("Mary brown"), new StringField("Lake Forest"), new IntegerField(42), new DoubleField(5.99), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1974")), new TextField("Short angry"), new ListField<Span>(spanList4) };
    Tuple tuple1 = new Tuple(schema, fields1);
    Tuple tuple2 = new Tuple(schema, fields2);
    Tuple tuple3 = new Tuple(schema, fields3);
    Tuple tuple4 = new Tuple(schema, fields4);
    List<Tuple> expectedResultList = new ArrayList<>();
    expectedResultList.add(tuple1);
    expectedResultList.add(tuple2);
    expectedResultList.add(tuple3);
    expectedResultList.add(tuple4);
    List<Tuple> results = FuzzyTokenMatcherTestHelper.getQueryResults(PEOPLE_TABLE, query, threshold, attributeNames);
    boolean contains = TestUtils.equals(expectedResultList, results);
    Assert.assertTrue(contains);
}
Also used : Schema(edu.uci.ics.textdb.api.schema.Schema) ArrayList(java.util.ArrayList) IntegerField(edu.uci.ics.textdb.api.field.IntegerField) IField(edu.uci.ics.textdb.api.field.IField) Span(edu.uci.ics.textdb.api.span.Span) StringField(edu.uci.ics.textdb.api.field.StringField) TextField(edu.uci.ics.textdb.api.field.TextField) DateField(edu.uci.ics.textdb.api.field.DateField) SimpleDateFormat(java.text.SimpleDateFormat) DoubleField(edu.uci.ics.textdb.api.field.DoubleField) Tuple(edu.uci.ics.textdb.api.tuple.Tuple) Test(org.junit.Test)

Example 22 with Tuple

use of edu.uci.ics.textdb.api.tuple.Tuple in project textdb by TextDB.

the class KeywordConjunctionTest method testMatchingWithLimit.

@Test
public void testMatchingWithLimit() throws TextDBException, ParseException, java.text.ParseException {
    String query = "angry";
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(TestConstants.FIRST_NAME);
    attributeNames.add(TestConstants.LAST_NAME);
    attributeNames.add(TestConstants.DESCRIPTION);
    Attribute[] schemaAttributes = new Attribute[TestConstants.ATTRIBUTES_PEOPLE.length + 1];
    for (int count = 0; count < schemaAttributes.length - 1; count++) {
        schemaAttributes[count] = TestConstants.ATTRIBUTES_PEOPLE[count];
    }
    schemaAttributes[schemaAttributes.length - 1] = new Attribute(RESULTS, AttributeType.LIST);
    List<Tuple> resultList = KeywordTestHelper.getQueryResults(PEOPLE_TABLE, query, attributeNames, conjunction, 3, 0);
    List<Tuple> expectedList = new ArrayList<>();
    Span span1 = new Span("description", 5, 10, "angry", "Angry", 1);
    Span span2 = new Span("description", 6, 11, "angry", "Angry", 1);
    Span span3 = new Span("description", 40, 45, "angry", "Angry", 8);
    Span span4 = new Span("description", 6, 11, "angry", "angry", 1);
    List<Span> list1 = new ArrayList<>();
    list1.add(span1);
    IField[] fields1 = { new StringField("bruce"), new StringField("john Lee"), new IntegerField(46), new DoubleField(5.50), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-14-1970")), new TextField("Tall Angry"), new ListField<>(list1) };
    List<Span> list2 = new ArrayList<>();
    list2.add(span2);
    IField[] fields2 = { new StringField("brad lie angelina"), new StringField("pitt"), new IntegerField(44), new DoubleField(6.10), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-12-1972")), new TextField("White Angry"), new ListField<>(list2) };
    List<Span> list3 = new ArrayList<>();
    list3.add(span3);
    IField[] fields3 = { new StringField("george lin lin"), new StringField("lin clooney"), new IntegerField(43), new DoubleField(6.06), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1973")), new TextField("Lin Clooney is Short and lin clooney is Angry"), new ListField<>(list3) };
    List<Span> list4 = new ArrayList<>();
    list4.add(span4);
    IField[] fields4 = { new StringField("Mary brown"), new StringField("Lake Forest"), new IntegerField(42), new DoubleField(5.99), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1974")), new TextField("Short angry"), new ListField<>(list4) };
    Tuple tuple1 = new Tuple(new Schema(schemaAttributes), fields1);
    Tuple tuple2 = new Tuple(new Schema(schemaAttributes), fields2);
    Tuple tuple3 = new Tuple(new Schema(schemaAttributes), fields3);
    Tuple tuple4 = new Tuple(new Schema(schemaAttributes), fields4);
    expectedList.add(tuple1);
    expectedList.add(tuple2);
    expectedList.add(tuple3);
    expectedList.add(tuple4);
    Assert.assertEquals(expectedList.size(), 4);
    Assert.assertEquals(resultList.size(), 3);
    Assert.assertTrue(TestUtils.containsAll(expectedList, resultList));
}
Also used : Attribute(edu.uci.ics.textdb.api.schema.Attribute) Schema(edu.uci.ics.textdb.api.schema.Schema) ArrayList(java.util.ArrayList) IntegerField(edu.uci.ics.textdb.api.field.IntegerField) IField(edu.uci.ics.textdb.api.field.IField) Span(edu.uci.ics.textdb.api.span.Span) StringField(edu.uci.ics.textdb.api.field.StringField) TextField(edu.uci.ics.textdb.api.field.TextField) DateField(edu.uci.ics.textdb.api.field.DateField) SimpleDateFormat(java.text.SimpleDateFormat) Tuple(edu.uci.ics.textdb.api.tuple.Tuple) DoubleField(edu.uci.ics.textdb.api.field.DoubleField) Test(org.junit.Test)

Example 23 with Tuple

use of edu.uci.ics.textdb.api.tuple.Tuple in project textdb by TextDB.

the class KeywordPhraseTest method testCombinedSpanInMultipleFieldsQuery.

/**
     * Verifies: getNextTuple should return Combined Span info for the phrase
     * 
     * @throws Exception
     */
@Test
public void testCombinedSpanInMultipleFieldsQuery() throws Exception {
    // Prepare Query
    String query = "lin clooney";
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(TestConstants.FIRST_NAME);
    attributeNames.add(TestConstants.LAST_NAME);
    attributeNames.add(TestConstants.DESCRIPTION);
    // Prepare expected result list
    List<Span> list = new ArrayList<>();
    Span span1 = new Span("lastName", 0, 11, "lin clooney", "lin clooney");
    Span span2 = new Span("description", 0, 11, "lin clooney", "Lin Clooney");
    Span span3 = new Span("description", 25, 36, "lin clooney", "lin clooney");
    list.add(span1);
    list.add(span2);
    list.add(span3);
    Attribute[] schemaAttributes = new Attribute[TestConstants.ATTRIBUTES_PEOPLE.length + 1];
    for (int count = 0; count < schemaAttributes.length - 1; count++) {
        schemaAttributes[count] = TestConstants.ATTRIBUTES_PEOPLE[count];
    }
    schemaAttributes[schemaAttributes.length - 1] = new Attribute(RESULTS, AttributeType.LIST);
    IField[] fields1 = { new StringField("george lin lin"), new StringField("lin clooney"), new IntegerField(43), new DoubleField(6.06), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1973")), new TextField("Lin Clooney is Short and lin clooney is Angry"), new ListField<>(list) };
    Tuple tuple1 = new Tuple(new Schema(schemaAttributes), fields1);
    List<Tuple> expectedResultList = new ArrayList<>();
    expectedResultList.add(tuple1);
    // Perform Query
    List<Tuple> resultList = KeywordTestHelper.getQueryResults(PEOPLE_TABLE, query, attributeNames, phrase);
    // Perform Check
    boolean contains = TestUtils.equals(expectedResultList, resultList);
    Assert.assertTrue(contains);
}
Also used : Attribute(edu.uci.ics.textdb.api.schema.Attribute) Schema(edu.uci.ics.textdb.api.schema.Schema) ArrayList(java.util.ArrayList) IntegerField(edu.uci.ics.textdb.api.field.IntegerField) IField(edu.uci.ics.textdb.api.field.IField) Span(edu.uci.ics.textdb.api.span.Span) StringField(edu.uci.ics.textdb.api.field.StringField) TextField(edu.uci.ics.textdb.api.field.TextField) DateField(edu.uci.ics.textdb.api.field.DateField) SimpleDateFormat(java.text.SimpleDateFormat) DoubleField(edu.uci.ics.textdb.api.field.DoubleField) Tuple(edu.uci.ics.textdb.api.tuple.Tuple) Test(org.junit.Test)

Example 24 with Tuple

use of edu.uci.ics.textdb.api.tuple.Tuple in project textdb by TextDB.

the class KeywordPhraseTest method testWordInMultipleFieldsQueryWithStopWords1.

/**
     * Verifies: Query with Stop Words match corresponding phrases in the
     * document
     * 
     * @throws Exception
     */
@Test
public void testWordInMultipleFieldsQueryWithStopWords1() throws Exception {
    // Prepare Query
    String query = "lin and and angry";
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(TestConstants.FIRST_NAME);
    attributeNames.add(TestConstants.LAST_NAME);
    attributeNames.add(TestConstants.DESCRIPTION);
    // Prepare expected result list
    List<Span> list = new ArrayList<>();
    Span span1 = new Span("description", 25, 45, "lin and and angry", "lin clooney is Angry");
    list.add(span1);
    Attribute[] schemaAttributes = new Attribute[TestConstants.ATTRIBUTES_PEOPLE.length + 1];
    for (int count = 0; count < schemaAttributes.length - 1; count++) {
        schemaAttributes[count] = TestConstants.ATTRIBUTES_PEOPLE[count];
    }
    schemaAttributes[schemaAttributes.length - 1] = new Attribute(RESULTS, AttributeType.LIST);
    IField[] fields1 = { new StringField("george lin lin"), new StringField("lin clooney"), new IntegerField(43), new DoubleField(6.06), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-13-1973")), new TextField("Lin Clooney is Short and lin clooney is Angry"), new ListField<>(list) };
    Tuple tuple1 = new Tuple(new Schema(schemaAttributes), fields1);
    List<Tuple> expectedResultList = new ArrayList<>();
    expectedResultList.add(tuple1);
    // Perform Query
    List<Tuple> resultList = KeywordTestHelper.getQueryResults(PEOPLE_TABLE, query, attributeNames, phrase);
    // Perform Check
    boolean contains = TestUtils.equals(expectedResultList, resultList);
    Assert.assertTrue(contains);
}
Also used : Attribute(edu.uci.ics.textdb.api.schema.Attribute) Schema(edu.uci.ics.textdb.api.schema.Schema) ArrayList(java.util.ArrayList) IntegerField(edu.uci.ics.textdb.api.field.IntegerField) IField(edu.uci.ics.textdb.api.field.IField) Span(edu.uci.ics.textdb.api.span.Span) StringField(edu.uci.ics.textdb.api.field.StringField) TextField(edu.uci.ics.textdb.api.field.TextField) DateField(edu.uci.ics.textdb.api.field.DateField) SimpleDateFormat(java.text.SimpleDateFormat) DoubleField(edu.uci.ics.textdb.api.field.DoubleField) Tuple(edu.uci.ics.textdb.api.tuple.Tuple) Test(org.junit.Test)

Example 25 with Tuple

use of edu.uci.ics.textdb.api.tuple.Tuple in project textdb by TextDB.

the class KeywordPhraseTest method testWordInMultipleFieldsQueryWithStopWords5.

/**
     * Verifies: Query with Stop Words match corresponding phrases in the
     * document Used to cause exception sometimes if there is a space between
     * words
     * 
     * @throws Exception
     *             with Medline data
     */
@Test
public void testWordInMultipleFieldsQueryWithStopWords5() throws Exception {
    // Prepare Query
    String query = "gain weight";
    ArrayList<String> attributeNames = new ArrayList<>();
    attributeNames.add(keywordTestConstants.ABSTRACT);
    // Prepare expected result list
    List<Span> list = new ArrayList<>();
    Span span1 = new Span(keywordTestConstants.ABSTRACT, 26, 37, "gain weight", "gain weight");
    list.add(span1);
    Attribute[] schemaAttributes = new Attribute[keywordTestConstants.ATTRIBUTES_MEDLINE.length + 1];
    for (int count = 0; count < schemaAttributes.length - 1; count++) {
        schemaAttributes[count] = keywordTestConstants.ATTRIBUTES_MEDLINE[count];
    }
    schemaAttributes[schemaAttributes.length - 1] = new Attribute(RESULTS, AttributeType.LIST);
    IField[] fields = { new IntegerField(4566015), new TextField(""), new TextField("Significance of milk pH in newborn infants."), new TextField("V C Harrison, G Peat"), new StringField("4-5839 Dec 2, 1972"), new TextField("British medical journal"), new TextField(""), new TextField("Infant Nutritional Physiological Phenomena, Infant, Newborn, Milk"), new TextField("Bottle-fed infants do not gain weight as rapidly as breast-fed babies during the first week of life. This " + "weight lag can be corrected by the addition of a small amount of alkali (sodium bicarbonate or trometamol) to " + "the feeds. The alkali corrects the acidity of cow's milk which now assumes some of the properties of human breast " + "milk. It has a bacteriostatic effect on specific Escherichia coli in vitro, and in infants it produces a stool with" + " a preponderance of lactobacilli over E. coli organisms. When alkali is removed from the milk there is a decrease in" + " the weight of an infant and the stools contain excessive numbers of E. coli bacteria.A pH-corrected milk appears to" + " be more physiological than unaltered cow's milk and may provide some protection against gastroenteritis in early " + "life. Its bacteriostatic effect on specific E. coli may be of practical significance in feed preparations where " + "terminal sterilization and refrigeration are not available. The study was conducted during the week after birth, and " + "no conclusions are derived for older infants. The long-term effects of trometamol are unknown. No recommendation can " + "be given for the addition of sodium bicarbonate to milks containing a higher content of sodium."), new DoubleField(0.667832788), new ListField<>(list) };
    Tuple tuple1 = new Tuple(new Schema(schemaAttributes), fields);
    List<Tuple> expectedResultList = new ArrayList<>();
    expectedResultList.add(tuple1);
    List<Tuple> results = KeywordTestHelper.getQueryResults(MEDLINE_TABLE, query, attributeNames, phrase);
    // Perform Check
    boolean contains = TestUtils.equals(expectedResultList, results);
    Assert.assertTrue(contains);
}
Also used : Attribute(edu.uci.ics.textdb.api.schema.Attribute) Schema(edu.uci.ics.textdb.api.schema.Schema) ArrayList(java.util.ArrayList) IntegerField(edu.uci.ics.textdb.api.field.IntegerField) IField(edu.uci.ics.textdb.api.field.IField) Span(edu.uci.ics.textdb.api.span.Span) StringField(edu.uci.ics.textdb.api.field.StringField) TextField(edu.uci.ics.textdb.api.field.TextField) DoubleField(edu.uci.ics.textdb.api.field.DoubleField) Tuple(edu.uci.ics.textdb.api.tuple.Tuple) Test(org.junit.Test)

Aggregations

Tuple (edu.uci.ics.textdb.api.tuple.Tuple)234 ArrayList (java.util.ArrayList)144 Test (org.junit.Test)135 IField (edu.uci.ics.textdb.api.field.IField)102 Schema (edu.uci.ics.textdb.api.schema.Schema)95 TextField (edu.uci.ics.textdb.api.field.TextField)92 StringField (edu.uci.ics.textdb.api.field.StringField)85 Attribute (edu.uci.ics.textdb.api.schema.Attribute)84 IntegerField (edu.uci.ics.textdb.api.field.IntegerField)80 Span (edu.uci.ics.textdb.api.span.Span)80 DoubleField (edu.uci.ics.textdb.api.field.DoubleField)64 DateField (edu.uci.ics.textdb.api.field.DateField)61 SimpleDateFormat (java.text.SimpleDateFormat)60 DataWriter (edu.uci.ics.textdb.storage.DataWriter)31 Dictionary (edu.uci.ics.textdb.exp.dictionarymatcher.Dictionary)25 KeywordMatcherSourceOperator (edu.uci.ics.textdb.exp.keywordmatcher.KeywordMatcherSourceOperator)20 RelationManager (edu.uci.ics.textdb.storage.RelationManager)19 JoinDistancePredicate (edu.uci.ics.textdb.exp.join.JoinDistancePredicate)18 ScanBasedSourceOperator (edu.uci.ics.textdb.exp.source.scan.ScanBasedSourceOperator)17 ScanSourcePredicate (edu.uci.ics.textdb.exp.source.scan.ScanSourcePredicate)17