Examples with KeywordSourcePredicate edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate used on opensource projects

Example 1 with KeywordSourcePredicate

use of edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate in project textdb by TextDB.

the class JoinTestHelper method getKeywordSource.

/**
     * Provides a KeywordMatcherSourceOperator for a test table given a keyword.
     * ( KeywordMatcher is used in most of Join test cases )
     * @param tableName
     * @param query
     * @param matchingType
     * @return
     * @throws TextDBException
     */
public static KeywordMatcherSourceOperator getKeywordSource(String tableName, String query, KeywordMatchingType matchingType) throws TextDBException {
    KeywordSourcePredicate keywordSourcePredicate = new KeywordSourcePredicate(query, Arrays.asList(JoinTestConstants.AUTHOR, JoinTestConstants.TITLE, JoinTestConstants.REVIEW), RelationManager.getRelationManager().getTableAnalyzerString(tableName), matchingType, tableName, SchemaConstants.SPAN_LIST);
    KeywordMatcherSourceOperator keywordSource = new KeywordMatcherSourceOperator(keywordSourcePredicate);
    return keywordSource;
}

Example 2 with KeywordSourcePredicate

use of edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate in project textdb by TextDB.

the class PredicateBaseTest method testKeyword.

@Test
public void testKeyword() throws Exception {
    KeywordPredicate keywordPredicate = new KeywordPredicate("keyword", attributeNames, "standard", KeywordMatchingType.CONJUNCTION_INDEXBASED, "keywordResults");
    testPredicate(keywordPredicate);
    KeywordSourcePredicate keywordSourcePredicate = new KeywordSourcePredicate("keyword", attributeNames, "standard", KeywordMatchingType.CONJUNCTION_INDEXBASED, "tableName", "keywordSourceResults");
    testPredicate(keywordSourcePredicate);
}

Example 3 with KeywordSourcePredicate

use of edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate in project textdb by TextDB.

the class DictionaryMatcherSourceOperator method open.

/**
     * @about Opens dictionary matcher. Must call open() before calling
     *        getNextTuple().
     */
@Override
public void open() throws DataFlowException {
    try {
        currentDictionaryEntry = predicate.getDictionary().getNextEntry();
        if (currentDictionaryEntry == null) {
            throw new DataFlowException("Dictionary is empty");
        }
        if (predicate.getKeywordMatchingType() == KeywordMatchingType.SUBSTRING_SCANBASED) {
            // For Substring matching, create a scan source operator.
            indexSource = new ScanBasedSourceOperator(new ScanSourcePredicate(predicate.getTableName()));
            indexSource.open();
            // Substring matching's output schema needs to contains span
            // list.
            inputSchema = indexSource.getOutputSchema();
            outputSchema = inputSchema;
            if (inputSchema.containsField(predicate.getSpanListName())) {
                throw new DataFlowException(ErrorMessages.DUPLICATE_ATTRIBUTE(predicate.getSpanListName(), inputSchema));
            }
            outputSchema = Utils.addAttributeToSchema(outputSchema, new Attribute(predicate.getSpanListName(), AttributeType.LIST));
        } else {
            // For other keyword matching types (conjunction and phrase),
            // create keyword matcher based on index.
            keywordSource = new KeywordMatcherSourceOperator(new KeywordSourcePredicate(currentDictionaryEntry, predicate.getAttributeNames(), predicate.getAnalyzerString(), predicate.getKeywordMatchingType(), predicate.getTableName(), predicate.getSpanListName()));
            keywordSource.open();
            // Other keyword matching types uses a KeywordMatcher, so the
            // output schema is the same as keywordMatcher's schema
            inputSchema = keywordSource.getOutputSchema();
            outputSchema = keywordSource.getOutputSchema();
        }
    } catch (Exception e) {
        throw new DataFlowException(e.getMessage(), e);
    }
}

Example 4 with KeywordSourcePredicate

use of edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate in project textdb by TextDB.

the class KeywordMatcherPerformanceTest method match.

/*
     * This function does match for a list of queries
     */
public static void match(ArrayList<String> queryList, KeywordMatchingType opType, String luceneAnalyzerStr, String tableName) throws TextDBException, IOException {
    Attribute[] attributeList = new Attribute[] { MedlineIndexWriter.ABSTRACT_ATTR };
    for (String query : queryList) {
        KeywordSourcePredicate predicate = new KeywordSourcePredicate(query, Utils.getAttributeNames(attributeList), luceneAnalyzerStr, opType, tableName, null);
        KeywordMatcherSourceOperator keywordSource = new KeywordMatcherSourceOperator(predicate);
        long startMatchTime = System.currentTimeMillis();
        keywordSource.open();
        int counter = 0;
        Tuple nextTuple = null;
        while ((nextTuple = keywordSource.getNextTuple()) != null) {
            ListField<Span> spanListField = nextTuple.getField(SchemaConstants.SPAN_LIST);
            List<Span> spanList = spanListField.getValue();
            counter += spanList.size();
        }
        keywordSource.close();
        long endMatchTime = System.currentTimeMillis();
        double matchTime = (endMatchTime - startMatchTime) / 1000.0;
        timeResults.add(Double.parseDouble(String.format("%.4f", matchTime)));
        totalResultCount += counter;
    }
}

Example 5 with KeywordSourcePredicate

use of edu.uci.ics.textdb.exp.keywordmatcher.KeywordSourcePredicate in project textdb by TextDB.

the class DictionaryMatcherSourceOperator method getNextTuple.

/**
     * @about Gets the next matched tuple. <br>
     *        Returns the tuple with results in spanList. <br>
     * 
     *        Performs SCAN, KEYWORD_BASIC, or KEYWORD_PHRASE depends on the
     *        dictionary predicate. <br>
     * 
     *        DictionaryOperatorType.SCAN: <br>
     *        Scan the tuples using ScanSourceOperator. <br>
     *        For each tuple, loop through the dictionary and find results. <br>
     *        We assume the dictionary is smaller than the data at the source
     *        operator, we treat the data source as the outer relation to reduce
     *        the number of disk IOs. <br>
     * 
     *        DictionaryOperatorType.KEYWORD_BASIC, KEYWORD_PHRASE: <br>
     *        Use KeywordMatcher to find results. <br>
     * 
     *        KEYWORD_BASIC corresponds to KeywordOperatorType.BASIC, which
     *        performs keyword search on the document. The input query is
     *        tokenized. The order of the tokens doesn't matter. <br>
     * 
     *        KEYWORD_PHRASE corresponds to KeywordOperatorType.PHRASE, which
     *        performs phrase search on the document. The input query is
     *        tokenized. The order of the tokens does matter. Stopwords are
     *        treated as placeholders to indicate an arbitary token. <br>
     * 
     */
@Override
public Tuple getNextTuple() throws TextDBException {
    if (resultCursor >= limit + offset - 1) {
        return null;
    }
    if (predicate.getKeywordMatchingType() == KeywordMatchingType.PHRASE_INDEXBASED || predicate.getKeywordMatchingType() == KeywordMatchingType.CONJUNCTION_INDEXBASED) {
        while (true) {
            // If there's result from current keywordMatcher, return it.
            if ((sourceTuple = keywordSource.getNextTuple()) != null) {
                resultCursor++;
                if (resultCursor >= offset) {
                    return sourceTuple;
                }
                continue;
            }
            // return null if reach the end of dictionary.
            if ((currentDictionaryEntry = predicate.getDictionary().getNextEntry()) == null) {
                return null;
            }
            // Construct a new KeywordMatcher with the new dictionary entry.
            KeywordMatchingType keywordMatchingType;
            if (predicate.getKeywordMatchingType() == KeywordMatchingType.PHRASE_INDEXBASED) {
                keywordMatchingType = KeywordMatchingType.PHRASE_INDEXBASED;
            } else {
                keywordMatchingType = KeywordMatchingType.CONJUNCTION_INDEXBASED;
            }
            keywordSource.close();
            KeywordSourcePredicate keywordSourcePredicate = new KeywordSourcePredicate(currentDictionaryEntry, predicate.getAttributeNames(), predicate.getAnalyzerString(), keywordMatchingType, predicate.getTableName(), predicate.getSpanListName());
            keywordSource = new KeywordMatcherSourceOperator(keywordSourcePredicate);
            keywordSource.open();
        }
    } else // Substring matching (based on scan)
    {
        Tuple sourceTuple;
        Tuple resultTuple = null;
        while ((sourceTuple = indexSource.getNextTuple()) != null) {
            sourceTuple = DataflowUtils.getSpanTuple(sourceTuple.getFields(), new ArrayList<Span>(), outputSchema);
            resultTuple = computeMatchingResult(currentDictionaryEntry, sourceTuple);
            if (resultTuple != null) {
                resultCursor++;
            }
            if (resultTuple != null && resultCursor >= offset) {
                break;
            }
        }
        return resultTuple;
    }
}