use of de.ids_mannheim.korap.query.wrap.SpanSegmentQueryWrapper in project Krill by KorAP.
the class TestIndex method indexLucene.
@Test
public void indexLucene() throws Exception {
// Base analyzer for searching and indexing
StandardAnalyzer analyzer = new StandardAnalyzer();
// Based on
// http://lucene.apache.org/core/4_0_0/core/org/apache/lucene/
// analysis/Analyzer.html?is-external=true
// Create configuration with base analyzer
IndexWriterConfig config = new IndexWriterConfig(analyzer);
// Add a document 1 with the correct fields
IndexWriter w = new IndexWriter(index, config);
Collection docs = initIndexer();
@SuppressWarnings("unchecked") Iterator<Map<String, String>> i = (Iterator<Map<String, String>>) docs.iterator();
for (; i.hasNext(); ) {
addDoc(w, i.next());
}
;
assertEquals(3, w.numDocs());
w.close();
// Check directory
DirectoryReader reader = DirectoryReader.open(index);
assertEquals(docs.size(), reader.maxDoc());
assertEquals(docs.size(), reader.numDocs());
// Check searcher
IndexSearcher searcher = new IndexSearcher(reader);
// textClass
// All texts of text class "news"
assertEquals(2, searcher.search(new TermQuery(new Term("textClass", "news")), 10).totalHits);
// textClass
// All texts of text class "sports"
assertEquals(2, searcher.search(new TermQuery(new Term("textClass", "sports")), 10).totalHits);
// TextIndex
// All docs containing "l:nehmen"
assertEquals(1, searcher.search(new TermQuery(new Term("text", "l:nehmen")), 10).totalHits);
// TextIndex
// All docs containing "s:den"
assertEquals(2, searcher.search(new TermQuery(new Term("text", "s:den")), 10).totalHits);
/*
assertEquals(3,
searcher.search(
new TermQuery(
new Term("text", "T")
), 10
).totalHits
);
*/
// BooleanQuery
// All docs containing "s:den" and "l:sie"
TermQuery s_den = new TermQuery(new Term("text", "s:den"));
TermQuery l_sie = new TermQuery(new Term("text", "l:sie"));
BooleanQuery bool = new BooleanQuery();
bool.add(s_den, BooleanClause.Occur.MUST);
bool.add(l_sie, BooleanClause.Occur.MUST);
assertEquals(1, searcher.search(bool, 10).totalHits);
// BooleanQuery
// All docs containing "s:den" or "l:sie"
bool = new BooleanQuery();
bool.add(s_den, BooleanClause.Occur.SHOULD);
bool.add(l_sie, BooleanClause.Occur.SHOULD);
assertEquals(2, searcher.search(bool, 10).totalHits);
// RegexpQuery
// All docs containing ".{4}en" (liefen und Hunden)
RegexpQuery srquery = new RegexpQuery(new Term("text", "s:.{4}en"));
assertEquals(2, searcher.search(srquery, 10).totalHits);
// RegexpQuery
// All docs containing "E." (Er) (2x)
srquery = new RegexpQuery(new Term("text", "s:E."));
assertEquals(2, searcher.search(srquery, 10).totalHits);
SpanRegexQueryWrapper ssrquery = new SpanRegexQueryWrapper("text", "s:E.");
assertEquals(2, searcher.search(ssrquery.toQuery(), 10).totalHits);
// RegexpQuery
// All docs containing "E." (er) (0x)
srquery = new RegexpQuery(new Term("text", "s:e."));
assertEquals(0, searcher.search(srquery, 10).totalHits);
ssrquery = new SpanRegexQueryWrapper("text", "s:e.");
assertEquals(0, searcher.search(ssrquery.toQuery(), 10).totalHits);
// RegexpQuery
// All docs containing "E."/i ([Ee]r) (2x)
srquery = new RegexpQuery(new Term("text", "i:e."));
assertEquals(2, searcher.search(srquery, 10).totalHits);
ssrquery = new SpanRegexQueryWrapper("text", "s:e.", true);
assertEquals("SpanMultiTermQueryWrapper(text:/i:e./)", ssrquery.toQuery().toString());
assertEquals(2, searcher.search(ssrquery.toQuery(), 10).totalHits);
// All docs containing "ng"/x (Angst) (2x)
srquery = new RegexpQuery(new Term("text", "s:.*ng.*"));
assertEquals(2, searcher.search(srquery, 10).totalHits);
// Check http://comments.gmane.org/gmane.comp.jakarta.lucene.user/52283
// for Carstens question on wildcards
// Wildcardquery
// All docs containing ".{4}en" (liefen und Hunden)
WildcardQuery swquery = new WildcardQuery(new Term("text", "s:*ng*"));
assertEquals("text:s:*ng*", swquery.toString());
assertEquals(2, searcher.search(swquery, 10).totalHits);
// [base=angst]
SpanTermQuery stq = new SpanTermQuery(new Term("text", "l:angst"));
assertEquals(2, searcher.search(srquery, 10).totalHits);
// vor Angst
// [orth=vor][orth=Angst]
SpanNearQuery snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "s:vor")), new SpanTermQuery(new Term("text", "s:Angst")) }, 1, true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
// Spannearquery [p:VVFIN][]{,5}[m:nom:sg:fem]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:VVFIN")), new SpanSegmentQueryWrapper("text", "m:c:nom", "m:n:sg", "m:g:fem").toQuery() }, // slop
5, // inOrder
true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
// Spannearquery [p:VVFIN][m:acc:sg:masc]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:VVFIN")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "m:c:acc")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "m:n:sg")), new SpanTermQuery(new Term("text", "m:g:masc")) }, -1, false) }, // slop
-1, // inOrder
false) // new SpanTermQuery(new Term("text", "m:-acc:--sg:masc"))
}, // slop
0, // inOrder
true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
// Spannearquery [p:VVFIN|m:3:sg:past:ind]
// Exact match!
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:VVFIN")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "m:p:3")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "m:n:sg")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "m:t:past")), new SpanTermQuery(new Term("text", "m:m:ind")) }, -1, false) }, -1, false) }, -1, false) }, // slop
-1, // inOrder
false);
assertEquals(2, searcher.search(snquery, 10).totalHits);
// To make sure, this is not equal:
// Spannearquery [p:VVFIN & m:3:sg:past:ind]
// Exact match!
// Maybe it IS equal
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:VVFIN")), new SpanTermQuery(new Term("text", "m:p:3")), new SpanTermQuery(new Term("text", "m:n:sg")), new SpanTermQuery(new Term("text", "m:t:past")), new SpanTermQuery(new Term("text", "m:m:ind")) }, // slop
-1, // inOrder
false);
assertNotEquals(2, searcher.search(snquery, 10).totalHits);
// assertEquals(2, searcher.search(snquery, 10).totalHits);
// Spannearquery [p:VVFIN & m:3:sg & past:ind]
SpanSegmentQueryWrapper sniquery = new SpanSegmentQueryWrapper("text", "p:VVFIN", "m:p:3", "m:n:sg", "m:t:past", "m:m:ind");
assertEquals(2, searcher.search(sniquery.toQuery(), 10).totalHits);
// Todo:
/*
sniquery = new SpanSegmentQuery(
"text",
"p:VVFIN",
"m:p:3",
"m:n:sg",
"m:t:past",
"m:m:ind"
);
*/
// Spannearquery [p:VVFIN][]{,5}[m:nom:sg:fem]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:VVFIN")), new SpanSegmentQueryWrapper("text", "m:c:nom", "m:n:sg", "m:g:fem").toQuery() }, // slop
5, // inOrder
true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
sniquery = new SpanSegmentQueryWrapper("text", "p:VVFIN", "m:p:3", "m:t:past", "m:m:ind", "m:n:sg");
assertEquals(2, searcher.search(sniquery.toQuery(), 10).totalHits);
// [p = VVFIN & m:p = 3 & m:t = past & m:n != pl] or
// [p = VVFIN & m:p = 3 & m:t = past & !m:n = pl]
// TODO: Problem: What should happen in case the category does not exist?
// pssible solution: & ( m:n != pl & exists(m:n))
sniquery = new SpanSegmentQueryWrapper("text", "p:VVFIN", "m:p:3", "m:t:past");
SpanQuery snqquery = new SpanNotQuery(sniquery.toQuery(), new SpanTermQuery(new Term("text", "m:n:pl")));
assertEquals(2, searcher.search(snqquery, 10).totalHits);
// [p = NN & (m:c: = dat | m:c = acc)]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:NN")), new SpanOrQuery(new SpanTermQuery(new Term("text", "m:c:nom")), new SpanTermQuery(new Term("text", "m:c:acc"))) }, -1, false);
assertEquals(2, searcher.search(snqquery, 10).totalHits);
// [p = NN & !(m:c: = nom | m:c = acc)]
snqquery = new SpanNotQuery(new SpanTermQuery(new Term("text", "p:NN")), new SpanOrQuery(new SpanTermQuery(new Term("text", "m:c:nom")), new SpanTermQuery(new Term("text", "m:c:acc"))));
assertEquals(1, searcher.search(snqquery, 10).totalHits);
// [p = NN & !(m:c = nom)]
snqquery = new SpanNotQuery(new SpanTermQuery(new Term("text", "p:NN")), new SpanTermQuery(new Term("text", "m:c:nom")));
assertEquals(3, searcher.search(snqquery, 10).totalHits);
// [p=NN & !(m:c = acc)]
snqquery = new SpanNotQuery(new SpanTermQuery(new Term("text", "p:NN")), new SpanTermQuery(new Term("text", "m:c:acc")));
assertEquals(2, searcher.search(snqquery, 10).totalHits);
// [p=PPER][][p=ART]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "p:PPER")), new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "T")), new SpanTermQuery(new Term("text", "p:ART")) }, 0, true) }, 0, true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
// Todo:
// [orth=się][]{2,4}[base=bać]
// [orth=się][orth!="[.!?,:]"]{,5}[base=bać]|[base=bać][base="on|ja|ty|my|wy"]?[orth=się]
// [pos=subst & orth="a.*"]{2}
// [tag=subst:sg:nom:n]
// [case==acc & case==gen] ??
// [case~acc & case~gen]
// [case~~acc]
// [base=bać][orth!=się]+[orth=się] within s
// [][][p:VAFIN] within s
// [][p:VAFIN] within s
// [][][p:VAFIN]
snquery = new SpanNearQuery(new SpanQuery[] { new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "T")), new SpanTermQuery(new Term("text", "T")) }, 0, true), new SpanTermQuery(new Term("text", "p:VAFIN")) }, 0, true);
assertEquals(1, searcher.search(snquery, 10).totalHits);
/*
http://stackoverflow.com/questions/1311199/finding-the-position-of-search-hits-from-lucene
*/
StringBuilder payloadString = new StringBuilder();
Map<Term, TermContext> termContexts = new HashMap<>();
for (LeafReaderContext atomic : reader.leaves()) {
Bits bitset = atomic.reader().getLiveDocs();
// Spans spans = NearSpansOrdered();
Spans spans = snquery.getSpans(atomic, bitset, termContexts);
while (spans.next()) {
int docid = atomic.docBase + spans.doc();
if (spans.isPayloadAvailable()) {
for (byte[] payload : spans.getPayload()) {
/* retrieve payload for current matching span */
payloadString.append(new String(payload));
payloadString.append(" | ");
}
;
}
;
}
;
}
;
// assertEquals(33, payloadString.length());
assertEquals(0, payloadString.length());
// [][][p:VAFIN]
// without collecting payloads
snquery = new SpanNearQuery(new SpanQuery[] { new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term("text", "T")), new SpanTermQuery(new Term("text", "T")) }, 0, true, false), new SpanTermQuery(new Term("text", "p:VAFIN")) }, 0, true, false);
assertEquals(1, searcher.search(snquery, 10).totalHits);
payloadString = new StringBuilder();
termContexts = new HashMap<>();
for (LeafReaderContext atomic : reader.leaves()) {
Bits bitset = atomic.reader().getLiveDocs();
// Spans spans = NearSpansOrdered();
Spans spans = snquery.getSpans(atomic, bitset, termContexts);
while (spans.next()) {
int docid = atomic.docBase + spans.doc();
for (byte[] payload : spans.getPayload()) {
/* retrieve payload for current matching span */
payloadString.append(new String(payload));
payloadString.append(" | ");
}
;
}
;
}
;
assertEquals(0, payloadString.length());
// [][][p:VAFIN] in s
// ([e:s:<][]*[T] | [T & e:s:<]) [T] ([p:VAFIN & e:s:>] | [T][]*[e:s:>]
/*
SpanSegmentWithinQuery ssequery = new SpanSegmentWithinQuery(
"text","s", new SpanSegmentSequenceQuery("text", "T", "T", "p:VAFIN")
);
assertEquals(0, searcher.search(ssequery.toQuery(), 10).totalHits);
payloadString = new StringBuilder();
termContexts = new HashMap<>();
for (LeafReaderContext atomic : reader.leaves()) {
Bits bitset = atomic.reader().getLiveDocs();
// Spans spans = NearSpansOrdered();
Spans spans = ssequery.toQuery().getSpans(atomic, bitset, termContexts);
while (spans.next()) {
int docid = atomic.docBase + spans.doc();
for (byte[] payload : spans.getPayload()) {
/// retrieve payload for current matching span
payloadString.append(new String(payload));
payloadString.append(" | ");
};
};
};
assertEquals(0, payloadString.length(), 1);
ssequery = new SpanSegmentWithinQuery(
"text","s", new SpanSegmentSequenceQuery("text", "T", "p:VAFIN")
);
assertEquals("for " + ssequery.toQuery(),
1, searcher.search(ssequery.toQuery(), 10).totalHits);
payloadString = new StringBuilder();
termContexts = new HashMap<>();
for (LeafReaderContext atomic : reader.leaves()) {
Bits bitset = atomic.reader().getLiveDocs();
// Spans spans = NearSpansOrdered();
Spans spans = ssequery.toQuery().getSpans(atomic, bitset, termContexts);
while (spans.next()) {
int docid = atomic.docBase + spans.doc();
for (byte[] payload : spans.getPayload()) {
// retrieve payload for current matching span
payloadString.append(new String(payload));
payloadString.append(" | ");
};
fail("Doc: " + docid + " with " + spans.start() + "-" + spans.end() + " || " + payloadString.toString());
};
};
assertEquals(20, payloadString.length());
*/
// --------------------______>
// Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), ssequery.toQuery());
/*
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<String>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
while (spans.next()) {
Collection<byte[]> payloads = spans.getPayload();
for (final byte [] payload : payloads) {
payloadSet.add(new String(payload, "UTF-8"));
}
}
}
*/
/*
Alternativ:
IndexReader reader = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(reader);
PayloadSpanUtil psu = new PayloadSpanUtil(searcher.getTopReaderContext());
Collection<byte[]> payloads = psu.getPayloadsForQuery(new TermQuery(new Term(PayloadHelper.FIELD, "rr")));
if(VERBOSE)
System.out.println("Num payloads:" + payloads.size());
for (final byte [] bytes : payloads) {
if(VERBOSE)
System.out.println(new String(bytes, "UTF-8"));
}
*/
/* new: */
// PayloadHelper helper = new PayloadHelper();
// Map<Term, TermContext> termContexts = new HashMap<>();
// Spans spans;
// spans = snquery.getSpans(searcher.getIndexReader());
// searcher = helper.setUp(similarity, 1000);
/*
IndexReader reader = search.getReader(querycontainer.getFoundry());
Spans luceneSpans;
Bits bitset = atomic.reader().getLiveDocs();
for (byte[] payload : luceneSpans.getPayload())
/* Iterate over all matching documents */
/*
while (luceneSpans.next() && total < config.getMaxhits()) {
Span matchSpan;
StringBuilder payloadString = new StringBuilder();
int docid = atomic.docBase + luceneSpans.doc();
String docname = search.retrieveDocname(docid,
querycontainer.getFoundry());
total++;
for (byte[] payload : luceneSpans.getPayload())
*/
/* retrieve payload for current matching span */
// payloadString.append(new String(payload));
/* create span containing result */
/*
matchSpan = new Span(docname);
matchSpan.setIndexdocid(docid);
matchSpan.setLayer(querycontainer.getLayer());
matchSpan.storePayloads(payloadString.toString());
matchSpans.add(matchSpan);
*/
/*
* topdocs = searcher.search(new ConstantScoreQuery(corpusQ add
* position to list of positions to be considered for later
* searches
*/
/*
validValues.put(docname,
matchSpan.getPayload(config.getPrefix()));
}
*/
// Todo: API made by add() typisiert für queries, strings
// SpanPayloadCheckQuery for sentences!
/* Support regular expression in SpanSegmentQuery */
// new Regexp();
// new Term();
/*
Vielleicht: spanSegmentQuery(new Term(), new Wildcard(), new Regex());
*/
// And Not ->
// SpanTermDiffQuery
/*
SpanNearQuery poquery = new SpanNearQuery(
);
*/
reader.close();
}
use of de.ids_mannheim.korap.query.wrap.SpanSegmentQueryWrapper in project Krill by KorAP.
the class KrillQuery method _segFromJson.
// Deserialize koral:token
private SpanQueryWrapper _segFromJson(JsonNode json) throws QueryException {
if (!json.has("@type"))
throw new QueryException(701, "JSON-LD group has no @type attribute");
String type = json.get("@type").asText();
if (DEBUG)
log.trace("Wrap new token definition by {}", type);
// Branch on type
switch(type) {
case "koral:term":
// case "match:eq":
return this._termFromJson(json);
case "koral:termGroup":
if (!json.has("operands"))
throw new QueryException(742, "Term group needs operand list");
// Get operands
JsonNode operands = json.get("operands");
SpanSegmentQueryWrapper ssegqw = this.builder().seg();
if (!json.has("relation"))
throw new QueryException(743, "Term group expects a relation");
switch(json.get("relation").asText()) {
case "relation:and":
for (JsonNode operand : operands) {
SpanQueryWrapper part = this._segFromJson(operand);
if (part instanceof SpanAlterQueryWrapper) {
ssegqw.with((SpanAlterQueryWrapper) part);
} else if (part instanceof SpanRegexQueryWrapper) {
ssegqw.with((SpanRegexQueryWrapper) part);
} else if (part instanceof SpanSegmentQueryWrapper) {
ssegqw.with((SpanSegmentQueryWrapper) part);
} else {
throw new QueryException(744, "Operand not supported in term group");
}
;
}
;
return ssegqw;
case "relation:or":
SpanAlterQueryWrapper ssaq = new SpanAlterQueryWrapper(this.field);
for (JsonNode operand : operands) {
ssaq.or(this._segFromJson(operand));
}
;
return ssaq;
}
;
}
;
throw new QueryException(745, "Token type is not supported");
}
use of de.ids_mannheim.korap.query.wrap.SpanSegmentQueryWrapper in project Krill by KorAP.
the class TestSpanSegmentQuery method spanSegmentCloneQuery.
@Test
public void spanSegmentCloneQuery() throws QueryException {
SpanSegmentQueryWrapper ssquery = new SpanSegmentQueryWrapper("field", "a", "b");
assertEquals("spanSegment(field:a, field:b)", ssquery.toQuery().toString());
SpanSegmentQueryWrapper ssquery2 = new SpanSegmentQueryWrapper("field", ssquery);
assertEquals(ssquery.toQuery().toString(), ssquery2.toQuery().toString());
SpanSegmentQueryWrapper ssquery3 = ssquery2.clone();
assertEquals(ssquery.toQuery().toString(), ssquery3.toQuery().toString());
assertEquals(ssquery2.toQuery().toString(), ssquery3.toQuery().toString());
}
use of de.ids_mannheim.korap.query.wrap.SpanSegmentQueryWrapper in project Krill by KorAP.
the class TestSpanSegmentQuery method spanSegmentQueryExclusive.
@Test
public void spanSegmentQueryExclusive() throws QueryException {
SpanSegmentQueryWrapper ssquery = new SpanSegmentQueryWrapper("field", "a");
assertEquals("field:a", ssquery.toQuery().toString());
ssquery = new SpanSegmentQueryWrapper("field", "a", "b");
assertEquals("spanSegment(field:a, field:b)", ssquery.toQuery().toString());
ssquery.without("c");
assertEquals("spanNot(spanSegment(field:a, field:b), field:c, 0, 0)", ssquery.toQuery().toString());
ssquery.without("d");
assertEquals("spanNot(spanSegment(field:a, field:b), spanOr([field:c, field:d]), 0, 0)", ssquery.toQuery().toString());
}
use of de.ids_mannheim.korap.query.wrap.SpanSegmentQueryWrapper in project Krill by KorAP.
the class TestSpanSegmentQuery method spanSegmentRegexQuery.
@Test
public void spanSegmentRegexQuery() throws QueryException {
SpanSegmentQueryWrapper ssquery = new SpanSegmentQueryWrapper("field");
assertNull(ssquery.toQuery());
ssquery.with("a");
assertEquals("field:a", ssquery.toQuery().toString());
ssquery.with(new SpanRegexQueryWrapper("field", "a.*b"));
assertEquals("spanSegment(field:a, SpanMultiTermQueryWrapper(field:/a.*b/))", ssquery.toQuery().toString());
ssquery.with("c");
assertEquals("spanSegment(spanSegment(field:a, SpanMultiTermQueryWrapper(field:/a.*b/)), field:c)", ssquery.toQuery().toString());
ssquery.with("d").with("e");
assertEquals("spanSegment(spanSegment(spanSegment(spanSegment(field:a, SpanMultiTermQueryWrapper(field:/a.*b/)), field:c), field:d), field:e)", ssquery.toQuery().toString());
ssquery.without(new SpanRegexQueryWrapper("field", "x.?y"));
assertEquals("spanNot(spanSegment(spanSegment(spanSegment(spanSegment(field:a, SpanMultiTermQueryWrapper(field:/a.*b/)), field:c), field:d), field:e), SpanMultiTermQueryWrapper(field:/x.?y/), 0, 0)", ssquery.toQuery().toString());
ssquery.without(new SpanRegexQueryWrapper("field", "z{5,9}"));
assertEquals("spanNot(spanSegment(spanSegment(spanSegment(spanSegment(field:a, SpanMultiTermQueryWrapper(field:/a.*b/)), field:c), field:d), field:e), spanOr([SpanMultiTermQueryWrapper(field:/x.?y/), SpanMultiTermQueryWrapper(field:/z{5,9}/)]), 0, 0)", ssquery.toQuery().toString());
}
Aggregations