Examples with HalfFloatPoint org.apache.lucene.sandbox.document.HalfFloatPoint used on opensource projects

Example 1 with HalfFloatPoint

use of org.apache.lucene.sandbox.document.HalfFloatPoint in project OpenSearch by opensearch-project.

the class PercolatorFieldMapperTests method testExtractTermsAndRanges_numberFields.

public void testExtractTermsAndRanges_numberFields() throws Exception {
    addQueryFieldMappings();
    MemoryIndex memoryIndex = new MemoryIndex(false);
    memoryIndex.addField(new IntPoint("number_field1", 10), new WhitespaceAnalyzer());
    memoryIndex.addField(new LongPoint("number_field2", 20L), new WhitespaceAnalyzer());
    memoryIndex.addField(new LongPoint("number_field3", 30L), new WhitespaceAnalyzer());
    memoryIndex.addField(new HalfFloatPoint("number_field4", 30f), new WhitespaceAnalyzer());
    memoryIndex.addField(new FloatPoint("number_field5", 40f), new WhitespaceAnalyzer());
    memoryIndex.addField(new DoublePoint("number_field6", 50f), new WhitespaceAnalyzer());
    memoryIndex.addField(new InetAddressPoint("number_field7", InetAddresses.forString("192.168.1.12")), new WhitespaceAnalyzer());
    memoryIndex.addField(new InetAddressPoint("number_field7", InetAddresses.forString("192.168.1.20")), new WhitespaceAnalyzer());
    memoryIndex.addField(new InetAddressPoint("number_field7", InetAddresses.forString("192.168.1.24")), new WhitespaceAnalyzer());
    IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
    Tuple<List<BytesRef>, Map<String, List<byte[]>>> t = fieldType.extractTermsAndRanges(indexReader);
    assertEquals(0, t.v1().size());
    Map<String, List<byte[]>> rangesMap = t.v2();
    assertEquals(7, rangesMap.size());
    List<byte[]> range = rangesMap.get("number_field1");
    assertNotNull(range);
    assertEquals(10, IntPoint.decodeDimension(range.get(0), 0));
    assertEquals(10, IntPoint.decodeDimension(range.get(1), 0));
    range = rangesMap.get("number_field2");
    assertNotNull(range);
    assertEquals(20L, LongPoint.decodeDimension(range.get(0), 0));
    assertEquals(20L, LongPoint.decodeDimension(range.get(1), 0));
    range = rangesMap.get("number_field3");
    assertNotNull(range);
    assertEquals(30L, LongPoint.decodeDimension(range.get(0), 0));
    assertEquals(30L, LongPoint.decodeDimension(range.get(1), 0));
    range = rangesMap.get("number_field4");
    assertNotNull(range);
    assertEquals(30F, HalfFloatPoint.decodeDimension(range.get(0), 0), 0F);
    assertEquals(30F, HalfFloatPoint.decodeDimension(range.get(1), 0), 0F);
    range = rangesMap.get("number_field5");
    assertNotNull(range);
    assertEquals(40F, FloatPoint.decodeDimension(range.get(0), 0), 0F);
    assertEquals(40F, FloatPoint.decodeDimension(range.get(1), 0), 0F);
    range = rangesMap.get("number_field6");
    assertNotNull(range);
    assertEquals(50D, DoublePoint.decodeDimension(range.get(0), 0), 0D);
    assertEquals(50D, DoublePoint.decodeDimension(range.get(1), 0), 0D);
    range = rangesMap.get("number_field7");
    assertNotNull(range);
    assertEquals(InetAddresses.forString("192.168.1.12"), InetAddressPoint.decode(range.get(0)));
    assertEquals(InetAddresses.forString("192.168.1.24"), InetAddressPoint.decode(range.get(1)));
}

Example 2 with HalfFloatPoint

use of org.apache.lucene.sandbox.document.HalfFloatPoint in project OpenSearch by opensearch-project.

the class CandidateQueryTests method testDuelRangeQueries.

public void testDuelRangeQueries() throws Exception {
    List<ParseContext.Document> documents = new ArrayList<>();
    int lowerInt = randomIntBetween(0, 256);
    int upperInt = lowerInt + randomIntBetween(0, 32);
    addQuery(IntPoint.newRangeQuery("int_field", lowerInt, upperInt), documents);
    long lowerLong = randomIntBetween(0, 256);
    long upperLong = lowerLong + randomIntBetween(0, 32);
    addQuery(LongPoint.newRangeQuery("long_field", lowerLong, upperLong), documents);
    float lowerHalfFloat = randomIntBetween(0, 256);
    float upperHalfFloat = lowerHalfFloat + randomIntBetween(0, 32);
    addQuery(HalfFloatPoint.newRangeQuery("half_float_field", lowerHalfFloat, upperHalfFloat), documents);
    float lowerFloat = randomIntBetween(0, 256);
    float upperFloat = lowerFloat + randomIntBetween(0, 32);
    addQuery(FloatPoint.newRangeQuery("float_field", lowerFloat, upperFloat), documents);
    double lowerDouble = randomDoubleBetween(0, 256, true);
    double upperDouble = lowerDouble + randomDoubleBetween(0, 32, true);
    addQuery(DoublePoint.newRangeQuery("double_field", lowerDouble, upperDouble), documents);
    int lowerIpPart = randomIntBetween(0, 255);
    int upperIpPart = randomIntBetween(lowerIpPart, 255);
    addQuery(InetAddressPoint.newRangeQuery("ip_field", forString("192.168.1." + lowerIpPart), forString("192.168.1." + upperIpPart)), documents);
    indexWriter.addDocuments(documents);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    // Disable query cache, because ControlQuery cannot be cached...
    shardSearcher.setQueryCache(null);
    int randomInt = randomIntBetween(lowerInt, upperInt);
    Iterable<? extends IndexableField> doc = Collections.singleton(new IntPoint("int_field", randomInt));
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    TopDocs result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(0));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new IntPoint("int_field", randomInt()));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    long randomLong = randomIntBetween((int) lowerLong, (int) upperLong);
    doc = Collections.singleton(new LongPoint("long_field", randomLong));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(1));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new LongPoint("long_field", randomLong()));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    float randomHalfFloat = randomIntBetween((int) lowerHalfFloat, (int) upperHalfFloat);
    doc = Collections.singleton(new HalfFloatPoint("half_float_field", randomHalfFloat));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(2));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new HalfFloatPoint("half_float_field", randomFloat()));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    float randomFloat = randomIntBetween((int) lowerFloat, (int) upperFloat);
    doc = Collections.singleton(new FloatPoint("float_field", randomFloat));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(3));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new FloatPoint("float_field", randomFloat()));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    double randomDouble = randomDoubleBetween(lowerDouble, upperDouble, true);
    doc = Collections.singleton(new DoublePoint("double_field", randomDouble));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(4));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new DoublePoint("double_field", randomFloat()));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new InetAddressPoint("ip_field", forString("192.168.1." + randomIntBetween(lowerIpPart, upperIpPart))));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    result = executeQuery(queryStore, memoryIndex, shardSearcher);
    assertThat(result.scoreDocs.length, equalTo(1));
    assertThat(result.scoreDocs[0].doc, equalTo(5));
    duelRun(queryStore, memoryIndex, shardSearcher);
    doc = Collections.singleton(new InetAddressPoint("ip_field", forString("192.168.1." + randomIntBetween(0, 255))));
    memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
}

Example 3 with HalfFloatPoint

use of org.apache.lucene.sandbox.document.HalfFloatPoint in project OpenSearch by opensearch-project.

the class CandidateQueryTests method testRangeQueries.

public void testRangeQueries() throws Exception {
    List<ParseContext.Document> docs = new ArrayList<>();
    addQuery(IntPoint.newRangeQuery("int_field", 0, 5), docs);
    addQuery(LongPoint.newRangeQuery("long_field", 5L, 10L), docs);
    addQuery(HalfFloatPoint.newRangeQuery("half_float_field", 10, 15), docs);
    addQuery(FloatPoint.newRangeQuery("float_field", 15, 20), docs);
    addQuery(DoublePoint.newRangeQuery("double_field", 20, 25), docs);
    addQuery(InetAddressPoint.newRangeQuery("ip_field", forString("192.168.0.1"), forString("192.168.0.10")), docs);
    indexWriter.addDocuments(docs);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    shardSearcher.setQueryCache(null);
    Version v = VersionUtils.randomIndexCompatibleVersion(random());
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new IntPoint("int_field", 3)), new WhitespaceAnalyzer());
    IndexSearcher percolateSearcher = memoryIndex.createSearcher();
    Query query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    TopDocs topDocs = shardSearcher.search(query, 1);
    assertEquals(1L, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(0, topDocs.scoreDocs[0].doc);
    memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new LongPoint("long_field", 7L)), new WhitespaceAnalyzer());
    percolateSearcher = memoryIndex.createSearcher();
    query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    topDocs = shardSearcher.search(query, 1);
    assertEquals(1L, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(1, topDocs.scoreDocs[0].doc);
    memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new HalfFloatPoint("half_float_field", 12)), new WhitespaceAnalyzer());
    percolateSearcher = memoryIndex.createSearcher();
    query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    topDocs = shardSearcher.search(query, 1);
    assertEquals(1L, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(2, topDocs.scoreDocs[0].doc);
    memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new FloatPoint("float_field", 17)), new WhitespaceAnalyzer());
    percolateSearcher = memoryIndex.createSearcher();
    query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    topDocs = shardSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(3, topDocs.scoreDocs[0].doc);
    memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new DoublePoint("double_field", 21)), new WhitespaceAnalyzer());
    percolateSearcher = memoryIndex.createSearcher();
    query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    topDocs = shardSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(4, topDocs.scoreDocs[0].doc);
    memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new InetAddressPoint("ip_field", forString("192.168.0.4"))), new WhitespaceAnalyzer());
    percolateSearcher = memoryIndex.createSearcher();
    query = fieldType.percolateQuery("_name", queryStore, Collections.singletonList(new BytesArray("{}")), percolateSearcher, false, v);
    topDocs = shardSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(5, topDocs.scoreDocs[0].doc);
}

Example 4 with HalfFloatPoint

use of org.apache.lucene.sandbox.document.HalfFloatPoint in project OpenSearch by opensearch-project.

the class FieldSortBuilderTests method testGetMaxNumericSortValue.

public void testGetMaxNumericSortValue() throws IOException {
    QueryShardContext context = createMockShardContext();
    for (NumberFieldMapper.NumberType numberType : NumberFieldMapper.NumberType.values()) {
        String fieldName = "custom-" + numberType.numericType();
        assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName)));
        assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni")));
        try (Directory dir = newDirectory()) {
            int numDocs = randomIntBetween(10, 30);
            final Comparable[] values = new Comparable[numDocs];
            try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
                for (int i = 0; i < numDocs; i++) {
                    Document doc = new Document();
                    switch(numberType) {
                        case LONG:
                            long v1 = randomLong();
                            values[i] = v1;
                            doc.add(new LongPoint(fieldName, v1));
                            break;
                        case INTEGER:
                            int v2 = randomInt();
                            values[i] = (long) v2;
                            doc.add(new IntPoint(fieldName, v2));
                            break;
                        case DOUBLE:
                            double v3 = randomDouble();
                            values[i] = v3;
                            doc.add(new DoublePoint(fieldName, v3));
                            break;
                        case FLOAT:
                            float v4 = randomFloat();
                            values[i] = v4;
                            doc.add(new FloatPoint(fieldName, v4));
                            break;
                        case HALF_FLOAT:
                            float v5 = randomFloat();
                            values[i] = (double) v5;
                            doc.add(new HalfFloatPoint(fieldName, v5));
                            break;
                        case BYTE:
                            byte v6 = randomByte();
                            values[i] = (long) v6;
                            doc.add(new IntPoint(fieldName, v6));
                            break;
                        case SHORT:
                            short v7 = randomShort();
                            values[i] = (long) v7;
                            doc.add(new IntPoint(fieldName, v7));
                            break;
                        default:
                            throw new AssertionError("unknown type " + numberType);
                    }
                    writer.addDocument(doc);
                }
                Arrays.sort(values);
                try (DirectoryReader reader = writer.getReader()) {
                    QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader));
                    if (numberType == NumberFieldMapper.NumberType.HALF_FLOAT) {
                        assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni")));
                        assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)));
                    } else {
                        assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni")));
                        assertEquals(values[numDocs - 1], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax());
                        assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin());
                    }
                }
            }
        }
    }
}

Example 5 with HalfFloatPoint

use of org.apache.lucene.sandbox.document.HalfFloatPoint in project OpenSearch by opensearch-project.

the class NumberFieldTypeTests method testHalfFloatRange.

public void testHalfFloatRange() throws IOException {
    // make sure the accuracy loss of half floats only occurs at index time
    // this test checks that searching half floats yields the same results as
    // searching floats that are rounded to the closest half float
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    final int numDocs = 10000;
    for (int i = 0; i < numDocs; ++i) {
        Document doc = new Document();
        // Note: this test purposefully allows half-floats to be indexed over their dynamic range (65504), which
        // ends up being rounded to Infinity by halfFloatToSortableShort()
        float value = (randomFloat() * 2 - 1) * 70000;
        float rounded = HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value));
        doc.add(new HalfFloatPoint("half_float", value));
        doc.add(new FloatPoint("float", rounded));
        w.addDocument(doc);
    }
    final DirectoryReader reader = DirectoryReader.open(w);
    w.close();
    IndexSearcher searcher = newSearcher(reader);
    final int numQueries = 1000;
    for (int i = 0; i < numQueries; ++i) {
        float l = (randomFloat() * 2 - 1) * 65504;
        float u = (randomFloat() * 2 - 1) * 65504;
        boolean includeLower = randomBoolean();
        boolean includeUpper = randomBoolean();
        Query floatQ = NumberType.FLOAT.rangeQuery("float", l, u, includeLower, includeUpper, false, MOCK_QSC);
        Query halfFloatQ = NumberType.HALF_FLOAT.rangeQuery("half_float", l, u, includeLower, includeUpper, false, MOCK_QSC);
        assertEquals(searcher.count(floatQ), searcher.count(halfFloatQ));
    }
    IOUtils.close(reader, dir);
}