Examples with FieldMemoryStats org.opensearch.common.FieldMemoryStats used on opensource projects

Example 1 with FieldMemoryStats

use of org.opensearch.common.FieldMemoryStats in project OpenSearch by opensearch-project.

the class CompletionsStatsTests method testSerialize.

public void testSerialize() throws IOException {
    FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
    CompletionStats stats = new CompletionStats(randomNonNegativeLong(), map);
    BytesStreamOutput out = new BytesStreamOutput();
    stats.writeTo(out);
    StreamInput input = out.bytes().streamInput();
    CompletionStats read = new CompletionStats(input);
    assertEquals(-1, input.read());
    assertEquals(stats.getSizeInBytes(), read.getSizeInBytes());
    assertEquals(stats.getFields(), read.getFields());
}

Example 2 with FieldMemoryStats

use of org.opensearch.common.FieldMemoryStats in project OpenSearch by opensearch-project.

the class CompletionSuggestSearchIT method testThatStatsAreWorking.

public void testThatStatsAreWorking() throws Exception {
    String otherField = "testOtherField";
    client().admin().indices().prepareCreate(INDEX).setSettings(Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2)).get();
    ensureGreen();
    AcknowledgedResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setSource(jsonBuilder().startObject().startObject(MapperService.SINGLE_MAPPING_NAME).startObject("properties").startObject(FIELD).field("type", "completion").field("analyzer", "simple").endObject().startObject(otherField).field("type", "completion").field("analyzer", "simple").endObject().endObject().endObject().endObject()).get();
    assertThat(putMappingResponse.isAcknowledged(), is(true));
    // Index two entities
    client().prepareIndex(INDEX).setId("1").setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get();
    client().prepareIndex(INDEX).setId("2").setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get();
    refresh();
    ensureGreen();
    // load the fst index into ram
    client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f"))).get();
    client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f"))).get();
    // Get all stats
    IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get();
    CompletionStats completionStats = indicesStatsResponse.getIndex(INDEX).getPrimaries().completion;
    assertThat(completionStats, notNullValue());
    long totalSizeInBytes = completionStats.getSizeInBytes();
    assertThat(totalSizeInBytes, is(greaterThan(0L)));
    IndicesStatsResponse singleFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields(FIELD).get();
    long singleFieldSizeInBytes = singleFieldStats.getIndex(INDEX).getPrimaries().completion.getFields().get(FIELD);
    IndicesStatsResponse otherFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields(otherField).get();
    long otherFieldSizeInBytes = otherFieldStats.getIndex(INDEX).getPrimaries().completion.getFields().get(otherField);
    assertThat(singleFieldSizeInBytes + otherFieldSizeInBytes, is(totalSizeInBytes));
    // regexes
    IndicesStatsResponse regexFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields("*").get();
    FieldMemoryStats fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields();
    long regexSizeInBytes = fields.get(FIELD) + fields.get(otherField);
    assertThat(regexSizeInBytes, is(totalSizeInBytes));
}

Example 3 with FieldMemoryStats

use of org.opensearch.common.FieldMemoryStats in project OpenSearch by opensearch-project.

the class CompletionStatsCache method get.

CompletionStats get(String... fieldNamePatterns) {
    final PlainActionFuture<CompletionStats> newFuture = new PlainActionFuture<>();
    // final PlainActionFuture<CompletionStats> oldFuture = completionStatsFutureRef.compareAndExchange(null, newFuture);
    // except JDK8 doesn't have compareAndExchange so we emulate it:
    final PlainActionFuture<CompletionStats> oldFuture;
    synchronized (completionStatsFutureMutex) {
        if (completionStatsFuture == null) {
            completionStatsFuture = newFuture;
            oldFuture = null;
        } else {
            oldFuture = completionStatsFuture;
        }
    }
    if (oldFuture != null) {
        // we lost the race, someone else is already computing stats, so we wait for that to finish
        return filterCompletionStatsByFieldName(fieldNamePatterns, oldFuture.actionGet());
    }
    // we won the race, nobody else is already computing stats, so it's up to us
    ActionListener.completeWith(newFuture, () -> {
        long sizeInBytes = 0;
        final ObjectLongHashMap<String> completionFields = new ObjectLongHashMap<>();
        try (Engine.Searcher currentSearcher = searcherSupplier.get()) {
            for (LeafReaderContext atomicReaderContext : currentSearcher.getIndexReader().leaves()) {
                LeafReader atomicReader = atomicReaderContext.reader();
                for (FieldInfo info : atomicReader.getFieldInfos()) {
                    Terms terms = atomicReader.terms(info.name);
                    if (terms instanceof CompletionTerms) {
                        // TODO: currently we load up the suggester for reporting its size
                        final long fstSize = ((CompletionTerms) terms).suggester().ramBytesUsed();
                        completionFields.addTo(info.name, fstSize);
                        sizeInBytes += fstSize;
                    }
                }
            }
        }
        return new CompletionStats(sizeInBytes, new FieldMemoryStats(completionFields));
    });
    boolean success = false;
    final CompletionStats completionStats;
    try {
        completionStats = newFuture.actionGet();
        success = true;
    } finally {
        if (success == false) {
            // completionStatsFutureRef.compareAndSet(newFuture, null); except we're not using AtomicReference in JDK8
            synchronized (completionStatsFutureMutex) {
                if (completionStatsFuture == newFuture) {
                    completionStatsFuture = null;
                }
            }
        }
    }
    return filterCompletionStatsByFieldName(fieldNamePatterns, completionStats);
}

Example 4 with FieldMemoryStats

use of org.opensearch.common.FieldMemoryStats in project OpenSearch by opensearch-project.

the class CompletionStatsCache method filterCompletionStatsByFieldName.

private static CompletionStats filterCompletionStatsByFieldName(String[] fieldNamePatterns, CompletionStats fullCompletionStats) {
    final FieldMemoryStats fieldMemoryStats;
    if (CollectionUtils.isEmpty(fieldNamePatterns) == false) {
        final ObjectLongHashMap<String> completionFields = new ObjectLongHashMap<>(fieldNamePatterns.length);
        for (ObjectLongCursor<String> fieldCursor : fullCompletionStats.getFields()) {
            if (Regex.simpleMatch(fieldNamePatterns, fieldCursor.key)) {
                completionFields.addTo(fieldCursor.key, fieldCursor.value);
            }
        }
        fieldMemoryStats = new FieldMemoryStats(completionFields);
    } else {
        fieldMemoryStats = null;
    }
    return new CompletionStats(fullCompletionStats.getSizeInBytes(), fieldMemoryStats);
}

Example 5 with FieldMemoryStats

use of org.opensearch.common.FieldMemoryStats in project OpenSearch by opensearch-project.

the class FieldDataStatsTests method testSerialize.

public void testSerialize() throws IOException {
    FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
    FieldDataStats stats = new FieldDataStats(randomNonNegativeLong(), randomNonNegativeLong(), map);
    BytesStreamOutput out = new BytesStreamOutput();
    stats.writeTo(out);
    StreamInput input = out.bytes().streamInput();
    FieldDataStats read = new FieldDataStats(input);
    assertEquals(-1, input.read());
    assertEquals(stats.getEvictions(), read.getEvictions());
    assertEquals(stats.getMemorySize(), read.getMemorySize());
    assertEquals(stats.getFields(), read.getFields());
}