Examples with FieldStats org.elasticsearch.action.fieldstats.FieldStats used on opensource projects

Example 6 with FieldStats

use of org.elasticsearch.action.fieldstats.FieldStats in project elasticsearch by elastic.

the class FieldStatsTests method testMerge_notAvailable.

public void testMerge_notAvailable() {
    List<FieldStats> stats = new ArrayList<>();
    stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, true, 1L, 1L));
    stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, true, 1L, 1L));
    stats.add(new FieldStats.Long(1, 1L, 1L, 1L, true, false, 1L, 1L));
    FieldStats stat = new FieldStats.Long(1, -1L, -1L, -1L, false, true, 1L, 1L);
    for (FieldStats otherStat : stats) {
        stat.accumulate(otherStat);
    }
    assertEquals(stat.getMaxDoc(), 4L);
    assertEquals(stat.getDocCount(), -1L);
    assertEquals(stat.getSumDocFreq(), -1L);
    assertEquals(stat.getSumTotalTermFreq(), -1L);
    assertEquals(stat.isSearchable(), true);
    assertEquals(stat.isAggregatable(), true);
    assertEquals(stat.getDisplayType(), "integer");
    stats.add(new FieldStats.Long(1, -1L, -1L, -1L, false, true));
    stat = stats.remove(0);
    for (FieldStats otherStat : stats) {
        stat.accumulate(otherStat);
    }
    assertEquals(stat.getMaxDoc(), 4L);
    assertEquals(stat.getDocCount(), -1L);
    assertEquals(stat.getSumDocFreq(), -1L);
    assertEquals(stat.getSumTotalTermFreq(), -1L);
    assertEquals(stat.isSearchable(), true);
    assertEquals(stat.isAggregatable(), true);
    assertEquals(stat.getDisplayType(), "integer");
    assertNull(stat.getMaxValue());
    assertNull(stat.getMinValue());
}

Example 7 with FieldStats

use of org.elasticsearch.action.fieldstats.FieldStats in project elasticsearch by elastic.

the class MappedFieldType method stats.

/**
     * @return a {@link FieldStats} instance that maps to the type of this
     * field or {@code null} if the provided index has no stats about the
     * current field
     */
public FieldStats stats(IndexReader reader) throws IOException {
    int maxDoc = reader.maxDoc();
    FieldInfo fi = MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
    if (fi == null) {
        return null;
    }
    Terms terms = MultiFields.getTerms(reader, name());
    if (terms == null) {
        return new FieldStats.Text(maxDoc, 0, -1, -1, isSearchable(), isAggregatable());
    }
    FieldStats stats = new FieldStats.Text(maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), isSearchable(), isAggregatable(), terms.getMin(), terms.getMax());
    return stats;
}

Example 8 with FieldStats

use of org.elasticsearch.action.fieldstats.FieldStats in project elasticsearch by elastic.

the class InternalEngineTests method testConcurrentWritesAndCommits.

// this test writes documents to the engine while concurrently flushing/commit
// and ensuring that the commit points contain the correct sequence number data
public void testConcurrentWritesAndCommits() throws Exception {
    try (Store store = createStore();
        InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), new SnapshotDeletionPolicy(NoDeletionPolicy.INSTANCE), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
        final int numIndexingThreads = scaledRandomIntBetween(3, 6);
        final int numDocsPerThread = randomIntBetween(500, 1000);
        final CyclicBarrier barrier = new CyclicBarrier(numIndexingThreads + 1);
        final List<Thread> indexingThreads = new ArrayList<>();
        // create N indexing threads to index documents simultaneously
        for (int threadNum = 0; threadNum < numIndexingThreads; threadNum++) {
            final int threadIdx = threadNum;
            Thread indexingThread = new Thread(() -> {
                try {
                    // wait for all threads to start at the same time
                    barrier.await();
                    // index random number of docs
                    for (int i = 0; i < numDocsPerThread; i++) {
                        final String id = "thread" + threadIdx + "#" + i;
                        ParsedDocument doc = testParsedDocument(id, "test", null, testDocument(), B_1, null);
                        engine.index(indexForDoc(doc));
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
            indexingThreads.add(indexingThread);
        }
        // start the indexing threads
        for (Thread thread : indexingThreads) {
            thread.start();
        }
        // wait for indexing threads to all be ready to start
        barrier.await();
        // create random commit points
        boolean doneIndexing;
        do {
            doneIndexing = indexingThreads.stream().filter(Thread::isAlive).count() == 0;
        //engine.flush(); // flush and commit
        } while (doneIndexing == false);
        // now, verify all the commits have the correct docs according to the user commit data
        long prevLocalCheckpoint = SequenceNumbersService.NO_OPS_PERFORMED;
        long prevMaxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
        for (IndexCommit commit : DirectoryReader.listCommits(store.directory())) {
            Map<String, String> userData = commit.getUserData();
            long localCheckpoint = userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) ? Long.parseLong(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) : SequenceNumbersService.NO_OPS_PERFORMED;
            long maxSeqNo = userData.containsKey(SequenceNumbers.MAX_SEQ_NO) ? Long.parseLong(userData.get(SequenceNumbers.MAX_SEQ_NO)) : SequenceNumbersService.UNASSIGNED_SEQ_NO;
            // local checkpoint and max seq no shouldn't go backwards
            assertThat(localCheckpoint, greaterThanOrEqualTo(prevLocalCheckpoint));
            assertThat(maxSeqNo, greaterThanOrEqualTo(prevMaxSeqNo));
            try (IndexReader reader = DirectoryReader.open(commit)) {
                FieldStats stats = SeqNoFieldMapper.SeqNoDefaults.FIELD_TYPE.stats(reader);
                final long highestSeqNo;
                if (stats != null) {
                    highestSeqNo = (long) stats.getMaxValue();
                } else {
                    highestSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
                }
                // make sure localCheckpoint <= highest seq no found <= maxSeqNo
                assertThat(highestSeqNo, greaterThanOrEqualTo(localCheckpoint));
                assertThat(highestSeqNo, lessThanOrEqualTo(maxSeqNo));
                // make sure all sequence numbers up to and including the local checkpoint are in the index
                FixedBitSet seqNosBitSet = getSeqNosSet(reader, highestSeqNo);
                for (int i = 0; i <= localCheckpoint; i++) {
                    assertTrue("local checkpoint [" + localCheckpoint + "], _seq_no [" + i + "] should be indexed", seqNosBitSet.get(i));
                }
            }
            prevLocalCheckpoint = localCheckpoint;
            prevMaxSeqNo = maxSeqNo;
        }
    }
}

Example 9 with FieldStats

use of org.elasticsearch.action.fieldstats.FieldStats in project elasticsearch by elastic.

the class FieldStatsTests method testString.

public void testString() {
    createIndex("test", Settings.EMPTY, "test", "field_index", makeType("keyword", true, false, false), "field_dv", makeType("keyword", false, true, false), "field_stored", makeType("keyword", false, true, true), "field_source", makeType("keyword", false, false, false));
    for (int value = 0; value <= 10; value++) {
        String keyword = String.format(Locale.ENGLISH, "%03d", value);
        client().prepareIndex("test", "test").setSource("field_index", keyword, "field_dv", keyword, "field_stored", keyword, "field_source", keyword).get();
    }
    client().admin().indices().prepareRefresh().get();
    FieldStatsResponse result = client().prepareFieldStats().setFields("field_index", "field_dv", "field_stored", "field_source").get();
    assertEquals(result.getAllFieldStats().size(), 3);
    for (String field : new String[] { "field_index", "field_dv", "field_stored" }) {
        FieldStats stats = result.getAllFieldStats().get(field);
        assertEquals(stats.getMaxDoc(), 11L);
        assertEquals(stats.getDisplayType(), "string");
        if ("field_index".equals(field)) {
            assertEquals(stats.getMinValue(), new BytesRef(String.format(Locale.ENGLISH, "%03d", 0)));
            assertEquals(stats.getMaxValue(), new BytesRef(String.format(Locale.ENGLISH, "%03d", 10)));
            assertEquals(stats.getMinValueAsString(), String.format(Locale.ENGLISH, "%03d", 0));
            assertEquals(stats.getMaxValueAsString(), String.format(Locale.ENGLISH, "%03d", 10));
            assertEquals(stats.getDocCount(), 11L);
            assertEquals(stats.getDensity(), 100);
        } else {
            assertEquals(stats.getDocCount(), 0L);
            assertNull(stats.getMinValue());
            assertNull(stats.getMaxValue());
            assertEquals(stats.getDensity(), 0);
        }
    }
}

Example 10 with FieldStats

use of org.elasticsearch.action.fieldstats.FieldStats in project elasticsearch by elastic.

the class FieldStatsTests method testDouble.

public void testDouble() {
    createIndex("test", Settings.EMPTY, "test", "field_index", makeType("double", true, false, false), "field_dv", makeType("double", false, true, false), "field_stored", makeType("double", false, true, true), "field_source", makeType("double", false, false, false));
    for (double value = -1; value <= 9; value++) {
        client().prepareIndex("test", "test").setSource("field_index", value, "field_dv", value, "field_stored", value, "field_source", value).get();
    }
    client().admin().indices().prepareRefresh().get();
    FieldStatsResponse result = client().prepareFieldStats().setFields("field_index", "field_dv", "field_stored", "field_source").get();
    for (String field : new String[] { "field_index", "field_dv", "field_stored" }) {
        FieldStats stats = result.getAllFieldStats().get(field);
        assertEquals(stats.getMaxDoc(), 11L);
        assertEquals(stats.getDisplayType(), "float");
        if ("field_index".equals(field)) {
            assertEquals(stats.getDocCount(), 11L);
            assertEquals(stats.getDensity(), 100);
            assertEquals(stats.getMinValue(), -1d);
            assertEquals(stats.getMaxValue(), 9d);
            assertEquals(stats.getMinValueAsString(), Double.toString(-1));
        } else {
            assertEquals(stats.getDocCount(), 0L);
            assertNull(stats.getMinValue());
            assertNull(stats.getMaxValue());
            assertEquals(stats.getDensity(), 0);
        }
    }
}