Examples with SchemaField org.apache.solr.schema.SchemaField used on opensource projects

Example 86 with SchemaField

use of org.apache.solr.schema.SchemaField in project lucene-solr by apache.

the class StatsComponentTest method testIndividualStatLocalParams.

public void testIndividualStatLocalParams() throws Exception {
    final String kpre = ExpectedStat.KPRE;
    assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
    assertU(commit());
    SolrCore core = h.getCore();
    SchemaField field = core.getLatestSchema().getField("a_i");
    HllOptions hllOpts = HllOptions.parseHllOptions(params("cardinality", "true"), field);
    HLL hll = hllOpts.newHLL();
    HashFunction hasher = hllOpts.getHasher();
    AVLTreeDigest tdigest = new AVLTreeDigest(100);
    // some quick sanity check assertions...
    // trivial check that we only get the exact 2 we ask for
    assertQ("ask for and get only 2 stats", req("q", "*:*", "stats", "true", "stats.field", "{!key=k mean=true min=true}a_i"), kpre + "double[@name='mean'][.='9.0']", kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=2");
    // for stats that are true/false, sanity check false does it's job
    assertQ("min=true & max=false: only min should come back", req("q", "*:*", "stats", "true", "stats.field", "{!key=k max=false min=true}a_i"), kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=1");
    assertQ("min=false: localparam stat means ignore default set, " + "but since only local param is false no stats should be returned", req("q", "*:*", "stats", "true", "stats.field", "{!key=k min=false}a_i"), // section of stats for this field should exist ...
    XPRE + "lst[@name='stats_fields']/lst[@name='k']", // ...but be empty 
    "count(" + kpre + "*)=0");
    double sum = 0;
    double sumOfSquares = 0;
    final int count = 20;
    for (int i = 0; i < count; i++) {
        int a_i = i % 10;
        assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i", String.valueOf(a_i), "foo_t", "how now brown cow"));
        tdigest.add(a_i);
        hll.addRaw(hasher.hashInt(a_i).asLong());
        sum += a_i;
        sumOfSquares += (a_i) * (a_i);
    }
    double stddev = Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)));
    assertU(commit());
    ByteBuffer tdigestBuf = ByteBuffer.allocate(tdigest.smallByteSize());
    tdigest.asSmallBytes(tdigestBuf);
    byte[] hllBytes = hll.toBytes();
    EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
    final List<ExpectedStat> expected = new ArrayList<ExpectedStat>(allStats.size());
    ExpectedStat.createSimple(Stat.min, "true", "double", "0.0");
    ExpectedStat.createSimple(Stat.max, "true", "double", "9.0");
    ExpectedStat.createSimple(Stat.missing, "true", "long", "0");
    ExpectedStat.createSimple(Stat.sum, "true", "double", String.valueOf(sum));
    ExpectedStat.createSimple(Stat.count, "true", "long", String.valueOf(count));
    ExpectedStat.createSimple(Stat.mean, "true", "double", String.valueOf(sum / count));
    ExpectedStat.createSimple(Stat.sumOfSquares, "true", "double", String.valueOf(sumOfSquares));
    ExpectedStat.createSimple(Stat.stddev, "true", "double", String.valueOf(stddev));
    final String distinctValsXpath = "count(" + kpre + "arr[@name='distinctValues']/*)=10";
    ExpectedStat.create(Stat.distinctValues, "true", Collections.singletonList(distinctValsXpath), Collections.singletonList(distinctValsXpath));
    ExpectedStat.createSimple(Stat.countDistinct, "true", "long", "10");
    final String percentileShardXpath = kpre + "str[@name='percentiles'][.='" + Base64.byteArrayToBase64(tdigestBuf.array(), 0, tdigestBuf.array().length) + "']";
    final String p90 = "" + tdigest.quantile(0.90D);
    final String p99 = "" + tdigest.quantile(0.99D);
    ExpectedStat.create(Stat.percentiles, "'90, 99'", Collections.singletonList(percentileShardXpath), Arrays.asList("count(" + kpre + "lst[@name='percentiles']/*)=2", kpre + "lst[@name='percentiles']/double[@name='90.0'][.=" + p90 + "]", kpre + "lst[@name='percentiles']/double[@name='99.0'][.=" + p99 + "]"));
    final String cardinalityShardXpath = kpre + "str[@name='cardinality'][.='" + Base64.byteArrayToBase64(hllBytes, 0, hllBytes.length) + "']";
    final String cardinalityXpath = kpre + "long[@name='cardinality'][.='10']";
    ExpectedStat.create(Stat.cardinality, "true", Collections.singletonList(cardinalityShardXpath), Collections.singletonList(cardinalityXpath));
    // canary in the coal mine
    assertEquals("num of ExpectedStat doesn't match all known stats; " + "enum was updated w/o updating test?", ExpectedStat.ALL.size(), allStats.size());
    // whitebox test: explicitly ask for isShard=true with each individual stat
    for (ExpectedStat expect : ExpectedStat.ALL.values()) {
        Stat stat = expect.stat;
        StringBuilder exclude = new StringBuilder();
        List<String> testXpaths = new ArrayList<String>(5 + expect.perShardXpaths.size());
        testXpaths.addAll(expect.perShardXpaths);
        int numKeysExpected = 0;
        EnumSet<Stat> distribDeps = stat.getDistribDeps();
        for (Stat perShardDep : distribDeps) {
            numKeysExpected++;
            // the shard should return them since they are a dependency for the requested stat
            if (!stat.equals(perShardDep)) {
                // NOTE: this only works because all the cases where there are distribDeps
                // beyond a self dependency are simple true/false options
                exclude.append(perShardDep + "=false ");
            }
        }
        // we don't want to find anything we aren't expecting
        testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
        assertQ("ask for only " + stat + ", with isShard=true, and expect only deps: " + distribDeps, req("q", "*:*", "isShard", "true", "stats", "true", "stats.field", "{!key=k " + exclude + stat + "=" + expect.input + "}a_i"), testXpaths.toArray(new String[testXpaths.size()]));
    }
    // test all the possible combinations (of all possible sizes) of stats params
    for (int numParams = 1; numParams <= allStats.size(); numParams++) {
        for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
            // EnumSets use natural ordering, we want to randomize the order of the params
            List<Stat> combo = new ArrayList<Stat>(set);
            Collections.shuffle(combo, random());
            StringBuilder paras = new StringBuilder("{!key=k ");
            List<String> testXpaths = new ArrayList<String>(numParams + 5);
            int numKeysExpected = 0;
            for (Stat stat : combo) {
                ExpectedStat expect = ExpectedStat.ALL.get(stat);
                paras.append(stat + "=" + expect.input + " ");
                numKeysExpected++;
                testXpaths.addAll(expect.finalXpaths);
            }
            paras.append("}a_i");
            // we don't want to find anything we aren't expecting
            testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
            assertQ("ask for and get only: " + combo, req("q", "*:*", "stats", "true", "stats.field", paras.toString()), testXpaths.toArray(new String[testXpaths.size()]));
        }
    }
}

Example 87 with SchemaField

use of org.apache.solr.schema.SchemaField in project lucene-solr by apache.

the class UninvertDocValuesMergePolicyTest method implUpdateSchemaField.

private static void implUpdateSchemaField(TestHarness h, String fieldName, IntUnaryOperator propertiesModifier) {
    try (SolrCore core = h.getCoreInc()) {
        // Add docvalues to the field type
        IndexSchema schema = core.getLatestSchema();
        SchemaField oldSchemaField = schema.getField(fieldName);
        SchemaField newSchemaField = new SchemaField(fieldName, oldSchemaField.getType(), propertiesModifier.applyAsInt(oldSchemaField.getProperties()), oldSchemaField.getDefaultValue());
        schema.getFields().put(fieldName, newSchemaField);
    }
}

Example 88 with SchemaField

use of org.apache.solr.schema.SchemaField in project lucene-solr by apache.

the class TestIndexSearcher method getStringVal.

private String getStringVal(SolrQueryRequest sqr, String field, int doc) throws IOException {
    SchemaField sf = sqr.getSchema().getField(field);
    ValueSource vs = sf.getType().getValueSource(sf, null);
    Map context = ValueSource.newContext(sqr.getSearcher());
    vs.createWeight(context, sqr.getSearcher());
    IndexReaderContext topReaderContext = sqr.getSearcher().getTopReaderContext();
    List<LeafReaderContext> leaves = topReaderContext.leaves();
    int idx = ReaderUtil.subIndex(doc, leaves);
    LeafReaderContext leaf = leaves.get(idx);
    FunctionValues vals = vs.getValues(context, leaf);
    return vals.strVal(doc - leaf.docBase);
}

Example 89 with SchemaField

use of org.apache.solr.schema.SchemaField in project lucene-solr by apache.

the class SortSpecParsingTest method testSort.

@Test
public void testSort() throws Exception {
    Sort sort;
    SortSpec spec;
    SolrQueryRequest req = req();
    sort = doParseSortSpec("score desc", req).getSort();
    //only 1 thing in the list, no Sort specified
    assertNull("sort", sort);
    spec = doParseSortSpec("score desc", req);
    assertNotNull("spec", spec);
    assertNull(spec.getSort());
    assertNotNull(spec.getSchemaFields());
    assertEquals(0, spec.getSchemaFields().size());
    // SOLR-4458 - using different case variations of asc and desc
    sort = doParseSortSpec("score aSc", req).getSort();
    SortField[] flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.SCORE);
    assertTrue(flds[0].getReverse());
    spec = doParseSortSpec("score aSc", req);
    flds = spec.getSort().getSort();
    assertEquals(1, flds.length);
    assertEquals(flds[0].getType(), SortField.Type.SCORE);
    assertTrue(flds[0].getReverse());
    assertEquals(1, spec.getSchemaFields().size());
    assertNull(spec.getSchemaFields().get(0));
    sort = doParseSortSpec("weight dEsC", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[0].getReverse(), true);
    spec = doParseSortSpec("weight dEsC", req);
    flds = spec.getSort().getSort();
    assertEquals(1, flds.length);
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[0].getReverse(), true);
    assertEquals(1, spec.getSchemaFields().size());
    assertNotNull(spec.getSchemaFields().get(0));
    assertEquals("weight", spec.getSchemaFields().get(0).getName());
    sort = doParseSortSpec("weight desc,bday ASC", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[0].getReverse(), true);
    assertEquals(flds[1].getType(), SortField.Type.LONG);
    assertEquals(flds[1].getField(), "bday");
    assertEquals(flds[1].getReverse(), false);
    //order aliases
    sort = doParseSortSpec("weight top,bday asc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[0].getReverse(), true);
    assertEquals(flds[1].getType(), SortField.Type.LONG);
    assertEquals(flds[1].getField(), "bday");
    assertEquals(flds[1].getReverse(), false);
    sort = doParseSortSpec("weight top,bday bottom", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[0].getReverse(), true);
    assertEquals(flds[1].getType(), SortField.Type.LONG);
    assertEquals(flds[1].getField(), "bday");
    assertEquals(flds[1].getReverse(), false);
    //test weird spacing
    sort = doParseSortSpec("weight         DESC,            bday         asc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    assertEquals(flds[1].getField(), "bday");
    assertEquals(flds[1].getType(), SortField.Type.LONG);
    //handles trailing commas
    sort = doParseSortSpec("weight desc,", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    //test functions
    sort = SortSpecParsing.parseSortSpec("pow(weight, 2) desc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.REWRITEABLE);
    //Not thrilled about the fragility of string matching here, but...
    //the value sources get wrapped, so the out field is different than the input
    assertEquals(flds[0].getField(), "pow(float(weight),const(2))");
    //test functions (more deep)
    sort = SortSpecParsing.parseSortSpec("sum(product(r_f1,sum(d_f1,t_f1,1.0)),a_f1) asc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.REWRITEABLE);
    assertEquals(flds[0].getField(), "sum(product(float(r_f1),sum(float(d_f1),float(t_f1),const(1.0))),float(a_f1))");
    sort = SortSpecParsing.parseSortSpec("pow(weight,                 2.0)         desc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.REWRITEABLE);
    //Not thrilled about the fragility of string matching here, but...
    //the value sources get wrapped, so the out field is different than the input
    assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
    spec = SortSpecParsing.parseSortSpec("pow(weight, 2.0) desc, weight    desc,   bday    asc", req);
    flds = spec.getSort().getSort();
    List<SchemaField> schemaFlds = spec.getSchemaFields();
    assertEquals(3, flds.length);
    assertEquals(3, schemaFlds.size());
    assertEquals(flds[0].getType(), SortField.Type.REWRITEABLE);
    //Not thrilled about the fragility of string matching here, but...
    //the value sources get wrapped, so the out field is different than the input
    assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
    assertNull(schemaFlds.get(0));
    assertEquals(flds[1].getType(), SortField.Type.FLOAT);
    assertEquals(flds[1].getField(), "weight");
    assertNotNull(schemaFlds.get(1));
    assertEquals("weight", schemaFlds.get(1).getName());
    assertEquals(flds[2].getField(), "bday");
    assertEquals(flds[2].getType(), SortField.Type.LONG);
    assertNotNull(schemaFlds.get(2));
    assertEquals("bday", schemaFlds.get(2).getName());
    //handles trailing commas
    sort = doParseSortSpec("weight desc,", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.FLOAT);
    assertEquals(flds[0].getField(), "weight");
    //Test literals in functions
    sort = SortSpecParsing.parseSortSpec("strdist(foo_s1, \"junk\", jw) desc", req).getSort();
    flds = sort.getSort();
    assertEquals(flds[0].getType(), SortField.Type.REWRITEABLE);
    //the value sources get wrapped, so the out field is different than the input
    assertEquals(flds[0].getField(), "strdist(str(foo_s1),literal(junk), dist=org.apache.lucene.search.spell.JaroWinklerDistance)");
    sort = doParseSortSpec("", req).getSort();
    assertNull(sort);
    spec = doParseSortSpec("", req);
    assertNotNull(spec);
    assertNull(spec.getSort());
    req.close();
}

Example 90 with SchemaField

use of org.apache.solr.schema.SchemaField in project lucene-solr by apache.

the class UUIDUpdateProcessorFactory method getInstance.

public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
    if (StringUtils.isEmpty(fieldName)) {
        SchemaField schemaField = req.getSchema().getUniqueKeyField();
        fieldName = schemaField.getName();
    }
    return new AbstractDefaultValueUpdateProcessorFactory.DefaultValueUpdateProcessor(fieldName, next) {

        @Override
        public Object getDefaultValue() {
            return UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
        }
    };
}