Examples with FieldDescriptor com.rbmhtechnology.vind.model.FieldDescriptor used on opensource projects

Example 1 with FieldDescriptor

use of com.rbmhtechnology.vind.model.FieldDescriptor in project vind by RBMHTechnology.

the class SolrSearchServer method execute.

@Override
public boolean execute(Update update, DocumentFactory factory) {
    // Check if document is updatable and all its fields are stored.
    final boolean isUpdatable = factory.isUpdatable() && factory.getFields().values().stream().allMatch(descriptor -> descriptor.isUpdate());
    if (isUpdatable) {
        final SolrInputDocument sdoc = new SolrInputDocument();
        sdoc.addField(ID, update.getId());
        sdoc.addField(TYPE, factory.getType());
        HashMap<FieldDescriptor<?>, HashMap<String, SortedSet<UpdateOperation>>> updateOptions = update.getOptions();
        updateOptions.keySet().forEach(fieldDescriptor -> Stream.of(UseCase.values()).forEach(useCase -> updateOptions.get(fieldDescriptor).keySet().stream().forEach(context -> {
            // NOTE: Backwards compatibility
            final String updateContext = Objects.isNull(context) ? update.getUpdateContext() : context;
            final String fieldName = getFieldname(fieldDescriptor, useCase, updateContext);
            if (fieldName != null) {
                final Map<String, Object> fieldModifiers = new HashMap<>();
                updateOptions.get(fieldDescriptor).get(context).stream().forEach(entry -> {
                    UpdateOperations opType = entry.getType();
                    if (fieldName.startsWith("dynamic_single_") && useCase.equals(UseCase.Sort) && opType.equals(UpdateOperations.add)) {
                        opType = set;
                    }
                    fieldModifiers.put(opType.name(), toSolrJType(SolrUtils.FieldValue.getFieldCaseValue(entry.getValue(), fieldDescriptor, useCase)));
                });
                sdoc.addField(fieldName, fieldModifiers);
            }
        })));
        try {
            if (solrClientLogger.isTraceEnabled()) {
                solrClientLogger.debug(">>> add({}): {}", update.getId(), sdoc);
            } else {
                solrClientLogger.debug(">>> add({})", update.getId());
            }
            SolrInputDocument finalDoc = sdoc;
            // Get the original document
            final SolrDocument updatedDoc = solrClient.getById(update.getId());
            // Setting the document version for optimistic concurrency
            final Object version = updatedDoc.getFieldValue("_version_");
            if (Objects.nonNull(version)) {
                finalDoc.setField("_version_", version);
            } else {
                log.warn("Error updating document '{}': " + "Atomic updates in nested documents are not supported by Solr", updatedDoc.get(ID));
                return false;
            }
            // Get the nested docs of the document if existing
            final NamedList<Object> paramList = new NamedList<>();
            paramList.add(CommonParams.Q, "!( _id_:" + update.getId() + ")&(_root_:" + update.getId() + ")");
            final QueryResponse query = solrClient.query(SolrParams.toSolrParams(paramList), SolrRequest.METHOD.POST);
            // if the document has nested docs solr does not support atomic updates
            if (CollectionUtils.isNotEmpty(query.getResults())) {
                log.info("Update document `{}`: doc has {} nested documents, changing from partial update to full index.", finalDoc.getFieldValue(SolrUtils.Fieldname.ID), query.getResults().size());
                // Get the list of nested documents
                final List<SolrInputDocument> childDocs = query.getResults().stream().map(nestedDoc -> ClientUtils.toSolrInputDocument(nestedDoc)).collect(Collectors.toList());
                finalDoc = this.getUpdatedSolrDocument(sdoc, updatedDoc, childDocs);
            }
            try {
                log.info("Updating document `{}`: current version `{}`", finalDoc.getFieldValue(SolrUtils.Fieldname.ID), version);
                solrClient.add(finalDoc);
                return true;
            } catch (HttpSolrClient.RemoteSolrException e) {
                log.warn("Error updating document {}: {}", finalDoc.getFieldValue(ID), e.getMessage(), e);
                return false;
            }
        } catch (SolrServerException | IOException e) {
            log.error("Unable to perform solr partial update on document with id [{}]", update.getId(), e);
            throw new SearchServerException("Can not execute solr partial update.", e);
        }
    } else {
        Exception e = new SearchServerException("It is not safe to execute solr partial update: Document contains non stored fields");
        log.error("Unable to perform solr partial update on document with id [{}]", update.getId(), e);
        throw new RuntimeException("Can not execute solr partial update.", e);
    }
}

Example 2 with FieldDescriptor

use of com.rbmhtechnology.vind.model.FieldDescriptor in project vind by RBMHTechnology.

the class SolrSearchServer method buildSolrQuery.

protected SolrQuery buildSolrQuery(ExecutableSuggestionSearch search, DocumentFactory assets, DocumentFactory childFactory) {
    final String searchContext = search.getSearchContext();
    final SolrQuery query = new SolrQuery();
    query.setRequestHandler("/suggester");
    if (search.isStringSuggestion()) {
        StringSuggestionSearch s = (StringSuggestionSearch) search;
        query.setParam("suggestion.field", s.getSuggestionFields().stream().map(name -> {
            if (Objects.nonNull(childFactory)) {
                final FieldDescriptor<?> field = Objects.nonNull(assets.getField(name)) ? assets.getField(name) : childFactory.getField(name);
                if (Objects.isNull(field)) {
                    log.warn("No field descriptor found for field name {} in factories: {}, {}", name, assets.getType(), childFactory.getType());
                }
                return getFieldname(field, UseCase.Suggest, searchContext);
            } else {
                if (Objects.isNull(assets.getField(name))) {
                    log.warn("No field descriptor found for field name {} in factory: {}", name, assets.getType());
                }
                return getFieldname(assets.getField(name), UseCase.Suggest, searchContext);
            }
        }).filter(Objects::nonNull).toArray(String[]::new));
    } else {
        DescriptorSuggestionSearch s = (DescriptorSuggestionSearch) search;
        query.setParam("suggestion.field", s.getSuggestionFields().stream().map(descriptor -> getFieldname(descriptor, UseCase.Suggest, searchContext)).filter(Objects::nonNull).toArray(String[]::new));
    }
    query.setParam("q", search.getInput());
    // TODO: somehow this is still needed here, it should by configuration
    query.setParam("suggestion.df", SUGGESTION_DF_FIELD);
    query.setParam("suggestion.limit", String.valueOf(search.getLimit()));
    String parentTypeFilter = "_type_:" + assets.getType();
    if (Objects.nonNull(childFactory)) {
        parentTypeFilter = "(" + parentTypeFilter + " OR _type_:" + childFactory.getType() + ")";
    }
    query.add(CommonParams.FQ, parentTypeFilter);
    // filters
    if (search.hasFilter()) {
        SolrUtils.Query.buildFilterString(search.getFilter(), assets, childFactory, query, searchContext, false);
        new SolrChildrenSerializerVisitor(assets, childFactory, searchContext, false);
    }
    // suggestion deep search
    if (Objects.nonNull(childFactory)) {
        if (search.hasFilter()) {
            // TODO clean up!
            final String parentFilterQuery = "(" + String.join(" AND ", query.getFilterQueries()) + ")";
            final String childrenFilterQuery = search.getFilter().accept(new SolrChildrenSerializerVisitor(assets, childFactory, searchContext, false));
            final String childrenBJQ = "{!child of=\"_type_:" + assets.getType() + "\" v='" + childrenFilterQuery + "'}";
            query.set(CommonParams.FQ, String.join(" ", parentFilterQuery, "OR", childrenBJQ));
        }
    }
    return query;
}

Example 3 with FieldDescriptor

use of com.rbmhtechnology.vind.model.FieldDescriptor in project vind by RBMHTechnology.

the class SolrSearchServer method buildSolrQuery.

protected SolrQuery buildSolrQuery(FulltextSearch search, DocumentFactory factory) {
    // build query
    final SolrQuery query = new SolrQuery();
    final String searchContext = search.getSearchContext();
    if (search.getTimeZone() != null) {
        query.set(CommonParams.TZ, search.getTimeZone());
    }
    // fulltext search
    query.set(CommonParams.Q, search.getSearchString());
    if (SearchConfiguration.get(SearchConfiguration.SEARCH_RESULT_SHOW_SCORE, true)) {
        query.set(CommonParams.FL, "*,score");
    } else {
        query.set(CommonParams.FL, "*");
    }
    if (search.getGeoDistance() != null) {
        final FieldDescriptor descriptor = factory.getField(search.getGeoDistance().getFieldName());
        if (Objects.nonNull(descriptor)) {
            query.setParam(CommonParams.FL, query.get(CommonParams.FL) + "," + DISTANCE + ":geodist()");
            query.setParam("pt", search.getGeoDistance().getLocation().toString());
            query.setParam("sfield", getFieldname(descriptor, UseCase.Facet, searchContext));
        }
    }
    Collection<FieldDescriptor<?>> fulltext = factory.listFields().stream().filter(FieldDescriptor::isFullText).collect(Collectors.toList());
    if (!fulltext.isEmpty()) {
        query.setParam(DisMaxParams.QF, SolrUtils.Query.buildQueryFieldString(fulltext, searchContext));
        query.setParam("defType", "edismax");
    } else {
        query.setParam(CommonParams.DF, TEXT);
    }
    // filters
    query.add(CommonParams.FQ, "_type_:" + factory.getType());
    if (search.hasFilter()) {
        SolrUtils.Query.buildFilterString(search.getFilter(), factory, search.getChildrenFactory(), query, searchContext, search.getStrict());
    }
    // fulltext search deep search
    if (search.isChildrenSearchEnabled()) {
        // append childCount facet
        search.facet(new Facet.SubdocumentFacet(factory));
        // TODO: move to SolrUtils
        final String parentSearchQuery = "((" + query.get(CommonParams.Q) + ") AND " + TYPE + ":" + factory.getType() + ")";
        final String childrenSearchQuery = "_query_:\"{!parent which=" + TYPE + ":" + factory.getType() + "}(" + TYPE + ":" + search.getChildrenFactory().getType() + " AND (" + search.getChildrenSearchString().getEscapedSearchString() + "))\"";
        query.set(CommonParams.Q, String.join(" ", parentSearchQuery, search.getChildrenSearchOperator().name(), childrenSearchQuery));
        if (search.getChildrenSearchString().hasFilter()) {
            // TODO clean up!
            final String parentFilterQuery = "(" + String.join(" AND ", query.getFilterQueries()) + ")";
            final String childrenFilterQuery = search.getChildrenSearchString().getFilter().accept(new SolrChildrenSerializerVisitor(factory, search.getChildrenFactory(), searchContext, search.getStrict()));
            query.set(CommonParams.FQ, String.join(" ", parentFilterQuery, search.getChildrenSearchOperator().name(), "(" + childrenFilterQuery + ")"));
        }
    }
    if (search.hasFacet()) {
        query.setFacet(true);
        query.setFacetMinCount(search.getFacetMinCount());
        query.setFacetLimit(search.getFacetLimit());
        // Query facets
        search.getFacets().values().stream().filter(facet -> Facet.QueryFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.QueryFacet) genericFacet).forEach(queryFacet -> query.addFacetQuery(StringUtils.join(SolrUtils.Query.buildSolrFacetCustomName(SolrUtils.Query.buildFilterString(queryFacet.getFilter(), factory, search.getChildrenFactory(), searchContext, search.getStrict()), queryFacet))));
        // Numeric Range facet
        search.getFacets().values().stream().filter(facet -> Facet.NumericRangeFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.NumericRangeFacet) genericFacet).forEach(numericRangeFacet -> {
            final UseCase useCase = UseCase.valueOf(numericRangeFacet.getScope().name());
            final String fieldName = getFieldname(numericRangeFacet.getFieldDescriptor(), useCase, searchContext);
            query.add(FacetParams.FACET_RANGE, SolrUtils.Query.buildSolrFacetCustomName(fieldName, numericRangeFacet));
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_START), numericRangeFacet.getStart().toString());
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_END), numericRangeFacet.getEnd().toString());
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_GAP), numericRangeFacet.getGap().toString());
        /*query.addNumericRangeFacet(
                                    SolrUtils.Query.buildSolrFacetCustomName(fieldName, numericRangeFacet.getName()),
                                    numericRangeFacet.getStart(),
                                    numericRangeFacet.getEnd(),
                                    numericRangeFacet.getGap());*/
        });
        // Interval Range facet
        search.getFacets().values().stream().filter(facet -> Facet.IntervalFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.IntervalFacet) genericFacet).forEach(intervalFacet -> {
            final UseCase useCase = UseCase.valueOf(intervalFacet.getScope().name());
            final String fieldName = getFieldname(intervalFacet.getFieldDescriptor(), useCase, searchContext);
            query.add(FacetParams.FACET_INTERVAL, SolrUtils.Query.buildSolrFacetKey(intervalFacet.getName()) + fieldName);
            for (Object o : intervalFacet.getIntervals()) {
                // TODO why is this necessary?
                Interval i = (Interval) o;
                query.add(String.format("f.%s.%s", fieldName, FacetParams.FACET_INTERVAL_SET), String.format("%s%s%s,%s%s", SolrUtils.Query.buildSolrFacetKey(i.getName()), i.includesStart() ? "[" : "(", i.getStart() == null ? SOLR_WILDCARD : SolrUtils.Query.buildSolrQueryValue(i.getStart()), i.getEnd() == null ? SOLR_WILDCARD : SolrUtils.Query.buildSolrQueryValue(i.getEnd()), i.includesEnd() ? "]" : ")"));
            }
        });
        // Date Range facet
        search.getFacets().values().stream().filter(facet -> Facet.DateRangeFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.DateRangeFacet) genericFacet).forEach(dateRangeFacet -> generateDateRangeQuery(dateRangeFacet, query, searchContext));
        // stats
        search.getFacets().values().stream().filter(facet -> Facet.StatsFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.StatsFacet) genericFacet).forEach(statsFacet -> {
            final UseCase useCase = UseCase.valueOf(statsFacet.getScope().name());
            String fieldName = getFieldname(statsFacet.getField(), useCase, searchContext);
            query.add(StatsParams.STATS, "true");
            query.add(StatsParams.STATS_FIELD, SolrUtils.Query.buildSolrStatsQuery(fieldName, statsFacet));
        });
        // pivot facet
        search.getFacets().values().stream().filter(facet -> Facet.PivotFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.PivotFacet) genericFacet).forEach(pivotFacet -> {
            String[] fieldNames = pivotFacet.getFieldDescriptors().stream().map(fieldDescriptor -> getFieldname(fieldDescriptor, UseCase.Facet, searchContext)).toArray(String[]::new);
            query.add(FacetParams.FACET_PIVOT, SolrUtils.Query.buildSolrPivotSubFacetName(pivotFacet.getName(), fieldNames));
        });
        // facet fields
        final HashMap<String, Object> strings = SolrUtils.Query.buildJsonTermFacet(search.getFacets(), search.getFacetLimit(), factory, search.getChildrenFactory(), searchContext);
        query.add("json.facet", strings.toString().replaceAll("=", ":"));
        // facet Subdocument count
        final String subdocumentFacetString = SolrUtils.Query.buildSubdocumentFacet(search, factory, searchContext);
        if (Objects.nonNull(subdocumentFacetString)) {
            query.add("json.facet", subdocumentFacetString);
        }
    }
    // sorting
    if (search.hasSorting()) {
        final String sortString = SolrUtils.Query.buildSortString(search, search.getSorting(), factory);
        query.set(CommonParams.SORT, sortString);
    }
    // TODO this is a mess
    if (search.hasSorting()) {
        query.set(DisMaxParams.BF, SolrUtils.Query.buildBoostFunction(search.getSorting(), searchContext));
    }
    // paging
    switch(search.getResultSet().getType()) {
        case page:
            {
                final Page resultSet = (Page) search.getResultSet();
                query.setStart(resultSet.getOffset());
                query.setRows(resultSet.getPagesize());
                break;
            }
        case slice:
            {
                final Slice resultSet = (Slice) search.getResultSet();
                query.setStart(resultSet.getOffset());
                query.setRows(resultSet.getSliceSize());
                break;
            }
    }
    return query;
}

Example 4 with FieldDescriptor

use of com.rbmhtechnology.vind.model.FieldDescriptor in project vind by RBMHTechnology.

the class SuggestionTest method testSpellcheck.

@Test
public void testSpellcheck() {
    SolrClient client = Mockito.mock(SolrClient.class);
    SolrSearchServer server = new SolrSearchServer(client, false);
    FieldDescriptor descriptor = Mockito.mock(FieldDescriptor.class);
    when(descriptor.getType()).thenReturn(String.class);
    when(descriptor.isSuggest()).thenReturn(true);
    DocumentFactory factory = Mockito.mock(DocumentFactory.class);
    when(factory.getField(any())).thenReturn(descriptor);
    ExecutableSuggestionSearch search = Search.suggest("abc").fields("field");
    SolrQuery query = server.buildSolrQuery(search, factory, null);
    assertEquals("abc", query.get("q"));
    assertEquals("dynamic_single_suggest_string_null", query.get("suggestion.field"));
    assertEquals(SolrSearchServer.SUGGESTION_DF_FIELD, query.get("suggestion.df"));
    assertEquals("10", query.get("suggestion.limit"));
    search.setLimit(100);
    query = server.buildSolrQuery(search, factory, null);
    assertEquals("100", query.get("suggestion.limit"));
    ExecutableSuggestionSearch search2 = Search.suggest("abc").fields(descriptor).clearFilter();
    query = server.buildSolrQuery(search2, factory, null);
    assertEquals("10", query.get("suggestion.limit"));
}

Example 5 with FieldDescriptor

use of com.rbmhtechnology.vind.model.FieldDescriptor in project vind by RBMHTechnology.

the class RESTMetadataProvider method getDocument.

@Override
public Document getDocument(Document document, DocumentFactory factory) throws IOException {
    HttpMethod request = new GetMethod(baseURL);
    // TODO
    request.addRequestHeader("Authorization", String.format("Bearer %s", bearerToken));
    request.setPath(path + document.getId());
    int status = client.executeMethod(request);
    if (status == 200) {
        JsonNode json = mapper.readValue(request.getResponseBody(), JsonNode.class);
        for (FieldDescriptor descriptor : factory.listFields()) {
            try {
                Object value = getValue(json, descriptor);
                if (value != null) {
                    document.setValue(descriptor, value);
                } else
                    LOG.warn("No data found for id {}", document.getId());
            } catch (IOException e) {
                LOG.warn("Cannot use data for id {}: {}", document.getId(), e.getMessage());
            }
        }
        return document;
    } else
        throw new IOException(request.getStatusText());
}