Search in sources :

Example 21 with StringUtils.join

use of org.apache.commons.lang3.StringUtils.join in project spring-roo by spring-projects.

the class RepositoryJpaCustomImplMetadata method buildQuery.

/**
 * Builds the search query
 *
 * @param bodyBuilder method body builder
 * @param entityVariable name of the variable that contains the Q entity
 * @param globalSearch global search variable name
 * @param referencedFieldParamName
 * @param referencedField
 * @param referencedFieldIdentifierPathName
 * @param formBeanType the JavaType which contains the fields to use for filtering.
 *            Can be null in findAll queries.
 * @param formBeanParameterName the name of the search param
 * @param returnType
 * @param finderName the name of the finder. Only available when method is a
 *            projection/DTO finder.
 * @param partTree
 */
private void buildQuery(InvocableMemberBodyBuilder bodyBuilder, String entityVariable, JavaSymbolName globalSearch, JavaSymbolName referencedFieldParamName, FieldMetadata referencedField, String referencedFieldIdentifierPathName, JavaType formBeanType, String formBeanParameterName, JavaType returnType, JavaSymbolName finderName, PartTree partTree) {
    // Prepare leftJoin for compositions oneToOne
    StringBuilder fetchJoins = new StringBuilder();
    for (RelationInfo relationInfo : entityMetadata.getRelationInfos().values()) {
        if (relationInfo.type == JpaRelationType.COMPOSITION && relationInfo.cardinality == Cardinality.ONE_TO_ONE) {
            fetchJoins.append(".leftJoin(");
            fetchJoins.append(entityVariable);
            fetchJoins.append(".");
            fetchJoins.append(relationInfo.fieldName);
            fetchJoins.append(")");
        }
    }
    // JPQLQuery query = from(qEntity);
    bodyBuilder.appendFormalLine(String.format("%s query = from(%s)%s;", getNameOfJavaType(getJPQLQueryFor(entity)), entityVariable, fetchJoins));
    bodyBuilder.newLine();
    if (formBeanType != null) {
        // Query for finder
        // if (formSearch != null) {
        bodyBuilder.appendFormalLine(String.format("if (%s != null) {", formBeanParameterName));
        if (partTree != null) {
            buildFormBeanFilterBody(bodyBuilder, formBeanType, formBeanParameterName, entityVariable, partTree);
        } else {
            // formBean is an entity, filter by all its fields
            for (Entry<String, FieldMetadata> field : this.typesFieldsMetadata.get(formBeanType).entrySet()) {
                // if (formSearch.getField() != null) {
                String accessorMethodName = BeanInfoUtils.getAccessorMethodName(field.getValue()).getSymbolName();
                bodyBuilder.appendIndent();
                bodyBuilder.appendFormalLine(String.format("if (%s.%s() != null) {", formBeanParameterName, accessorMethodName));
                // Get path field name from field mappings
                String pathFieldName = getValueOfPairFor(this.typesFieldMaps.get(formBeanType), field.getKey());
                // query.where(myEntity.field.eq(formBean.getField()));
                bodyBuilder.appendIndent();
                bodyBuilder.appendIndent();
                bodyBuilder.appendFormalLine(String.format("query.where(%s.eq(%s.%s()));", pathFieldName, formBeanParameterName, accessorMethodName));
                // }
                bodyBuilder.appendIndent();
                bodyBuilder.appendFormalLine("}");
            }
        }
        // }
        bodyBuilder.appendFormalLine("}");
        bodyBuilder.newLine();
    } else if (referencedFieldParamName != null && referencedFieldIdentifierPathName != null) {
        // Query for reference
        // Assert.notNull(referenced, "referenced is required");
        bodyBuilder.appendFormalLine(String.format("%s.notNull(%s, \"%s is required\");", SpringJavaType.ASSERT.getNameIncludingTypeParameters(false, importResolver), referencedFieldParamName, referencedFieldParamName));
        bodyBuilder.newLine();
        // Query should include a where clause
        if (referencedField.getAnnotation(JpaJavaType.MANY_TO_MANY) != null) {
            // query.where(referencedFieldPath.contains(referencedFieldName));
            bodyBuilder.appendFormalLine(String.format("query.where(%s.contains(%s));", referencedFieldIdentifierPathName, referencedFieldParamName));
        } else {
            // query.where(referencedFieldPath.eq(referencedFieldName));
            bodyBuilder.appendFormalLine(String.format("query.where(%s.eq(%s));", referencedFieldIdentifierPathName, referencedFieldParamName));
        }
    }
    // Path<?>[] paths = new Path[] { .... };
    bodyBuilder.appendIndent();
    final String pathType = getNameOfJavaType(QUERYDSL_PATH);
    bodyBuilder.append(String.format("%s<?>[] paths = new %s<?>[] {", pathType, pathType));
    List<String> toAppend = new ArrayList<String>();
    // ... returnType.field1, returnType.field2);
    if (!this.typesAreProjections.get(returnType)) {
        // Return type is the same entity
        Map<String, FieldMetadata> projectionFields = this.typesFieldsMetadata.get(returnType);
        Iterator<Entry<String, FieldMetadata>> iterator = projectionFields.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<String, FieldMetadata> field = iterator.next();
            String fieldName = field.getValue().getFieldName().getSymbolName();
            toAppend.add(entityVariable + "." + fieldName);
        }
    } else {
        // Return type is a projection
        List<Pair<String, String>> projectionFields = this.typesFieldMaps.get(returnType);
        Validate.notNull(projectionFields, "Couldn't get projection fields for %s", this.defaultReturnType);
        Iterator<Pair<String, String>> iterator = projectionFields.iterator();
        while (iterator.hasNext()) {
            Entry<String, String> entry = iterator.next();
            toAppend.add(entry.getValue());
        }
    }
    bodyBuilder.append(StringUtils.join(toAppend, ','));
    bodyBuilder.append("};");
    bodyBuilder.newLine();
    // applyGlobalSearch(search, query, paths);
    bodyBuilder.appendFormalLine("applyGlobalSearch(globalSearch, query, paths);");
}
Also used : FieldMetadata(org.springframework.roo.classpath.details.FieldMetadata) ToStringBuilder(org.apache.commons.lang3.builder.ToStringBuilder) ArrayList(java.util.ArrayList) Entry(java.util.Map.Entry) RelationInfo(org.springframework.roo.addon.jpa.addon.entity.JpaEntityMetadata.RelationInfo) Pair(org.apache.commons.lang3.tuple.Pair)

Example 22 with StringUtils.join

use of org.apache.commons.lang3.StringUtils.join in project vind by RBMHTechnology.

the class SolrSearchServer method execute.

@Override
public void execute(Delete delete, DocumentFactory factory) {
    String query = SolrUtils.Query.buildFilterString(delete.getQuery(), factory, delete.getUpdateContext(), true);
    try {
        solrClientLogger.debug(">>> delete query({})", query);
        // Finding the ID of the documents to delete
        final SolrQuery solrQuery = new SolrQuery();
        solrQuery.setParam(CommonParams.Q, "*:*");
        solrQuery.setParam(CommonParams.FQ, query.trim().replaceAll("^\\+", "").split("\\+"));
        final QueryResponse response = solrClient.query(solrQuery, SolrRequest.METHOD.POST);
        if (Objects.nonNull(response) && CollectionUtils.isNotEmpty(response.getResults())) {
            final List<String> idList = response.getResults().stream().map(doc -> (String) doc.get(ID)).collect(Collectors.toList());
            solrClient.deleteById(idList);
            // Deleting nested documents
            solrClient.deleteByQuery("_root_:(" + StringUtils.join(idList, " OR ") + ")");
        }
    } catch (SolrServerException | IOException e) {
        log.error("Cannot delete with query {}", query, e);
        throw new SearchServerException("Cannot delete with query", e);
    }
}
Also used : Delete(com.rbmhtechnology.vind.api.query.delete.Delete) SearchServerException(com.rbmhtechnology.vind.SearchServerException) URISyntaxException(java.net.URISyntaxException) ZonedDateTime(java.time.ZonedDateTime) LoggerFactory(org.slf4j.LoggerFactory) Asserts(org.apache.http.util.Asserts) StringUtils(org.apache.commons.lang3.StringUtils) SolrServerException(org.apache.solr.client.solrj.SolrServerException) Fieldname(com.rbmhtechnology.vind.solr.backend.SolrUtils.Fieldname) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) UpdateOperation(com.rbmhtechnology.vind.api.query.update.UpdateOperation) ParseException(java.text.ParseException) FileSystemUtils(com.rbmhtechnology.vind.utils.FileSystemUtils) Path(java.nio.file.Path) SearchServer(com.rbmhtechnology.vind.api.SearchServer) Slice(com.rbmhtechnology.vind.api.query.division.Slice) AnnotationUtil(com.rbmhtechnology.vind.annotations.AnnotationUtil) SchemaResponse(org.apache.solr.client.solrj.response.schema.SchemaResponse) Update(com.rbmhtechnology.vind.api.query.update.Update) SchemaRequest(org.apache.solr.client.solrj.request.schema.SchemaRequest) DescriptorSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.DescriptorSuggestionSearch) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) com.rbmhtechnology.vind.api.query(com.rbmhtechnology.vind.api.query) RealTimeGet(com.rbmhtechnology.vind.api.query.get.RealTimeGet) Stream(java.util.stream.Stream) HttpSolrClient(org.apache.solr.client.solrj.impl.HttpSolrClient) SolrPingResponse(org.apache.solr.client.solrj.response.SolrPingResponse) SolrQuery(org.apache.solr.client.solrj.SolrQuery) com.rbmhtechnology.vind.api.result(com.rbmhtechnology.vind.api.result) ClientUtils(org.apache.solr.client.solrj.util.ClientUtils) ServiceProvider(com.rbmhtechnology.vind.api.ServiceProvider) SolrInputDocument(org.apache.solr.common.SolrInputDocument) Interval(com.rbmhtechnology.vind.api.query.facet.Interval) SolrRequest(org.apache.solr.client.solrj.SolrRequest) java.util(java.util) ExecutableSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.ExecutableSuggestionSearch) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) NumberFormat(java.text.NumberFormat) UpdateOperations.set(com.rbmhtechnology.vind.api.query.update.Update.UpdateOperations.set) CollectionUtils(org.apache.commons.collections.CollectionUtils) SearchConfiguration(com.rbmhtechnology.vind.configure.SearchConfiguration) Logger(org.slf4j.Logger) StringSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.StringSuggestionSearch) LatLng(com.rbmhtechnology.vind.model.value.LatLng) Resources(com.google.common.io.Resources) DateUtil(org.apache.solr.common.util.DateUtil) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) org.apache.solr.common.params(org.apache.solr.common.params) IOException(java.io.IOException) NamedList(org.apache.solr.common.util.NamedList) Document(com.rbmhtechnology.vind.api.Document) SolrClient(org.apache.solr.client.solrj.SolrClient) SolrDocument(org.apache.solr.common.SolrDocument) DocumentFactory(com.rbmhtechnology.vind.model.DocumentFactory) Page(com.rbmhtechnology.vind.api.query.division.Page) UpdateOperations(com.rbmhtechnology.vind.api.query.update.Update.UpdateOperations) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) SolrServerException(org.apache.solr.client.solrj.SolrServerException) IOException(java.io.IOException) SearchServerException(com.rbmhtechnology.vind.SearchServerException) SolrQuery(org.apache.solr.client.solrj.SolrQuery)

Example 23 with StringUtils.join

use of org.apache.commons.lang3.StringUtils.join in project vind by RBMHTechnology.

the class SolrSearchServer method buildSolrQuery.

protected SolrQuery buildSolrQuery(FulltextSearch search, DocumentFactory factory) {
    // build query
    final SolrQuery query = new SolrQuery();
    final String searchContext = search.getSearchContext();
    if (search.getTimeZone() != null) {
        query.set(CommonParams.TZ, search.getTimeZone());
    }
    // fulltext search
    query.set(CommonParams.Q, search.getSearchString());
    if (SearchConfiguration.get(SearchConfiguration.SEARCH_RESULT_SHOW_SCORE, true)) {
        query.set(CommonParams.FL, "*,score");
    } else {
        query.set(CommonParams.FL, "*");
    }
    if (search.getGeoDistance() != null) {
        final FieldDescriptor descriptor = factory.getField(search.getGeoDistance().getFieldName());
        if (Objects.nonNull(descriptor)) {
            query.setParam(CommonParams.FL, query.get(CommonParams.FL) + "," + DISTANCE + ":geodist()");
            query.setParam("pt", search.getGeoDistance().getLocation().toString());
            query.setParam("sfield", getFieldname(descriptor, UseCase.Facet, searchContext));
        }
    }
    Collection<FieldDescriptor<?>> fulltext = factory.listFields().stream().filter(FieldDescriptor::isFullText).collect(Collectors.toList());
    if (!fulltext.isEmpty()) {
        query.setParam(DisMaxParams.QF, SolrUtils.Query.buildQueryFieldString(fulltext, searchContext));
        query.setParam("defType", "edismax");
    } else {
        query.setParam(CommonParams.DF, TEXT);
    }
    // filters
    query.add(CommonParams.FQ, "_type_:" + factory.getType());
    if (search.hasFilter()) {
        SolrUtils.Query.buildFilterString(search.getFilter(), factory, search.getChildrenFactory(), query, searchContext, search.getStrict());
    }
    // fulltext search deep search
    if (search.isChildrenSearchEnabled()) {
        // append childCount facet
        search.facet(new Facet.SubdocumentFacet(factory));
        // TODO: move to SolrUtils
        final String parentSearchQuery = "((" + query.get(CommonParams.Q) + ") AND " + TYPE + ":" + factory.getType() + ")";
        final String childrenSearchQuery = "_query_:\"{!parent which=" + TYPE + ":" + factory.getType() + "}(" + TYPE + ":" + search.getChildrenFactory().getType() + " AND (" + search.getChildrenSearchString().getEscapedSearchString() + "))\"";
        query.set(CommonParams.Q, String.join(" ", parentSearchQuery, search.getChildrenSearchOperator().name(), childrenSearchQuery));
        if (search.getChildrenSearchString().hasFilter()) {
            // TODO clean up!
            final String parentFilterQuery = "(" + String.join(" AND ", query.getFilterQueries()) + ")";
            final String childrenFilterQuery = search.getChildrenSearchString().getFilter().accept(new SolrChildrenSerializerVisitor(factory, search.getChildrenFactory(), searchContext, search.getStrict()));
            query.set(CommonParams.FQ, String.join(" ", parentFilterQuery, search.getChildrenSearchOperator().name(), "(" + childrenFilterQuery + ")"));
        }
    }
    if (search.hasFacet()) {
        query.setFacet(true);
        query.setFacetMinCount(search.getFacetMinCount());
        query.setFacetLimit(search.getFacetLimit());
        // Query facets
        search.getFacets().values().stream().filter(facet -> Facet.QueryFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.QueryFacet) genericFacet).forEach(queryFacet -> query.addFacetQuery(StringUtils.join(SolrUtils.Query.buildSolrFacetCustomName(SolrUtils.Query.buildFilterString(queryFacet.getFilter(), factory, search.getChildrenFactory(), searchContext, search.getStrict()), queryFacet))));
        // Numeric Range facet
        search.getFacets().values().stream().filter(facet -> Facet.NumericRangeFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.NumericRangeFacet) genericFacet).forEach(numericRangeFacet -> {
            final UseCase useCase = UseCase.valueOf(numericRangeFacet.getScope().name());
            final String fieldName = getFieldname(numericRangeFacet.getFieldDescriptor(), useCase, searchContext);
            query.add(FacetParams.FACET_RANGE, SolrUtils.Query.buildSolrFacetCustomName(fieldName, numericRangeFacet));
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_START), numericRangeFacet.getStart().toString());
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_END), numericRangeFacet.getEnd().toString());
            query.add(String.format(Locale.ROOT, "f.%s.%s", fieldName, FacetParams.FACET_RANGE_GAP), numericRangeFacet.getGap().toString());
        /*query.addNumericRangeFacet(
                                    SolrUtils.Query.buildSolrFacetCustomName(fieldName, numericRangeFacet.getName()),
                                    numericRangeFacet.getStart(),
                                    numericRangeFacet.getEnd(),
                                    numericRangeFacet.getGap());*/
        });
        // Interval Range facet
        search.getFacets().values().stream().filter(facet -> Facet.IntervalFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.IntervalFacet) genericFacet).forEach(intervalFacet -> {
            final UseCase useCase = UseCase.valueOf(intervalFacet.getScope().name());
            final String fieldName = getFieldname(intervalFacet.getFieldDescriptor(), useCase, searchContext);
            query.add(FacetParams.FACET_INTERVAL, SolrUtils.Query.buildSolrFacetKey(intervalFacet.getName()) + fieldName);
            for (Object o : intervalFacet.getIntervals()) {
                // TODO why is this necessary?
                Interval i = (Interval) o;
                query.add(String.format("f.%s.%s", fieldName, FacetParams.FACET_INTERVAL_SET), String.format("%s%s%s,%s%s", SolrUtils.Query.buildSolrFacetKey(i.getName()), i.includesStart() ? "[" : "(", i.getStart() == null ? SOLR_WILDCARD : SolrUtils.Query.buildSolrQueryValue(i.getStart()), i.getEnd() == null ? SOLR_WILDCARD : SolrUtils.Query.buildSolrQueryValue(i.getEnd()), i.includesEnd() ? "]" : ")"));
            }
        });
        // Date Range facet
        search.getFacets().values().stream().filter(facet -> Facet.DateRangeFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.DateRangeFacet) genericFacet).forEach(dateRangeFacet -> generateDateRangeQuery(dateRangeFacet, query, searchContext));
        // stats
        search.getFacets().values().stream().filter(facet -> Facet.StatsFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.StatsFacet) genericFacet).forEach(statsFacet -> {
            final UseCase useCase = UseCase.valueOf(statsFacet.getScope().name());
            String fieldName = getFieldname(statsFacet.getField(), useCase, searchContext);
            query.add(StatsParams.STATS, "true");
            query.add(StatsParams.STATS_FIELD, SolrUtils.Query.buildSolrStatsQuery(fieldName, statsFacet));
        });
        // pivot facet
        search.getFacets().values().stream().filter(facet -> Facet.PivotFacet.class.isAssignableFrom(facet.getClass())).map(genericFacet -> (Facet.PivotFacet) genericFacet).forEach(pivotFacet -> {
            String[] fieldNames = pivotFacet.getFieldDescriptors().stream().map(fieldDescriptor -> getFieldname(fieldDescriptor, UseCase.Facet, searchContext)).toArray(String[]::new);
            query.add(FacetParams.FACET_PIVOT, SolrUtils.Query.buildSolrPivotSubFacetName(pivotFacet.getName(), fieldNames));
        });
        // facet fields
        final HashMap<String, Object> strings = SolrUtils.Query.buildJsonTermFacet(search.getFacets(), search.getFacetLimit(), factory, search.getChildrenFactory(), searchContext);
        query.add("json.facet", strings.toString().replaceAll("=", ":"));
        // facet Subdocument count
        final String subdocumentFacetString = SolrUtils.Query.buildSubdocumentFacet(search, factory, searchContext);
        if (Objects.nonNull(subdocumentFacetString)) {
            query.add("json.facet", subdocumentFacetString);
        }
    }
    // sorting
    if (search.hasSorting()) {
        final String sortString = SolrUtils.Query.buildSortString(search, search.getSorting(), factory);
        query.set(CommonParams.SORT, sortString);
    }
    // TODO this is a mess
    if (search.hasSorting()) {
        query.set(DisMaxParams.BF, SolrUtils.Query.buildBoostFunction(search.getSorting(), searchContext));
    }
    // paging
    switch(search.getResultSet().getType()) {
        case page:
            {
                final Page resultSet = (Page) search.getResultSet();
                query.setStart(resultSet.getOffset());
                query.setRows(resultSet.getPagesize());
                break;
            }
        case slice:
            {
                final Slice resultSet = (Slice) search.getResultSet();
                query.setStart(resultSet.getOffset());
                query.setRows(resultSet.getSliceSize());
                break;
            }
    }
    return query;
}
Also used : Delete(com.rbmhtechnology.vind.api.query.delete.Delete) SearchServerException(com.rbmhtechnology.vind.SearchServerException) URISyntaxException(java.net.URISyntaxException) ZonedDateTime(java.time.ZonedDateTime) LoggerFactory(org.slf4j.LoggerFactory) Asserts(org.apache.http.util.Asserts) StringUtils(org.apache.commons.lang3.StringUtils) SolrServerException(org.apache.solr.client.solrj.SolrServerException) Fieldname(com.rbmhtechnology.vind.solr.backend.SolrUtils.Fieldname) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) UpdateOperation(com.rbmhtechnology.vind.api.query.update.UpdateOperation) ParseException(java.text.ParseException) FileSystemUtils(com.rbmhtechnology.vind.utils.FileSystemUtils) Path(java.nio.file.Path) SearchServer(com.rbmhtechnology.vind.api.SearchServer) Slice(com.rbmhtechnology.vind.api.query.division.Slice) AnnotationUtil(com.rbmhtechnology.vind.annotations.AnnotationUtil) SchemaResponse(org.apache.solr.client.solrj.response.schema.SchemaResponse) Update(com.rbmhtechnology.vind.api.query.update.Update) SchemaRequest(org.apache.solr.client.solrj.request.schema.SchemaRequest) DescriptorSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.DescriptorSuggestionSearch) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) com.rbmhtechnology.vind.api.query(com.rbmhtechnology.vind.api.query) RealTimeGet(com.rbmhtechnology.vind.api.query.get.RealTimeGet) Stream(java.util.stream.Stream) HttpSolrClient(org.apache.solr.client.solrj.impl.HttpSolrClient) SolrPingResponse(org.apache.solr.client.solrj.response.SolrPingResponse) SolrQuery(org.apache.solr.client.solrj.SolrQuery) com.rbmhtechnology.vind.api.result(com.rbmhtechnology.vind.api.result) ClientUtils(org.apache.solr.client.solrj.util.ClientUtils) ServiceProvider(com.rbmhtechnology.vind.api.ServiceProvider) SolrInputDocument(org.apache.solr.common.SolrInputDocument) Interval(com.rbmhtechnology.vind.api.query.facet.Interval) SolrRequest(org.apache.solr.client.solrj.SolrRequest) java.util(java.util) ExecutableSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.ExecutableSuggestionSearch) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) NumberFormat(java.text.NumberFormat) UpdateOperations.set(com.rbmhtechnology.vind.api.query.update.Update.UpdateOperations.set) CollectionUtils(org.apache.commons.collections.CollectionUtils) SearchConfiguration(com.rbmhtechnology.vind.configure.SearchConfiguration) Logger(org.slf4j.Logger) StringSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.StringSuggestionSearch) LatLng(com.rbmhtechnology.vind.model.value.LatLng) Resources(com.google.common.io.Resources) DateUtil(org.apache.solr.common.util.DateUtil) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) org.apache.solr.common.params(org.apache.solr.common.params) IOException(java.io.IOException) NamedList(org.apache.solr.common.util.NamedList) Document(com.rbmhtechnology.vind.api.Document) SolrClient(org.apache.solr.client.solrj.SolrClient) SolrDocument(org.apache.solr.common.SolrDocument) DocumentFactory(com.rbmhtechnology.vind.model.DocumentFactory) Page(com.rbmhtechnology.vind.api.query.division.Page) UpdateOperations(com.rbmhtechnology.vind.api.query.update.Update.UpdateOperations) Page(com.rbmhtechnology.vind.api.query.division.Page) SolrQuery(org.apache.solr.client.solrj.SolrQuery) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) Slice(com.rbmhtechnology.vind.api.query.division.Slice) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) Interval(com.rbmhtechnology.vind.api.query.facet.Interval)

Example 24 with StringUtils.join

use of org.apache.commons.lang3.StringUtils.join in project nifi-registry by apache.

the class LdapUserGroupProvider method load.

/**
 * Reloads the tenants.
 */
private void load(final ContextSource contextSource) {
    // create the ldapTemplate based on the context source. use a single source context to use the same connection
    // to support paging when configured
    final SingleContextSource singleContextSource = new SingleContextSource(contextSource.getReadOnlyContext());
    final LdapTemplate ldapTemplate = new LdapTemplate(singleContextSource);
    try {
        final List<User> userList = new ArrayList<>();
        final List<Group> groupList = new ArrayList<>();
        // group dn -> user identifiers lookup
        final Map<String, Set<String>> groupToUserIdentifierMappings = new HashMap<>();
        // user dn -> user lookup
        final Map<String, User> userLookup = new HashMap<>();
        if (performUserSearch) {
            // search controls
            final SearchControls userControls = new SearchControls();
            userControls.setSearchScope(userSearchScope.ordinal());
            // consider paging support for users
            final DirContextProcessor userProcessor;
            if (pageSize == null) {
                userProcessor = new NullDirContextProcessor();
            } else {
                userProcessor = new PagedResultsDirContextProcessor(pageSize);
            }
            // looking for objects matching the user object class
            final AndFilter userFilter = new AndFilter();
            userFilter.and(new EqualsFilter("objectClass", userObjectClass));
            // if a filter has been provided by the user, we add it to the filter
            if (StringUtils.isNotBlank(userSearchFilter)) {
                userFilter.and(new HardcodedFilter(userSearchFilter));
            }
            do {
                userList.addAll(ldapTemplate.search(userSearchBase, userFilter.encode(), userControls, new AbstractContextMapper<User>() {

                    @Override
                    protected User doMapFromContext(DirContextOperations ctx) {
                        // get the user identity
                        final String identity = getUserIdentity(ctx);
                        // build the user
                        final User user = new User.Builder().identifierGenerateFromSeed(identity).identity(identity).build();
                        // store the user for group member later
                        userLookup.put(getReferencedUserValue(ctx), user);
                        if (StringUtils.isNotBlank(userGroupNameAttribute)) {
                            final Attribute attributeGroups = ctx.getAttributes().get(userGroupNameAttribute);
                            if (attributeGroups == null) {
                                logger.warn("User group name attribute [" + userGroupNameAttribute + "] does not exist. Ignoring group membership.");
                            } else {
                                try {
                                    final NamingEnumeration<String> groupValues = (NamingEnumeration<String>) attributeGroups.getAll();
                                    while (groupValues.hasMoreElements()) {
                                        // store the group -> user identifier mapping
                                        groupToUserIdentifierMappings.computeIfAbsent(groupValues.next(), g -> new HashSet<>()).add(user.getIdentifier());
                                    }
                                } catch (NamingException e) {
                                    throw new AuthorizationAccessException("Error while retrieving user group name attribute [" + userIdentityAttribute + "].");
                                }
                            }
                        }
                        return user;
                    }
                }, userProcessor));
            } while (hasMorePages(userProcessor));
        }
        if (performGroupSearch) {
            final SearchControls groupControls = new SearchControls();
            groupControls.setSearchScope(groupSearchScope.ordinal());
            // consider paging support for groups
            final DirContextProcessor groupProcessor;
            if (pageSize == null) {
                groupProcessor = new NullDirContextProcessor();
            } else {
                groupProcessor = new PagedResultsDirContextProcessor(pageSize);
            }
            // looking for objects matching the group object class
            AndFilter groupFilter = new AndFilter();
            groupFilter.and(new EqualsFilter("objectClass", groupObjectClass));
            // if a filter has been provided by the user, we add it to the filter
            if (StringUtils.isNotBlank(groupSearchFilter)) {
                groupFilter.and(new HardcodedFilter(groupSearchFilter));
            }
            do {
                groupList.addAll(ldapTemplate.search(groupSearchBase, groupFilter.encode(), groupControls, new AbstractContextMapper<Group>() {

                    @Override
                    protected Group doMapFromContext(DirContextOperations ctx) {
                        final String dn = ctx.getDn().toString();
                        // get the group identity
                        final String name = getGroupName(ctx);
                        // get the value of this group that may associate it to users
                        final String referencedGroupValue = getReferencedGroupValue(ctx);
                        if (!StringUtils.isBlank(groupMemberAttribute)) {
                            Attribute attributeUsers = ctx.getAttributes().get(groupMemberAttribute);
                            if (attributeUsers == null) {
                                logger.warn("Group member attribute [" + groupMemberAttribute + "] does not exist. Ignoring group membership.");
                            } else {
                                try {
                                    final NamingEnumeration<String> userValues = (NamingEnumeration<String>) attributeUsers.getAll();
                                    while (userValues.hasMoreElements()) {
                                        final String userValue = userValues.next();
                                        if (performUserSearch) {
                                            // find the user by it's referenced attribute and add the identifier to this group
                                            final User user = userLookup.get(userValue);
                                            // ensure the user is known
                                            if (user != null) {
                                                groupToUserIdentifierMappings.computeIfAbsent(referencedGroupValue, g -> new HashSet<>()).add(user.getIdentifier());
                                            } else {
                                                logger.warn(String.format("%s contains member %s but that user was not found while searching users. Ignoring group membership.", name, userValue));
                                            }
                                        } else {
                                            // since performUserSearch is false, then the referenced group attribute must be blank... the user value must be the dn
                                            final String userDn = userValue;
                                            final String userIdentity;
                                            if (useDnForUserIdentity) {
                                                // use the user value to avoid the unnecessary look up
                                                userIdentity = userDn;
                                            } else {
                                                // lookup the user to extract the user identity
                                                userIdentity = getUserIdentity((DirContextAdapter) ldapTemplate.lookup(userDn));
                                            }
                                            // build the user
                                            final User user = new User.Builder().identifierGenerateFromSeed(userIdentity).identity(userIdentity).build();
                                            // add this user
                                            userList.add(user);
                                            groupToUserIdentifierMappings.computeIfAbsent(referencedGroupValue, g -> new HashSet<>()).add(user.getIdentifier());
                                        }
                                    }
                                } catch (NamingException e) {
                                    throw new AuthorizationAccessException("Error while retrieving group name attribute [" + groupNameAttribute + "].");
                                }
                            }
                        }
                        // build this group
                        final Group.Builder groupBuilder = new Group.Builder().identifierGenerateFromSeed(name).name(name);
                        // add all users that were associated with this referenced group attribute
                        if (groupToUserIdentifierMappings.containsKey(referencedGroupValue)) {
                            groupToUserIdentifierMappings.remove(referencedGroupValue).forEach(userIdentifier -> groupBuilder.addUser(userIdentifier));
                        }
                        return groupBuilder.build();
                    }
                }, groupProcessor));
            } while (hasMorePages(groupProcessor));
            // any remaining groupDn's were referenced by a user but not found while searching groups
            groupToUserIdentifierMappings.forEach((referencedGroupValue, userIdentifiers) -> {
                logger.warn(String.format("[%s] are members of %s but that group was not found while searching users. Ignoring group membership.", StringUtils.join(userIdentifiers, ", "), referencedGroupValue));
            });
        } else {
            // since performGroupSearch is false, then the referenced user attribute must be blank... the group value must be the dn
            // groups are not being searched so lookup any groups identified while searching users
            groupToUserIdentifierMappings.forEach((groupDn, userIdentifiers) -> {
                final String groupName;
                if (useDnForGroupName) {
                    // use the dn to avoid the unnecessary look up
                    groupName = groupDn;
                } else {
                    groupName = getGroupName((DirContextAdapter) ldapTemplate.lookup(groupDn));
                }
                // define the group
                final Group.Builder groupBuilder = new Group.Builder().identifierGenerateFromSeed(groupName).name(groupName);
                // add each user
                userIdentifiers.forEach(userIdentifier -> groupBuilder.addUser(userIdentifier));
                // build the group
                groupList.add(groupBuilder.build());
            });
        }
        // record the updated tenants
        tenants.set(new TenantHolder(new HashSet<>(userList), new HashSet<>(groupList)));
    } finally {
        singleContextSource.destroy();
    }
}
Also used : SSLContext(javax.net.ssl.SSLContext) AbstractContextMapper(org.springframework.ldap.core.support.AbstractContextMapper) AndFilter(org.springframework.ldap.filter.AndFilter) ClientAuth(org.apache.nifi.registry.security.util.SslContextFactory.ClientAuth) LoggerFactory(org.slf4j.LoggerFactory) LdapTemplate(org.springframework.ldap.core.LdapTemplate) NamingException(javax.naming.NamingException) KeyStoreException(java.security.KeyStoreException) StringUtils(org.apache.commons.lang3.StringUtils) Attribute(javax.naming.directory.Attribute) Map(java.util.Map) DirContextAdapter(org.springframework.ldap.core.DirContextAdapter) ThreadFactory(java.util.concurrent.ThreadFactory) LdapsSocketFactory(org.apache.nifi.registry.security.ldap.LdapsSocketFactory) LdapContextSource(org.springframework.ldap.core.support.LdapContextSource) AuthorizerContext(org.apache.nifi.registry.security.authorization.annotation.AuthorizerContext) UserGroupProvider(org.apache.nifi.registry.security.authorization.UserGroupProvider) Set(java.util.Set) KeyManagementException(java.security.KeyManagementException) HardcodedFilter(org.springframework.ldap.filter.HardcodedFilter) LdapAuthenticationStrategy(org.apache.nifi.registry.security.ldap.LdapAuthenticationStrategy) SslContextFactory(org.apache.nifi.registry.security.util.SslContextFactory) Executors(java.util.concurrent.Executors) List(java.util.List) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) NamingEnumeration(javax.naming.NamingEnumeration) SingleContextSource(org.springframework.ldap.core.support.SingleContextSource) IdentityMapping(org.apache.nifi.registry.properties.util.IdentityMapping) DirContextOperations(org.springframework.ldap.core.DirContextOperations) PagedResultsDirContextProcessor(org.springframework.ldap.control.PagedResultsDirContextProcessor) ReferralStrategy(org.apache.nifi.registry.security.ldap.ReferralStrategy) NullDirContextProcessor(org.springframework.ldap.core.LdapTemplate.NullDirContextProcessor) DirContextProcessor(org.springframework.ldap.core.DirContextProcessor) IdentityMappingUtil(org.apache.nifi.registry.properties.util.IdentityMappingUtil) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) SearchControls(javax.naming.directory.SearchControls) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Group(org.apache.nifi.registry.security.authorization.Group) SimpleDirContextAuthenticationStrategy(org.springframework.ldap.core.support.SimpleDirContextAuthenticationStrategy) UserGroupProviderInitializationContext(org.apache.nifi.registry.security.authorization.UserGroupProviderInitializationContext) NiFiRegistryProperties(org.apache.nifi.registry.properties.NiFiRegistryProperties) UserAndGroups(org.apache.nifi.registry.security.authorization.UserAndGroups) UnrecoverableKeyException(java.security.UnrecoverableKeyException) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AbstractTlsDirContextAuthenticationStrategy(org.springframework.ldap.core.support.AbstractTlsDirContextAuthenticationStrategy) SecurityProviderDestructionException(org.apache.nifi.registry.security.exception.SecurityProviderDestructionException) DefaultTlsDirContextAuthenticationStrategy(org.springframework.ldap.core.support.DefaultTlsDirContextAuthenticationStrategy) Context(javax.naming.Context) AuthorizationAccessException(org.apache.nifi.registry.security.authorization.exception.AuthorizationAccessException) Logger(org.slf4j.Logger) ContextSource(org.springframework.ldap.core.ContextSource) SecurityProviderCreationException(org.apache.nifi.registry.security.exception.SecurityProviderCreationException) User(org.apache.nifi.registry.security.authorization.User) IOException(java.io.IOException) CertificateException(java.security.cert.CertificateException) TimeUnit(java.util.concurrent.TimeUnit) EqualsFilter(org.springframework.ldap.filter.EqualsFilter) AuthorizerConfigurationContext(org.apache.nifi.registry.security.authorization.AuthorizerConfigurationContext) PropertyValue(org.apache.nifi.registry.util.PropertyValue) Collections(java.util.Collections) FormatUtils(org.apache.nifi.registry.util.FormatUtils) Group(org.apache.nifi.registry.security.authorization.Group) User(org.apache.nifi.registry.security.authorization.User) Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) Attribute(javax.naming.directory.Attribute) ArrayList(java.util.ArrayList) NamingEnumeration(javax.naming.NamingEnumeration) PagedResultsDirContextProcessor(org.springframework.ldap.control.PagedResultsDirContextProcessor) NullDirContextProcessor(org.springframework.ldap.core.LdapTemplate.NullDirContextProcessor) DirContextProcessor(org.springframework.ldap.core.DirContextProcessor) PagedResultsDirContextProcessor(org.springframework.ldap.control.PagedResultsDirContextProcessor) LdapTemplate(org.springframework.ldap.core.LdapTemplate) AuthorizationAccessException(org.apache.nifi.registry.security.authorization.exception.AuthorizationAccessException) DirContextAdapter(org.springframework.ldap.core.DirContextAdapter) SearchControls(javax.naming.directory.SearchControls) NamingException(javax.naming.NamingException) EqualsFilter(org.springframework.ldap.filter.EqualsFilter) HashSet(java.util.HashSet) SingleContextSource(org.springframework.ldap.core.support.SingleContextSource) NullDirContextProcessor(org.springframework.ldap.core.LdapTemplate.NullDirContextProcessor) HardcodedFilter(org.springframework.ldap.filter.HardcodedFilter) AndFilter(org.springframework.ldap.filter.AndFilter) AbstractContextMapper(org.springframework.ldap.core.support.AbstractContextMapper) DirContextOperations(org.springframework.ldap.core.DirContextOperations)

Example 25 with StringUtils.join

use of org.apache.commons.lang3.StringUtils.join in project DataX by alibaba.

the class HdfsHelper method transportOneRecord.

public static MutablePair<Text, Boolean> transportOneRecord(Record record, char fieldDelimiter, List<Configuration> columnsConfiguration, TaskPluginCollector taskPluginCollector) {
    MutablePair<List<Object>, Boolean> transportResultList = transportOneRecord(record, columnsConfiguration, taskPluginCollector);
    //保存<转换后的数据,是否是脏数据>
    MutablePair<Text, Boolean> transportResult = new MutablePair<Text, Boolean>();
    transportResult.setRight(false);
    if (null != transportResultList) {
        Text recordResult = new Text(StringUtils.join(transportResultList.getLeft(), fieldDelimiter));
        transportResult.setRight(transportResultList.getRight());
        transportResult.setLeft(recordResult);
    }
    return transportResult;
}
Also used : MutablePair(org.apache.commons.lang3.tuple.MutablePair) Text(org.apache.hadoop.io.Text)

Aggregations

StringUtils (org.apache.commons.lang3.StringUtils)34 List (java.util.List)30 Collectors (java.util.stream.Collectors)23 ArrayList (java.util.ArrayList)21 Map (java.util.Map)17 HashMap (java.util.HashMap)15 Set (java.util.Set)14 Logger (org.slf4j.Logger)14 LoggerFactory (org.slf4j.LoggerFactory)14 IOException (java.io.IOException)13 HashSet (java.util.HashSet)11 Arrays (java.util.Arrays)10 Collections (java.util.Collections)10 Date (java.util.Date)9 File (java.io.File)6 StopWatch (org.apache.commons.lang3.time.StopWatch)6 InputStream (java.io.InputStream)5 java.util (java.util)5 Pair (org.apache.commons.lang3.tuple.Pair)5 Path (java.nio.file.Path)4