Examples with PDStorageManager com.helger.pd.indexer.storage.PDStorageManager used on opensource projects

Example 1 with PDStorageManager

use of com.helger.pd.indexer.storage.PDStorageManager in project phoss-directory by phax.

the class PDMetaManager method onAfterInstantiation.

@Override
protected void onAfterInstantiation(@Nonnull final IScope aScope) {
    try {
        m_aLucene = new PDLucene();
        m_aStorageMgr = new PDStorageManager(m_aLucene);
        m_aIndexerMgr = new PDIndexerManager(m_aStorageMgr);
        LOGGER.info(ClassHelper.getClassLocalName(this) + " was initialized");
    } catch (final Exception ex) {
        if (GlobalDebug.isProductionMode()) {
            new InternalErrorBuilder().setThrowable(ex).addErrorMessage(ClassHelper.getClassLocalName(this) + " init failed").handle();
        }
        throw new InitializationException("Failed to init " + ClassHelper.getClassLocalName(this), ex);
    }
}

Example 2 with PDStorageManager

use of com.helger.pd.indexer.storage.PDStorageManager in project phoss-directory by phax.

the class PublicSearchXServletHandler method handleRequest.

public void handleRequest(@Nonnull final IRequestWebScopeWithoutResponse aRequestScope, @Nonnull final UnifiedResponse aUnifiedResponse) throws Exception {
    if (SearchRateLimit.INSTANCE.rateLimiter() != null) {
        final String sRateLimitKey = "ip:" + aRequestScope.getRemoteAddr();
        final boolean bOverLimit = SearchRateLimit.INSTANCE.rateLimiter().overLimitWhenIncremented(sRateLimitKey);
        if (bOverLimit) {
            // Too Many Requests
            if (LOGGER.isDebugEnabled())
                LOGGER.debug("REST search rate limit exceeded for " + sRateLimitKey);
            aUnifiedResponse.setStatus(CHttp.HTTP_TOO_MANY_REQUESTS);
            return;
        }
    }
    final IRequestParamContainer aParams = aRequestScope.params();
    // http://127.0.0.1:8080/search -> null
    // http://127.0.0.1:8080/search/ -> "/"
    // http://127.0.0.1:8080/search/x -> "/x"
    final String sPathInfo = StringHelper.getNotNull(aRequestScope.getPathInfo(), "");
    final ESearchVersion eSearchVersion = ESearchVersion.getFromPathInfoOrNull(sPathInfo);
    if (eSearchVersion == ESearchVersion.V1) {
        // Version 1.0
        // Determine output format
        final ICommonsList<String> aParts = StringHelper.getExploded('/', sPathInfo.substring(1));
        final String sFormat = aParts.getAtIndex(1);
        final EPDOutputFormat eOutputFormat = EPDOutputFormat.getFromIDCaseInsensitiveOrDefault(sFormat, EPDOutputFormat.XML);
        if (LOGGER.isDebugEnabled())
            LOGGER.debug("Using REST query API 1.0 with output format " + eOutputFormat + " (" + sPathInfo + ") from '" + aRequestScope.getUserAgent().getAsString() + "'");
        // Determine result offset and count
        final int nResultPageIndex = aParams.getAsInt(PARAM_RESULT_PAGE_INDEX, aParams.getAsInt("rpi", DEFAULT_RESULT_PAGE_INDEX));
        if (nResultPageIndex < 0) {
            LOGGER.error("ResultPageIndex " + nResultPageIndex + " is invalid. It must be >= 0.");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        final int nResultPageCount = aParams.getAsInt(PARAM_RESULT_PAGE_COUNT, aParams.getAsInt("rpc", DEFAULT_RESULT_PAGE_COUNT));
        if (nResultPageCount <= 0) {
            LOGGER.error("ResultPageCount " + nResultPageCount + " is invalid. It must be > 0.");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        final int nFirstResultIndex = nResultPageIndex * nResultPageCount;
        final int nLastResultIndex = (nResultPageIndex + 1) * nResultPageCount - 1;
        if (nFirstResultIndex > MAX_RESULTS) {
            LOGGER.error("The first result index " + nFirstResultIndex + " is invalid. It must be <= " + MAX_RESULTS + ".");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        if (nLastResultIndex > MAX_RESULTS) {
            LOGGER.error("The last result index " + nLastResultIndex + " is invalid. It must be <= " + MAX_RESULTS + ".");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        // Format output?
        final boolean bBeautify = aParams.getAsBoolean(PARAM_BEAUTIFY, false);
        // Determine query terms
        final StringBuilder aSBQueryString = new StringBuilder();
        final ICommonsMap<EPDSearchField, ICommonsList<String>> aQueryValues = new CommonsHashMap<>();
        for (final EPDSearchField eSF : EPDSearchField.values()) {
            final String sFieldName = eSF.getFieldName();
            // Check if one or more request parameters are present for the current
            // search field
            final ICommonsList<String> aValues = aParams.getAsStringList(sFieldName);
            if (aValues != null && aValues.isNotEmpty()) {
                aQueryValues.put(eSF, aValues);
                for (final String sValue : aValues) {
                    if (aSBQueryString.length() > 0)
                        aSBQueryString.append('&');
                    aSBQueryString.append(sFieldName).append('=').append(sValue);
                }
            }
        }
        if (aQueryValues.isEmpty()) {
            LOGGER.error("No valid query term provided!");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        if (LOGGER.isDebugEnabled())
            LOGGER.debug("Using the following query terms: " + aQueryValues);
        final ICommonsList<Query> aQueries = new CommonsArrayList<>();
        for (final Map.Entry<EPDSearchField, ICommonsList<String>> aEntry : aQueryValues.entrySet()) {
            final EPDSearchField eField = aEntry.getKey();
            for (final String sQuery : aEntry.getValue()) {
                final Query aQuery = eField.getQuery(sQuery);
                if (aQuery != null)
                    aQueries.add(aQuery);
                else
                    LOGGER.error("Failed to create query '" + sQuery + "' of field " + eField + " - ignoring term!");
            }
        }
        if (aQueries.isEmpty()) {
            LOGGER.error("No valid queries could be created!");
            aUnifiedResponse.setStatus(CHttp.HTTP_BAD_REQUEST);
            return;
        }
        // Build final query term
        Query aLuceneQuery;
        if (aQueries.size() == 1) {
            aLuceneQuery = aQueries.getFirst();
        } else {
            // Connect all with "AND"
            final BooleanQuery.Builder aBuilder = new BooleanQuery.Builder();
            for (final Query aQuery : aQueries) aBuilder.add(aQuery, Occur.MUST);
            aLuceneQuery = aBuilder.build();
        }
        // Only-non deleted
        aLuceneQuery = PDQueryManager.andNotDeleted(aLuceneQuery);
        // How many results to deliver at most
        final int nMaxResults = nLastResultIndex + 1;
        // Search all documents
        final PDStorageManager aStorageMgr = PDMetaManager.getStorageMgr();
        final ICommonsList<PDStoredBusinessEntity> aResultDocs = aStorageMgr.getAllDocuments(aLuceneQuery, nMaxResults);
        // Also get the total hit count for UI display. May be < 0 in case of
        // error
        final int nTotalBEs = aStorageMgr.getCount(aLuceneQuery);
        if (LOGGER.isDebugEnabled())
            LOGGER.debug("  Result for <" + aLuceneQuery + "> (max=" + nMaxResults + ") " + (nTotalBEs == 1 ? "is 1 document" : "are " + nTotalBEs + " documents"));
        // Filter by index/count
        final int nEffectiveLastIndex = Math.min(nLastResultIndex, aResultDocs.size() - 1);
        final List<PDStoredBusinessEntity> aResultView = nFirstResultIndex >= aResultDocs.size() ? Collections.emptyList() : aResultDocs.subList(nFirstResultIndex, nEffectiveLastIndex + 1);
        // Group results by participant ID
        final ICommonsMap<IParticipantIdentifier, ICommonsList<PDStoredBusinessEntity>> aGroupedDocs = PDStorageManager.getGroupedByParticipantID(aResultView);
        final ZonedDateTime aNow = PDTFactory.getCurrentZonedDateTimeUTC();
        // build result
        switch(eOutputFormat) {
            case XML:
                {
                    final XMLWriterSettings aXWS = new XMLWriterSettings().setIndent(bBeautify ? EXMLSerializeIndent.INDENT_AND_ALIGN : EXMLSerializeIndent.NONE);
                    final IMicroDocument aDoc = new MicroDocument();
                    final IMicroElement eRoot = aDoc.appendElement("resultlist");
                    eRoot.setAttribute(RESPONSE_VERSION, eSearchVersion.getVersion());
                    eRoot.setAttribute(RESPONSE_TOTAL_RESULT_COUNT, nTotalBEs);
                    eRoot.setAttribute(RESPONSE_USED_RESULT_COUNT, aResultView.size());
                    eRoot.setAttribute(RESPONSE_RESULT_PAGE_INDEX, nResultPageIndex);
                    eRoot.setAttribute(RESPONSE_RESULT_PAGE_COUNT, nResultPageCount);
                    eRoot.setAttribute(RESPONSE_FIRST_RESULT_INDEX, nFirstResultIndex);
                    eRoot.setAttribute(RESPONSE_LAST_RESULT_INDEX, nEffectiveLastIndex);
                    eRoot.setAttribute(RESPONSE_QUERY_TERMS, aSBQueryString.toString());
                    eRoot.setAttribute(RESPONSE_CREATION_DT, PDTWebDateHelper.getAsStringXSD(aNow));
                    for (final ICommonsList<PDStoredBusinessEntity> aPerParticipant : aGroupedDocs.values()) {
                        final IMicroElement eItem = PDStoredBusinessEntity.getAsSearchResultMicroElement(aPerParticipant);
                        eRoot.appendChild(eItem);
                    }
                    if (false) {
                        // Demo validation
                        final CollectingSAXErrorHandler aErrHdl = new CollectingSAXErrorHandler();
                        final Validator v = new XMLSchemaCache(aErrHdl).getValidator(new ClassPathResource("/schema/directory-search-result-list-v1.xsd"));
                        v.validate(TransformSourceFactory.create(MicroWriter.getNodeAsBytes(aDoc, aXWS)));
                        for (final IError aError : aErrHdl.getErrorList()) LOGGER.error(aError.getAsString(AppCommonUI.DEFAULT_LOCALE));
                    }
                    aUnifiedResponse.disableCaching();
                    aUnifiedResponse.setMimeType(eOutputFormat.getMimeType());
                    aUnifiedResponse.setContent(MicroWriter.getNodeAsBytes(aDoc, aXWS));
                    break;
                }
            case JSON:
                final JsonWriterSettings aJWS = new JsonWriterSettings().setIndentEnabled(bBeautify);
                final IJsonObject aDoc = new JsonObject();
                aDoc.add(RESPONSE_VERSION, eSearchVersion.getVersion());
                aDoc.add(RESPONSE_TOTAL_RESULT_COUNT, nTotalBEs);
                aDoc.add(RESPONSE_USED_RESULT_COUNT, aResultView.size());
                aDoc.add(RESPONSE_RESULT_PAGE_INDEX, nResultPageIndex);
                aDoc.add(RESPONSE_RESULT_PAGE_COUNT, nResultPageCount);
                aDoc.add(RESPONSE_FIRST_RESULT_INDEX, nFirstResultIndex);
                aDoc.add(RESPONSE_LAST_RESULT_INDEX, nEffectiveLastIndex);
                aDoc.add(RESPONSE_QUERY_TERMS, aSBQueryString.toString());
                aDoc.add(RESPONSE_CREATION_DT, PDTWebDateHelper.getAsStringXSD(aNow));
                final IJsonArray aMatches = new JsonArray();
                for (final ICommonsList<PDStoredBusinessEntity> aPerParticipant : aGroupedDocs.values()) {
                    final IJsonObject aItem = PDStoredBusinessEntity.getAsSearchResultJsonObject(aPerParticipant);
                    aMatches.add(aItem);
                }
                aDoc.addJson("matches", aMatches);
                aUnifiedResponse.disableCaching();
                aUnifiedResponse.setMimeType(eOutputFormat.getMimeType());
                aUnifiedResponse.setContentAndCharset(aDoc.getAsJsonString(aJWS), StandardCharsets.UTF_8);
                break;
            default:
                throw new IllegalStateException("Unsupported output format: " + eOutputFormat);
        }
    } else {
        LOGGER.error("Unsupported version provided (" + sPathInfo + ")");
        aUnifiedResponse.setStatus(CHttp.HTTP_NOT_FOUND);
    }
}

Example 3 with PDStorageManager

use of com.helger.pd.indexer.storage.PDStorageManager in project phoss-directory by phax.

the class PagePublicSearchSimple method _showResultList.

private void _showResultList(@Nonnull final WebPageExecutionContext aWPEC, @Nonnull @Nonempty final String sQuery, @Nonnegative final int nMaxResults) {
    final HCNodeList aNodeList = aWPEC.getNodeList();
    final Locale aDisplayLocale = aWPEC.getDisplayLocale();
    final IRequestWebScopeWithoutResponse aRequestScope = aWPEC.getRequestScope();
    final PDStorageManager aStorageMgr = PDMetaManager.getStorageMgr();
    // Search all documents
    if (LOGGER.isInfoEnabled())
        LOGGER.info("Searching generically for '" + sQuery + "'");
    // Build Lucene query
    Query aLuceneQuery = PDQueryManager.convertQueryStringToLuceneQuery(PDMetaManager.getLucene(), CPDStorage.FIELD_ALL_FIELDS, sQuery);
    aLuceneQuery = EQueryMode.NON_DELETED_ONLY.getEffectiveQuery(aLuceneQuery);
    if (LOGGER.isDebugEnabled())
        LOGGER.debug("Created query for '" + sQuery + "' is <" + aLuceneQuery + ">");
    PDSessionSingleton.getInstance().setLastQuery(aLuceneQuery);
    // Search all documents
    final ICommonsList<PDStoredBusinessEntity> aResultBEs = aStorageMgr.getAllDocuments(aLuceneQuery, nMaxResults);
    // Also get the total hit count for UI display. May be < 0 in case of
    // error
    final int nTotalBEs = aStorageMgr.getCount(aLuceneQuery);
    if (LOGGER.isInfoEnabled())
        LOGGER.info("  Result for <" + aLuceneQuery + "> (max=" + nMaxResults + ") " + (aResultBEs.size() == 1 ? "is 1 document" : "are " + aResultBEs.size() + " documents") + "." + (nTotalBEs >= 0 ? " " + nTotalBEs + " total hits are available." : ""));
    // Group by participant ID
    final ICommonsMap<IParticipantIdentifier, ICommonsList<PDStoredBusinessEntity>> aGroupedBEs = PDStorageManager.getGroupedByParticipantID(aResultBEs);
    // Display results
    if (aGroupedBEs.isEmpty()) {
        aNodeList.addChild(info("No search results found for query '" + sQuery + "'"));
    } else {
        aNodeList.addChild(div(badgeSuccess("Found " + (aGroupedBEs.size() == 1 ? "1 entity" : aGroupedBEs.size() + " entities") + " matching '" + sQuery + "'")));
        if (nTotalBEs > nMaxResults) {
            aNodeList.addChild(div(badgeWarn("Found more entities than displayed (" + nTotalBEs + " entries exist). Try to be more specific.")));
        }
        // Show basic information
        final HCOL aOL = new HCOL().setStart(1);
        for (final Map.Entry<IParticipantIdentifier, ICommonsList<PDStoredBusinessEntity>> aEntry : aGroupedBEs.entrySet()) {
            final IParticipantIdentifier aDocParticipantID = aEntry.getKey();
            final ICommonsList<PDStoredBusinessEntity> aDocs = aEntry.getValue();
            // Start result document
            final HCDiv aResultItem = div().addClass(CSS_CLASS_RESULT_DOC);
            final HCDiv aHeadRow = aResultItem.addAndReturnChild(new HCDiv());
            {
                final boolean bIsPeppolDefault = aDocParticipantID.hasScheme(PEPPOL_DEFAULT_SCHEME);
                IHCNode aParticipantNode = null;
                if (bIsPeppolDefault) {
                    final IParticipantIdentifierScheme aScheme = ParticipantIdentifierSchemeManager.getSchemeOfIdentifier(aDocParticipantID);
                    if (aScheme != null) {
                        aParticipantNode = new HCNodeList().addChild(aDocParticipantID.getValue());
                        if (StringHelper.hasText(aScheme.getSchemeAgency()))
                            ((HCNodeList) aParticipantNode).addChild(" (" + aScheme.getSchemeAgency() + ")");
                    }
                }
                if (aParticipantNode == null) {
                    // Fallback
                    aParticipantNode = code(aDocParticipantID.getURIEncoded());
                }
                aHeadRow.addChild("Participant ID: ").addChild(aParticipantNode);
            }
            if (aDocs.size() > 1)
                aHeadRow.addChild(" (" + aDocs.size() + " entities)");
            // Show all entities of the stored document
            final HCUL aUL = aResultItem.addAndReturnChild(new HCUL());
            for (final PDStoredBusinessEntity aStoredDoc : aEntry.getValue()) {
                final BootstrapTable aTable = new BootstrapTable(HCCol.perc(20), HCCol.star());
                aTable.setCondensed(true);
                if (aStoredDoc.hasCountryCode()) {
                    // Add country flag (if available)
                    final String sCountryCode = aStoredDoc.getCountryCode();
                    final Locale aCountry = CountryCache.getInstance().getCountry(sCountryCode);
                    aTable.addBodyRow().addCell("Country:").addCell(new HCNodeList().addChild(PDCommonUI.getFlagNode(sCountryCode)).addChild(" ").addChild(span(aCountry != null ? aCountry.getDisplayCountry(aDisplayLocale) + " (" + sCountryCode + ")" : sCountryCode).addClass(CSS_CLASS_RESULT_DOC_COUNTRY_CODE)));
                }
                if (aStoredDoc.names().isNotEmpty()) {
                    // TODO add locale filter here
                    final ICommonsList<PDStoredMLName> aNames = PDCommonUI.getUIFilteredNames(aStoredDoc.names(), aDisplayLocale);
                    IHCNode aNameCtrl;
                    if (aNames.size() == 1)
                        aNameCtrl = PDCommonUI.getMLNameNode(aNames.getFirst(), aDisplayLocale);
                    else {
                        final HCUL aNameUL = new HCUL();
                        aNames.forEach(x -> aNameUL.addItem(PDCommonUI.getMLNameNode(x, aDisplayLocale)));
                        aNameCtrl = aNameUL;
                    }
                    aTable.addBodyRow().addCell("Entity Name:").addCell(span(aNameCtrl).addClass(CSS_CLASS_RESULT_DOC_NAME));
                }
                if (aStoredDoc.hasGeoInfo())
                    aTable.addBodyRow().addCell("Geographical information:").addCell(div(HCExtHelper.nl2divList(aStoredDoc.getGeoInfo())).addClass(CSS_CLASS_RESULT_DOC_GEOINFO));
                if (aStoredDoc.hasAdditionalInformation())
                    aTable.addBodyRow().addCell("Additional information:").addCell(div(HCExtHelper.nl2divList(aStoredDoc.getAdditionalInformation())).addClass(CSS_CLASS_RESULT_DOC_FREETEXT));
                aUL.addAndReturnItem(aTable).addClass(CSS_CLASS_RESULT_DOC_HEADER);
            }
            final BootstrapButton aShowDetailsBtn = new BootstrapButton(EBootstrapButtonType.SUCCESS, EBootstrapButtonSize.DEFAULT).addChild("Show details").setIcon(EDefaultIcon.MAGNIFIER).addClass(CSS_CLASS_RESULT_DOC_SDBUTTON).setOnClick(aWPEC.getSelfHref().add(FIELD_QUERY, sQuery).add(CPageParam.PARAM_ACTION, CPageParam.ACTION_VIEW).add(FIELD_PARTICIPANT_ID, aDocParticipantID.getURIEncoded()));
            aResultItem.addChild(div(aShowDetailsBtn));
            aOL.addItem(aResultItem);
            // Is the max result limit reached?
            if (aOL.getChildCount() >= nMaxResults)
                break;
        }
        aNodeList.addChild(aOL);
        aNodeList.addChild(div(new BootstrapButton().setOnClick(AJAX_EXPORT_LAST.getInvocationURL(aRequestScope)).addChild("Download results as XML").setIcon(EDefaultIcon.SAVE_ALL)));
    }
}