Examples with Indexer org.nzbhydra.indexers.Indexer used on opensource projects

Example 6 with Indexer

use of org.nzbhydra.indexers.Indexer in project nzbhydra2 by theotherp.

the class ResultAcceptorTest method shouldCheckForCategoryDisabledForIndexer.

@Test
public void shouldCheckForCategoryDisabledForIndexer() {
    Indexer indexer = new Newznab();
    indexer.initialize(indexerConfig, new IndexerEntity());
    item.setIndexer(indexer);
    // All categories enabled
    when(indexerConfig.getEnabledCategories()).thenReturn(Collections.emptyList());
    assertTrue(testee.checkForCategoryDisabledForIndexer(searchRequest, HashMultiset.create(), item));
    // Used category enabled
    when(indexerConfig.getEnabledCategories()).thenReturn(Arrays.asList(category.getName()));
    assertTrue(testee.checkForCategoryDisabledForIndexer(searchRequest, HashMultiset.create(), item));
    // Only other category enabled
    when(indexerConfig.getEnabledCategories()).thenReturn(Arrays.asList("Other"));
    assertFalse(testee.checkForCategoryDisabledForIndexer(searchRequest, HashMultiset.create(), item));
}

Example 7 with Indexer

use of org.nzbhydra.indexers.Indexer in project nzbhydra2 by theotherp.

the class Stats method indexerApiAccesses.

List<IndexerApiAccessStatsEntry> indexerApiAccesses(final StatsRequest statsRequest) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    logger.debug("Calculating indexer API stats");
    Set<Integer> indexerIdsToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(x -> x.getIndexerEntity().getId()).filter(id -> indexerRepository.findOne(id) != null).collect(Collectors.toSet());
    String averageIndexerAccessesPerDay = "SELECT\n" + "  indexer_id,\n" + "  avg(count)\n" + "FROM (\n" + "  (SELECT\n" + "     INDEXER_ID,\n" + "     cast(count(INDEXER_ID) AS FLOAT) AS count" + "   FROM INDEXERAPIACCESS\n" + buildWhereFromStatsRequest(false, statsRequest) + "   GROUP BY INDEXER_ID,\n" + "     truncate(time)))\n" + "GROUP BY INDEXER_ID";
    Map<Integer, Double> accessesPerDayCountMap = new HashMap<>();
    Query query = entityManager.createNativeQuery(averageIndexerAccessesPerDay);
    // query = query.setParameter("indexerIds", indexerIdsToInclude);
    List results = query.getResultList();
    for (Object resultObject : results) {
        Object[] array = (Object[]) resultObject;
        Integer indexerId = (Integer) array[0];
        if (!indexerIdsToInclude.contains(indexerId)) {
            continue;
        }
        Double avg = (Double) array[1];
        accessesPerDayCountMap.put(indexerId, avg);
    }
    logger.debug(LoggingMarkers.PERFORMANCE, "Calculating accesses per day took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    stopwatch.start();
    String countByResultSql = "SELECT\n" + "     INDEXER_ID,\n" + "     RESULT,\n" + "     count(result) AS count\n" + "   FROM INDEXERAPIACCESS\n" + buildWhereFromStatsRequest(false, statsRequest) + "   GROUP BY INDEXER_ID, RESULT\n" + "   ORDER BY INDEXER_ID, RESULT";
    Map<Integer, Integer> successCountMap = new HashMap<>();
    Map<Integer, Integer> connectionErrorCountMap = new HashMap<>();
    Map<Integer, Integer> allAccessesCountMap = new HashMap<>();
    query = entityManager.createNativeQuery(countByResultSql);
    // query = query.setParameter("indexerIds", indexerIdsToInclude);
    results = query.getResultList();
    for (Object resultObject : results) {
        Object[] array = (Object[]) resultObject;
        Integer indexerId = (Integer) array[0];
        if (!indexerIdsToInclude.contains(indexerId)) {
            continue;
        }
        String result = (String) array[1];
        int count = ((BigInteger) array[2]).intValue();
        if (result.equals(IndexerAccessResult.SUCCESSFUL.name())) {
            successCountMap.put(indexerId, count);
        } else if (result.equals(IndexerAccessResult.CONNECTION_ERROR.name())) {
            connectionErrorCountMap.put(indexerId, count);
        }
        if (allAccessesCountMap.containsKey(indexerId)) {
            allAccessesCountMap.put(indexerId, allAccessesCountMap.get(indexerId) + count);
        } else {
            allAccessesCountMap.put(indexerId, count);
        }
    }
    List<IndexerApiAccessStatsEntry> indexerApiAccessStatsEntries = new ArrayList<>();
    for (Integer id : indexerIdsToInclude) {
        IndexerApiAccessStatsEntry entry = new IndexerApiAccessStatsEntry();
        IndexerEntity indexerEntity = indexerRepository.findOne(id);
        entry.setIndexerName(indexerEntity.getName());
        if (allAccessesCountMap.containsKey(id) && allAccessesCountMap.get(id) != null) {
            if (successCountMap.get(id) != null) {
                Double percentSuccessFul = 100D / (allAccessesCountMap.get(id).doubleValue() / successCountMap.get(id).doubleValue());
                entry.setPercentSuccessful(percentSuccessFul);
            }
            if (connectionErrorCountMap.get(id) != null) {
                Double percentConnectionError = 100D / (allAccessesCountMap.get(id).doubleValue() / connectionErrorCountMap.get(id).doubleValue());
                entry.setPercentConnectionError(percentConnectionError);
            }
        }
        if (accessesPerDayCountMap.containsKey(id) && accessesPerDayCountMap.get(id) != null) {
            entry.setAverageAccessesPerDay(accessesPerDayCountMap.get(id));
        }
        indexerApiAccessStatsEntries.add(entry);
    }
    logger.debug(LoggingMarkers.PERFORMANCE, "Calculating success/failure stats took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    return indexerApiAccessStatsEntries;
}

Example 8 with Indexer

use of org.nzbhydra.indexers.Indexer in project nzbhydra2 by theotherp.

the class Stats method indexerSearchShares.

/**
 * Calculates how much of the search results indexers provide to searches on average and how much the share of unique results is. Excludes:
 * <ul>
 * <li>raw search engines from uniqe shares because they return result names that are rarely matched as duplicates</li>
 * <li>"update queries" because here some indexers return a "total" of 1000 and some of 1000000 or something like that</li>
 * <p>
 * </ul>
 */
@Transactional(readOnly = true)
List<IndexerSearchResultsShare> indexerSearchShares(final StatsRequest statsRequest) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    logger.debug("Calculating indexer search shares");
    List<IndexerSearchResultsShare> indexerSearchResultsShares = new ArrayList<>();
    Map<Integer, String> indexerIdToName = new HashMap<>();
    List<Indexer> indexersToInclude = statsRequest.isIncludeDisabled() ? searchModuleProvider.getIndexers() : searchModuleProvider.getEnabledIndexers();
    List<SearchModuleType> typesToUse = Arrays.asList(SearchModuleType.NEWZNAB, SearchModuleType.TORZNAB, SearchModuleType.ANIZB);
    indexersToInclude = indexersToInclude.stream().filter(x -> typesToUse.contains(x.getConfig().getSearchModuleType())).collect(Collectors.toList());
    for (Indexer indexer : indexersToInclude) {
        indexerIdToName.put(indexer.getIndexerEntity().getId(), indexer.getName());
    }
    Set<Integer> indexerIdsToInclude = indexerIdToName.keySet();
    String countResultsSql = "SELECT\n" + "  INDEXER_ENTITY_ID,\n" + "  INDEXERRESULTSSUM,\n" + "  ALLRESULTSSUM,\n" + "  INDEXERUNIQUERESULTSSUM,\n" + "  ALLUNIQUERESULTSSUM\n" + "FROM\n" + "  (SELECT\n" + "     SUM(INDEXERSEARCH.RESULTS_COUNT)  AS INDEXERRESULTSSUM,\n" + "     SUM(INDEXERSEARCH.UNIQUE_RESULTS) AS INDEXERUNIQUERESULTSSUM,\n" + "     INDEXERSEARCH.INDEXER_ENTITY_ID\n" + "   FROM indexersearch\n" + "   WHERE indexersearch.ID IN (SELECT INDEXERSEARCH.ID\n" + "                FROM indexersearch\n" + "                  LEFT JOIN SEARCH ON INDEXERSEARCH.SEARCH_ENTITY_ID = SEARCH.ID\n" + "                WHERE indexersearch.INDEXER_ENTITY_ID IN (:indexerIds) \n" + "                      AND INDEXERSEARCH.successful AND\n" + "                      INDEXERSEARCH.SEARCH_ENTITY_ID IN (SELECT SEARCH.ID\n" + "                                                         FROM SEARCH\n" + "                                                           LEFT JOIN SEARCH_IDENTIFIERS ON SEARCH.ID = SEARCH_IDENTIFIERS.SEARCH_ENTITY_ID\n" + "                                                         WHERE\n" + "                                                           (SEARCH.episode IS NOT NULL OR SEARCH.season IS NOT NULL OR SEARCH.query IS NOT NULL OR SEARCH_IDENTIFIERS.SEARCH_ENTITY_ID IS NOT NULL OR SEARCH.AUTHOR IS NOT NULL OR SEARCH.TITLE IS NOT NULL) \n" + buildWhereFromStatsRequest(true, statsRequest) + "                      )\n" + "   )" + "   GROUP BY INDEXER_ENTITY_ID" + ") FORINDEXER,\n" + "  (SELECT\n" + "      sum(INDEXERSEARCH.RESULTS_COUNT)  AS ALLRESULTSSUM,\n" + "      SUM(INDEXERSEARCH.UNIQUE_RESULTS) AS ALLUNIQUERESULTSSUM\n" + "    FROM INDEXERSEARCH\n" + "    WHERE INDEXERSEARCH.SEARCH_ENTITY_ID IN (SELECT SEARCH.ID\n" + "                                             FROM indexersearch\n" + "                                               LEFT JOIN SEARCH ON INDEXERSEARCH.SEARCH_ENTITY_ID = SEARCH.ID\n" + "                                               LEFT JOIN SEARCH_IDENTIFIERS ON SEARCH.ID = SEARCH_IDENTIFIERS.SEARCH_ENTITY_ID\n" + "                                             WHERE indexersearch.INDEXER_ENTITY_ID IN (:indexerIds) \n" + "                                                   AND INDEXERSEARCH.successful AND\n" + "                                                   INDEXERSEARCH.SEARCH_ENTITY_ID IN (SELECT SEARCH.ID\n" + "                                                                                      FROM SEARCH\n" + "                                                                                        LEFT JOIN SEARCH_IDENTIFIERS ON SEARCH.ID = SEARCH_IDENTIFIERS.SEARCH_ENTITY_ID\n" + "                                                                                      WHERE\n" + "                                                                                        (SEARCH.episode IS NOT NULL OR SEARCH.season IS NOT NULL OR SEARCH.query IS NOT NULL OR\n" + "                                                                                         SEARCH_IDENTIFIERS.SEARCH_ENTITY_ID IS NOT NULL OR SEARCH.AUTHOR IS NOT NULL OR SEARCH.TITLE IS NOT NULL)\n" + buildWhereFromStatsRequest(true, statsRequest) + "                                                   )\n" + "                                                   AND INDEXERSEARCH.successful)" + "         AND INDEXERSEARCH.successful\n" + "  ) FORALL";
    Query countQuery = entityManager.createNativeQuery(countResultsSql).setParameter("indexerIds", indexerIdsToInclude);
    List results = countQuery.getResultList();
    for (Object resultObject : results) {
        Object[] resultSet = (Object[]) resultObject;
        Float allShare = null;
        if (resultSet[1] != null && resultSet[2] != null) {
            BigInteger indexerResultsSum = (BigInteger) resultSet[1];
            BigInteger allResultsSum = (BigInteger) resultSet[2];
            if (indexerResultsSum.intValue() > 0) {
                allShare = 100 / (allResultsSum.floatValue() / indexerResultsSum.floatValue());
            }
        }
        Float uniqueShare = null;
        if (resultSet[3] != null && resultSet[4] != null) {
            BigInteger indexerUniqueResultsSum = (BigInteger) resultSet[3];
            BigInteger allUniqueResultsSum = (BigInteger) resultSet[4];
            if (allUniqueResultsSum.intValue() > 0) {
                uniqueShare = 100 / (allUniqueResultsSum.floatValue() / indexerUniqueResultsSum.floatValue());
            }
        }
        Integer indexerId = (Integer) resultSet[0];
        indexerSearchResultsShares.add(new IndexerSearchResultsShare(indexerIdToName.get(indexerId), allShare, uniqueShare));
    }
    logger.debug(LoggingMarkers.PERFORMANCE, "Calculated indexer search shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    return indexerSearchResultsShares;
}

Example 9 with Indexer

use of org.nzbhydra.indexers.Indexer in project nzbhydra2 by theotherp.

the class Stats method averageResponseTimes.

List<AverageResponseTime> averageResponseTimes(final StatsRequest statsRequest) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    logger.debug("Calculating average response times for indexers");
    List<AverageResponseTime> averageResponseTimes = new ArrayList<>();
    String sql = "SELECT\n" + "  NAME,\n" + "  avg(RESPONSE_TIME) AS avg\n" + "FROM INDEXERAPIACCESS\n" + "  LEFT JOIN indexer i ON INDEXERAPIACCESS.INDEXER_ID = i.ID\n" + buildWhereFromStatsRequest(false, statsRequest) + "GROUP BY INDEXER_ID\n" + "ORDER BY avg ASC";
    Query query = entityManager.createNativeQuery(sql);
    List resultList = query.getResultList();
    Set<String> indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet());
    OptionalDouble overallAverage = resultList.stream().filter(x -> ((Object[]) x)[1] != null).mapToLong(x -> ((BigInteger) ((Object[]) x)[1]).longValue()).average();
    for (Object result : resultList) {
        Object[] resultSet = (Object[]) result;
        String indexerName = (String) resultSet[0];
        if (resultSet[0] == null || resultSet[1] == null || !indexerNamesToInclude.contains(indexerName)) {
            continue;
        }
        Long averageResponseTime = ((BigInteger) resultSet[1]).longValue();
        averageResponseTimes.add(new AverageResponseTime(indexerName, averageResponseTime, averageResponseTime - overallAverage.orElse(0D)));
    }
    logger.debug(LoggingMarkers.PERFORMANCE, "Calculated average response times for indexers. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    return averageResponseTimes;
}