Examples with SegmentScoreElement org.vitrivr.cineast.core.data.score.SegmentScoreElement used on opensource projects

Example 1 with SegmentScoreElement

use of org.vitrivr.cineast.core.data.score.SegmentScoreElement in project cineast by vitrivr.

the class CineastQueryService method getSimilar.

// TODO This has enormous code duplication with the TemporalQueryMessageHandler
@Override
public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGrpc.QueryResult> responseObserver) {
    StopWatch watch = StopWatch.createStarted();
    MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    Set<String> sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>();
    CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig();
    ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config);
    QueryConfig qconf = new QueryConfig(rqconf);
    /* Prepare QueryConfig (so as to obtain a QueryId). */
    final String uuid = qconf.getQueryId().toString();
    final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
    qconf.setMaxResults(max);
    final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule();
    qconf.setResultsPerModule(resultsPerModule);
    List<Thread> metadataRetrievalThreads = new ArrayList<>();
    /* We iterate over all components independently, because they have a temporal context.*/
    for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) {
        List<QueryStage> stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx));
        /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/
        QueryConfig stageQConf = QueryConfig.clone(qconf);
        /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */
        HashSet<String> relevantSegments = new HashSet<>();
        /* Store for each queryterm per category all results to be sent at a later time */
        List<Map<String, List<StringDoublePair>>> cache = new ArrayList<>();
        /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */
        for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) {
            /* Initalize stage with this hashmap */
            cache.add(stageIndex, new HashMap<>());
            QueryStage stage = stages.get(stageIndex);
            List<Thread> qtThreads = new ArrayList<>();
            /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage.
         * The list is only updated once we've iterated over all terms
         */
            for (int i = 0; i < stage.getQueryTerms().size(); i++) {
                QueryTerm qt = stage.getQueryTerms().get(i);
                final int finalContainerIdx = containerIdx;
                final int finalStageIndex = stageIndex;
                Thread qtRetrievalThread = new Thread(() -> {
                    /* Prepare QueryTerm and perform sanity-checks */
                    if (qt == null) {
                        /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */
                        LOGGER.warn("QueryTerm was null for stage {}", stage);
                        return;
                    }
                    AbstractQueryTermContainer qc = qt.getContainer();
                    if (qc == null) {
                        LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
                        return;
                    }
                    List<Thread> categoryThreads = new ArrayList<>();
                    /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */
                    for (String category : qt.getCategories()) {
                        /* Merge partial results with score-map */
                        List<SegmentScoreElement> scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf);
                        /* Transform raw results into list of StringDoublePairs (segmentId -> score) */
                        final List<StringDoublePair> results = scores.stream().map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())).filter(p -> p.value > 0d).sorted(StringDoublePair.COMPARATOR).limit(max).collect(Collectors.toList());
                        if (results.isEmpty()) {
                            LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage);
                        }
                        if (cache.get(finalStageIndex).containsKey(category)) {
                            LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex);
                        }
                        cache.get(finalStageIndex).put(category, results);
                        results.forEach(res -> relevantSegments.add(res.key));
                        LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS));
                        /* If this is the last stage, we can send relevant results per category back to the UI.
               * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage
               */
                        if (finalStageIndex == stages.size() - 1) {
                            /* Finalize and submit per-container results */
                            responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(qt.getQueryConfig().getQueryId().toString(), category, results)));
                            List<String> segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList());
                            if (segmentIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaSegmentDescriptor> segments = mediaSegmentReader.lookUpSegments(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments(segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList())).build()));
                            List<MediaSegmentMetadataDescriptor> segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData(segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList())).build()));
                            sentSegmentIds.addAll(segmentIds);
                            List<String> objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList());
                            if (objectIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaObjectDescriptor> objects = mediaObjectReader.lookUpObjects(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects(objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList())).build()));
                            List<MediaObjectMetadataDescriptor> objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData(objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList())).build()));
                            sentObjectIds.addAll(objectIds);
                        }
                    }
                /* We're done for this querycontainer */
                });
                // TODO Better name
                qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories());
                qtThreads.add(qtRetrievalThread);
                qtRetrievalThread.start();
            }
            for (Thread thread : qtThreads) {
                try {
                    thread.join();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            /* After we are done with a stage, we add all relevant segments to the config for the next stage. */
            if (relevantSegments.size() == 0) {
                LOGGER.warn("No relevant segments anymore, aborting staged querying");
                /* Clear relevant segments (there are none) */
                stageQConf.setRelevantSegmentIds(relevantSegments);
                break;
            }
            stageQConf.setRelevantSegmentIds(relevantSegments);
            relevantSegments.clear();
        }
        /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */
        for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) {
            cache.get(stageIndex).forEach((category, results) -> {
                results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key));
                responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(// TODO This assumes that all queries in a temporalquery have the same uuid
                uuid, category, results)));
            });
        }
    /* There should be no carry-over from this block since temporal queries are executed independently */
    }
    /* At this point, all StagedQueries have been executed for this TemporalQuery.
     * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do.
     */
    responseObserver.onCompleted();
    mediaSegmentReader.close();
    mediaObjectReader.close();
    segmentMetadataReader.close();
    watch.stop();
    LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS));
}

Example 2 with SegmentScoreElement

use of org.vitrivr.cineast.core.data.score.SegmentScoreElement in project cineast by vitrivr.

the class CliUtils method retrieveAndLog.

public static void retrieveAndLog(List<Retriever> retrievers, ContinuousRetrievalLogic retrieval, int limit, boolean printDetail, AbstractQueryTermContainer qc) {
    System.out.println("Only printing the first " + limit + " results, change with --limit parameter");
    DBSelector selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get();
    retrievers.forEach(retriever -> {
        AtomicBoolean entityExists = new AtomicBoolean(true);
        retriever.getTableNames().forEach(table -> {
            if (!selector.existsEntity(table)) {
                System.out.println("Entity " + table + " does not exist");
                entityExists.set(false);
            }
        });
        if (!entityExists.get()) {
            System.out.println("Not retrieving for " + retriever.getClass().getSimpleName() + " because entity does not exist");
            return;
        }
        System.out.println("Retrieving for " + retriever.getClass().getSimpleName());
        long start = System.currentTimeMillis();
        List<SegmentScoreElement> results = retrieval.retrieveByRetriever(qc, retriever, new ConstrainedQueryConfig().setMaxResults(limit));
        long stop = System.currentTimeMillis();
        System.out.println("Results for " + retriever.getClass().getSimpleName() + ":, retrieved in " + (stop - start) + "ms");
        for (SegmentScoreElement e : results) {
            System.out.print(e.getSegmentId());
            System.out.print(": ");
            System.out.println(e.getScore());
            if (printDetail) {
                CliUtils.printInfoForSegment(e.getSegmentId(), selector, null, true);
            }
        }
        System.out.println();
    });
    retrieval.shutdown();
}

Example 3 with SegmentScoreElement

use of org.vitrivr.cineast.core.data.score.SegmentScoreElement in project cineast by vitrivr.

the class EvaluationRuntime method call.

/**
 * Executes the evaluation and returns a Triple that contains the number of files that were processed, skipped due to errors and skipped deliberately. The actual evaluation results are written to files.
 *
 * @return computed result
 * @throws EvaluationException if unable to compute a result
 * @see EvaluationConfig
 */
@Override
public Triple<Integer, Integer, Integer> call() throws EvaluationException, IOException {
    /* Tries to instantiate the converter. */
    final Converter converter = this.config.getConverter();
    if (converter == null) {
        throw new EvaluationException("Failed to instantiate the converter class.");
    }
    /* Instantiates the groundtruth and checks if it contains classes. */
    final Groundtruth gt = this.config.getGroundtruth();
    if (gt.numberOfClasses() == 0) {
        throw new EvaluationException(String.format("The specified ground truth '%s' does not contain any classes.", this.config.getClassfile()));
    }
    /* Updates the retrieval configuration. */
    Config.sharedConfig().getRetriever().setMaxResults(this.config.getSize());
    Config.sharedConfig().getRetriever().setResultsPerModule(this.config.getSize());
    /* Prepares the iterator for the test files. */
    final Iterator<Path> testfilesIterator;
    try {
        testfilesIterator = Files.walk(this.config.getTestfiles()).filter(p -> {
            try {
                return Files.exists(p) && Files.isRegularFile(p) && !Files.isHidden(p) && Files.isReadable(p);
            } catch (IOException e) {
                LOGGER.error("An IO exception occurred while testing the media file at '{}'.", p.toString(), LogHelper.getStackTrace(e));
                return false;
            }
        }).iterator();
    } catch (IOException exception) {
        throw new EvaluationException(String.format("Could not obtain test files under the specified path '%s'.", this.config.getTestfiles()));
    }
    /* Prepare folder structure per category. */
    for (String category : this.config.getCategories()) {
        Files.createDirectories(this.config.getResults().resolve(category));
    }
    /* Prepare a placeholder query-config. */
    final ReadableQueryConfig queryConfig = new ReadableQueryConfig(null);
    /* Prepare a random number generator that decides if a file should be used for evaluation or not. */
    final Random random = new Random();
    /* Perform evaluation for every file. */
    Path path;
    while (testfilesIterator.hasNext()) {
        path = testfilesIterator.next();
        if (random.nextBoolean() && config.getMode() == EvaluationConfig.EvaluationMode.RANDOM) {
            LOGGER.info("Randomly skipping file {}.", path);
            this.skipped += 1;
            continue;
        }
        /* Try to create a QueryContainer. If this fails, the file is skipped. */
        final AbstractQueryTermContainer container = converter.convert(path);
        if (container == null) {
            LOGGER.warn("Failed to convert the file {}. File is being skipped...", path.getFileName());
            this.error += 1;
            continue;
        }
        LOGGER.info("Starting evaluation for {}", path);
        for (String category : this.config.getCategories()) {
            List<SegmentScoreElement> scores = this.retrievalLogic.retrieve(container, category, queryConfig);
            EvaluationResult result = this.performEvaluation(scores, path, gt);
            this.writeToFile(category, result);
        }
        this.processed += 1;
    }
    return new ImmutableTriple<>(this.processed, this.error, this.skipped);
}

Example 4 with SegmentScoreElement

use of org.vitrivr.cineast.core.data.score.SegmentScoreElement in project cineast by vitrivr.

the class EvaluationRuntime method performEvaluation.

/**
 * Performs the actual evaluation for a result set and reference document. Now iterate through the list of retrieved documents and decides for each entry if its class is equal to the class of the reference document (i.e. if it is relevant or not). Based on the outcome, the EvaluationResult is updated.
 *
 * @param scores The result set - list of SegmentScoreElements.
 * @param path   Path to the reference document file. Its filename is used as docID
 * @param gt     Ground truth object used for evaluation.
 * @return EvaluationResult
 * @throws EvaluationException If something goes wrong during evaluation, e.g. reference document has no class or a item in the result set has no class
 */
private EvaluationResult performEvaluation(List<SegmentScoreElement> scores, Path path, Groundtruth gt) throws EvaluationException {
    /* Constructs a document ID from the filename and fetches the file's class from the ground truth. */
    String docID = path.getFileName().toString();
    String fileClass = gt.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", path.getFileName())));
    /* Prepare empty evaluation results. */
    EvaluationResult result = new EvaluationResult(docID, gt);
    /*
     * Now iterate through the list of retrieved documents and decide for each entry if it relevant according to its class.
     */
    for (int k = 1; k <= scores.size(); k++) {
        SegmentScoreElement score = scores.get(k - 1);
        MediaObjectDescriptor object = this.objectDescriptorForId(score.getSegmentId());
        if (object != null) {
            if (gt.classForDocId(object.getName()).orElse("<none>").equals(fileClass)) {
                result.documentAvailable(object.getName(), k, true);
            } else {
                result.documentAvailable(object.getName(), k, false);
            }
            if (result.done()) {
                LOGGER.info("All relevant objects were retrieved. Starting next round...", score.getSegmentId());
                break;
            }
        } else {
            throw new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", score.getSegmentId()));
        }
    }
    return result;
}

Example 5 with SegmentScoreElement

use of org.vitrivr.cineast.core.data.score.SegmentScoreElement in project cineast by vitrivr.

the class MFCCShingle method postprocessQuery.

/**
 * This method represents the last step that's executed when processing a query. A list of partial-results (DistanceElements) returned by the lookup stage is processed based on some internal method and finally converted to a list of ScoreElements. The filtered list of ScoreElements is returned by the feature module during retrieval.
 *
 * @param partialResults List of partial results returned by the lookup stage.
 * @param qc             A ReadableQueryConfig object that contains query-related configuration parameters.
 * @return List of final results. Is supposed to be de-duplicated and the number of items should not exceed the number of items per module.
 */
@Override
protected List<ScoreElement> postprocessQuery(List<SegmentDistanceElement> partialResults, ReadableQueryConfig qc) {
    /* Prepare helper data-structures. */
    final List<ScoreElement> results = new ArrayList<>();
    final TObjectIntHashMap<String> scoreMap = new TObjectIntHashMap<>();
    /* Set QueryConfig and extract correspondence function. */
    qc = this.setQueryConfig(qc);
    final CorrespondenceFunction correspondence = qc.getCorrespondenceFunction().orElse(this.correspondence);
    for (DistanceElement hit : partialResults) {
        if (hit.getDistance() < this.distanceThreshold) {
            scoreMap.adjustOrPutValue(hit.getId(), 1, scoreMap.get(hit.getId()) / 2);
        }
    }
    /* Prepare final result-set. */
    scoreMap.forEachEntry((key, value) -> results.add(new SegmentScoreElement(key, 1.0 - 1.0 / value)));
    ScoreElement.filterMaximumScores(results.stream());
    return results;
}