Examples with MediaObjectDescriptor org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor used on opensource projects

Example 1 with MediaObjectDescriptor

use of org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor in project cineast by vitrivr.

the class CineastQueryService method getSimilar.

// TODO This has enormous code duplication with the TemporalQueryMessageHandler
@Override
public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGrpc.QueryResult> responseObserver) {
    StopWatch watch = StopWatch.createStarted();
    MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    Set<String> sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>();
    CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig();
    ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config);
    QueryConfig qconf = new QueryConfig(rqconf);
    /* Prepare QueryConfig (so as to obtain a QueryId). */
    final String uuid = qconf.getQueryId().toString();
    final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
    qconf.setMaxResults(max);
    final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule();
    qconf.setResultsPerModule(resultsPerModule);
    List<Thread> metadataRetrievalThreads = new ArrayList<>();
    /* We iterate over all components independently, because they have a temporal context.*/
    for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) {
        List<QueryStage> stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx));
        /* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/
        QueryConfig stageQConf = QueryConfig.clone(qconf);
        /* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */
        HashSet<String> relevantSegments = new HashSet<>();
        /* Store for each queryterm per category all results to be sent at a later time */
        List<Map<String, List<StringDoublePair>>> cache = new ArrayList<>();
        /* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */
        for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) {
            /* Initalize stage with this hashmap */
            cache.add(stageIndex, new HashMap<>());
            QueryStage stage = stages.get(stageIndex);
            List<Thread> qtThreads = new ArrayList<>();
            /* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage.
         * The list is only updated once we've iterated over all terms
         */
            for (int i = 0; i < stage.getQueryTerms().size(); i++) {
                QueryTerm qt = stage.getQueryTerms().get(i);
                final int finalContainerIdx = containerIdx;
                final int finalStageIndex = stageIndex;
                Thread qtRetrievalThread = new Thread(() -> {
                    /* Prepare QueryTerm and perform sanity-checks */
                    if (qt == null) {
                        /* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */
                        LOGGER.warn("QueryTerm was null for stage {}", stage);
                        return;
                    }
                    AbstractQueryTermContainer qc = qt.getContainer();
                    if (qc == null) {
                        LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
                        return;
                    }
                    List<Thread> categoryThreads = new ArrayList<>();
                    /* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */
                    for (String category : qt.getCategories()) {
                        /* Merge partial results with score-map */
                        List<SegmentScoreElement> scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf);
                        /* Transform raw results into list of StringDoublePairs (segmentId -> score) */
                        final List<StringDoublePair> results = scores.stream().map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())).filter(p -> p.value > 0d).sorted(StringDoublePair.COMPARATOR).limit(max).collect(Collectors.toList());
                        if (results.isEmpty()) {
                            LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage);
                        }
                        if (cache.get(finalStageIndex).containsKey(category)) {
                            LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex);
                        }
                        cache.get(finalStageIndex).put(category, results);
                        results.forEach(res -> relevantSegments.add(res.key));
                        LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS));
                        /* If this is the last stage, we can send relevant results per category back to the UI.
               * Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage
               */
                        if (finalStageIndex == stages.size() - 1) {
                            /* Finalize and submit per-container results */
                            responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(qt.getQueryConfig().getQueryId().toString(), category, results)));
                            List<String> segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList());
                            if (segmentIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaSegmentDescriptor> segments = mediaSegmentReader.lookUpSegments(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments(segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList())).build()));
                            List<MediaSegmentMetadataDescriptor> segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData(segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList())).build()));
                            sentSegmentIds.addAll(segmentIds);
                            List<String> objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList());
                            if (objectIds.isEmpty()) {
                                continue;
                            }
                            Map<String, MediaObjectDescriptor> objects = mediaObjectReader.lookUpObjects(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects(objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList())).build()));
                            List<MediaObjectMetadataDescriptor> objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds);
                            responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData(objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList())).build()));
                            sentObjectIds.addAll(objectIds);
                        }
                    }
                /* We're done for this querycontainer */
                });
                // TODO Better name
                qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories());
                qtThreads.add(qtRetrievalThread);
                qtRetrievalThread.start();
            }
            for (Thread thread : qtThreads) {
                try {
                    thread.join();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            /* After we are done with a stage, we add all relevant segments to the config for the next stage. */
            if (relevantSegments.size() == 0) {
                LOGGER.warn("No relevant segments anymore, aborting staged querying");
                /* Clear relevant segments (there are none) */
                stageQConf.setRelevantSegmentIds(relevantSegments);
                break;
            }
            stageQConf.setRelevantSegmentIds(relevantSegments);
            relevantSegments.clear();
        }
        /* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */
        for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) {
            cache.get(stageIndex).forEach((category, results) -> {
                results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key));
                responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(// TODO This assumes that all queries in a temporalquery have the same uuid
                uuid, category, results)));
            });
        }
    /* There should be no carry-over from this block since temporal queries are executed independently */
    }
    /* At this point, all StagedQueries have been executed for this TemporalQuery.
     * Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do.
     */
    responseObserver.onCompleted();
    mediaSegmentReader.close();
    mediaObjectReader.close();
    segmentMetadataReader.close();
    watch.stop();
    LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS));
}

Example 2 with MediaObjectDescriptor

use of org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor in project cineast by vitrivr.

the class FindObjectGetHandler method doGet.

@Override
public MediaObjectQueryResult doGet(Context ctx) {
    final Map<String, String> parameters = ctx.pathParamMap();
    final String attribute = parameters.get(ATTRIBUTE_NAME);
    final String value = parameters.get(VALUE_NAME);
    final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    MediaObjectDescriptor object = null;
    switch(attribute.toLowerCase()) {
        case "id":
            {
                object = ol.lookUpObjectById(value);
                break;
            }
        case "name":
            {
                object = ol.lookUpObjectByName(value);
                break;
            }
        case "path":
            {
                object = ol.lookUpObjectByPath(value);
                break;
            }
        default:
            {
                LOGGER.error("Unknown attribute '{}' in FindObjectByActionHandler", attribute);
            }
    }
    ol.close();
    return new MediaObjectQueryResult("", Lists.newArrayList(object));
}

Example 3 with MediaObjectDescriptor

use of org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor in project cineast by vitrivr.

the class FindObjectAllGetHandler method doGet.

// public static final String ROUTE = "find/objects/all/:"+TYPE_NAME;
@Override
public MediaObjectQueryResult doGet(Context ctx) {
    // TODO :type is not being used
    final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
    final List<MediaObjectDescriptor> multimediaobjectIds = ol.getAllObjects();
    ol.close();
    return new MediaObjectQueryResult("", multimediaobjectIds);
}

Example 4 with MediaObjectDescriptor

use of org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor in project cineast by vitrivr.

the class EvaluationRuntime method objectDescriptorForId.

/**
 * Returns a MediaObjectDescriptor for the provided docID. This method uses a cache to speedup lookup of objects.
 *
 * @param docID ID of the segment for which the MediaObjectDescriptor is required.
 * @return MediaObjectDescriptor
 */
private MediaObjectDescriptor objectDescriptorForId(String docID) {
    if (this.cache.containsKey(docID)) {
        return this.cache.get(docID);
    }
    Optional<MediaSegmentDescriptor> descriptor = this.mediaSegmentReader.lookUpSegment(docID);
    if (descriptor.isPresent()) {
        MediaObjectDescriptor object = this.mediaObjectReader.lookUpObjectById(descriptor.get().getObjectId());
        this.cache.put(docID, object);
        return object;
    } else {
        return null;
    }
}

Example 5 with MediaObjectDescriptor

use of org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor in project cineast by vitrivr.

the class EvaluationRuntime method performEvaluation.

/**
 * Performs the actual evaluation for a result set and reference document. Now iterate through the list of retrieved documents and decides for each entry if its class is equal to the class of the reference document (i.e. if it is relevant or not). Based on the outcome, the EvaluationResult is updated.
 *
 * @param scores The result set - list of SegmentScoreElements.
 * @param path   Path to the reference document file. Its filename is used as docID
 * @param gt     Ground truth object used for evaluation.
 * @return EvaluationResult
 * @throws EvaluationException If something goes wrong during evaluation, e.g. reference document has no class or a item in the result set has no class
 */
private EvaluationResult performEvaluation(List<SegmentScoreElement> scores, Path path, Groundtruth gt) throws EvaluationException {
    /* Constructs a document ID from the filename and fetches the file's class from the ground truth. */
    String docID = path.getFileName().toString();
    String fileClass = gt.classForDocId(docID).orElseThrow(() -> new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", path.getFileName())));
    /* Prepare empty evaluation results. */
    EvaluationResult result = new EvaluationResult(docID, gt);
    /*
     * Now iterate through the list of retrieved documents and decide for each entry if it relevant according to its class.
     */
    for (int k = 1; k <= scores.size(); k++) {
        SegmentScoreElement score = scores.get(k - 1);
        MediaObjectDescriptor object = this.objectDescriptorForId(score.getSegmentId());
        if (object != null) {
            if (gt.classForDocId(object.getName()).orElse("<none>").equals(fileClass)) {
                result.documentAvailable(object.getName(), k, true);
            } else {
                result.documentAvailable(object.getName(), k, false);
            }
            if (result.done()) {
                LOGGER.info("All relevant objects were retrieved. Starting next round...", score.getSegmentId());
                break;
            }
        } else {
            throw new EvaluationException(String.format("The provided test file %s does not have a class associated with it.", score.getSegmentId()));
        }
    }
    return result;
}