use of org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor in project cineast by vitrivr.
the class CineastQueryService method getSimilar.
// TODO This has enormous code duplication with the TemporalQueryMessageHandler
@Override
public void getSimilar(CineastGrpc.TemporalQuery query, StreamObserver<CineastGrpc.QueryResult> responseObserver) {
StopWatch watch = StopWatch.createStarted();
MediaSegmentReader mediaSegmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
MediaObjectReader mediaObjectReader = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
MediaSegmentMetadataReader segmentMetadataReader = new MediaSegmentMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
MediaObjectMetadataReader objectMetadataReader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
Set<String> sentSegmentIds = new HashSet<>(), sentObjectIds = new HashSet<>();
CineastGrpc.QueryConfig config = query.getQueryList().get(0).getConfig();
ReadableQueryConfig rqconf = QueryContainerUtil.queryConfig(config);
QueryConfig qconf = new QueryConfig(rqconf);
/* Prepare QueryConfig (so as to obtain a QueryId). */
final String uuid = qconf.getQueryId().toString();
final int max = qconf.getMaxResults().orElse(Config.sharedConfig().getRetriever().getMaxResults());
qconf.setMaxResults(max);
final int resultsPerModule = qconf.getRawResultsPerModule() == -1 ? Config.sharedConfig().getRetriever().getMaxResultsPerModule() : qconf.getResultsPerModule();
qconf.setResultsPerModule(resultsPerModule);
List<Thread> metadataRetrievalThreads = new ArrayList<>();
/* We iterate over all components independently, because they have a temporal context.*/
for (int containerIdx = 0; containerIdx < query.getQueryCount(); containerIdx++) {
List<QueryStage> stages = QueryContainerUtil.query(query.getQueryList().get(containerIdx));
/* We make a new stagedQueryConfig per stage because the relevant segments will differ for each stage. This also resets the filter (relevant ids in the config)*/
QueryConfig stageQConf = QueryConfig.clone(qconf);
/* For the first stage, there will be no relevant segments when querying. This is ok because the retrieval engine handles this appropriately */
HashSet<String> relevantSegments = new HashSet<>();
/* Store for each queryterm per category all results to be sent at a later time */
List<Map<String, List<StringDoublePair>>> cache = new ArrayList<>();
/* For the terms of a stage, ordering matters. The assumption is that each term is used as a filter for its successor */
for (int stageIndex = 0; stageIndex < stages.size(); stageIndex++) {
/* Initalize stage with this hashmap */
cache.add(stageIndex, new HashMap<>());
QueryStage stage = stages.get(stageIndex);
List<Thread> qtThreads = new ArrayList<>();
/* We now iterate over all QueryTerms for this stage, simply adding their results to the list of relevant segments for the next querystage.
* The list is only updated once we've iterated over all terms
*/
for (int i = 0; i < stage.getQueryTerms().size(); i++) {
QueryTerm qt = stage.getQueryTerms().get(i);
final int finalContainerIdx = containerIdx;
final int finalStageIndex = stageIndex;
Thread qtRetrievalThread = new Thread(() -> {
/* Prepare QueryTerm and perform sanity-checks */
if (qt == null) {
/* In rare instances, it is possible to have null as query stage. If this happens to you, please report this to the developers so we can try to fix it. */
LOGGER.warn("QueryTerm was null for stage {}", stage);
return;
}
AbstractQueryTermContainer qc = qt.getContainer();
if (qc == null) {
LOGGER.warn("Likely an empty query, as it could not be converted to a query container. Ignoring it");
return;
}
List<Thread> categoryThreads = new ArrayList<>();
/* For each category of a specific queryterm, we actually go and retrieve. Be aware that we do not change the relevant ids after this call */
for (String category : qt.getCategories()) {
/* Merge partial results with score-map */
List<SegmentScoreElement> scores = continuousRetrievalLogic.retrieve(qc, category, stageQConf);
/* Transform raw results into list of StringDoublePairs (segmentId -> score) */
final List<StringDoublePair> results = scores.stream().map(elem -> new StringDoublePair(elem.getSegmentId(), elem.getScore())).filter(p -> p.value > 0d).sorted(StringDoublePair.COMPARATOR).limit(max).collect(Collectors.toList());
if (results.isEmpty()) {
LOGGER.warn("No results found for category {} and qt {} in stage with id {}. Full compoment: {}", category, qt, finalContainerIdx, stage);
}
if (cache.get(finalStageIndex).containsKey(category)) {
LOGGER.error("Category {} was used twice in stage {}. This erases the results of the previous category... ", category, finalStageIndex);
}
cache.get(finalStageIndex).put(category, results);
results.forEach(res -> relevantSegments.add(res.key));
LOGGER.trace("Category {} at stage {} executed @ {} ms", category, finalStageIndex, watch.getTime(TimeUnit.MILLISECONDS));
/* If this is the last stage, we can send relevant results per category back to the UI.
* Otherwise, we cannot since we might send results to the UI which would be filtered at a later stage
*/
if (finalStageIndex == stages.size() - 1) {
/* Finalize and submit per-container results */
responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(qt.getQueryConfig().getQueryId().toString(), category, results)));
List<String> segmentIds = results.stream().map(x -> x.key).filter(x -> !sentSegmentIds.contains(x)).collect(Collectors.toList());
if (segmentIds.isEmpty()) {
continue;
}
Map<String, MediaSegmentDescriptor> segments = mediaSegmentReader.lookUpSegments(segmentIds);
responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentQueryResult.newBuilder().addAllSegments(segments.values().stream().map(MediaSegmentUtil::fromMediaSegmentDescriptor).collect(Collectors.toList())).build()));
List<MediaSegmentMetadataDescriptor> segmentMetaData = segmentMetadataReader.lookupMultimediaMetadata(segmentIds);
responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaSegmentMetaDataQueryResult.newBuilder().addAllSegmentMetaData(segmentMetaData.stream().map(QueryContainerUtil::mediaSegmentMetaData).collect(Collectors.toList())).build()));
sentSegmentIds.addAll(segmentIds);
List<String> objectIds = segments.values().stream().map(MediaSegmentDescriptor::getObjectId).filter(x -> !sentObjectIds.contains(x)).collect(Collectors.toList());
if (objectIds.isEmpty()) {
continue;
}
Map<String, MediaObjectDescriptor> objects = mediaObjectReader.lookUpObjects(objectIds);
responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectQueryResult.newBuilder().addAllObjects(objects.values().stream().map(MediaObjectUtil::fromMediaObjectDescriptor).collect(Collectors.toList())).build()));
List<MediaObjectMetadataDescriptor> objectMetaData = objectMetadataReader.lookupMultimediaMetadata(objectIds);
responseObserver.onNext(QueryContainerUtil.queryResult(CineastGrpc.MediaObjectMetaDataQueryResult.newBuilder().addAllObjectMetaData(objectMetaData.stream().map(QueryContainerUtil::mediaObjectMetaData).collect(Collectors.toList())).build()));
sentObjectIds.addAll(objectIds);
}
}
/* We're done for this querycontainer */
});
// TODO Better name
qtRetrievalThread.setName("qt-stage" + stageIndex + "-" + qt.getCategories());
qtThreads.add(qtRetrievalThread);
qtRetrievalThread.start();
}
for (Thread thread : qtThreads) {
try {
thread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/* After we are done with a stage, we add all relevant segments to the config for the next stage. */
if (relevantSegments.size() == 0) {
LOGGER.warn("No relevant segments anymore, aborting staged querying");
/* Clear relevant segments (there are none) */
stageQConf.setRelevantSegmentIds(relevantSegments);
break;
}
stageQConf.setRelevantSegmentIds(relevantSegments);
relevantSegments.clear();
}
/* At this point, we have iterated over all stages. Now, we need to go back for all stages and send the results for the relevant ids. */
for (int stageIndex = 0; stageIndex < stages.size() - 1; stageIndex++) {
cache.get(stageIndex).forEach((category, results) -> {
results.removeIf(pair -> !stageQConf.getRelevantSegmentIds().contains(pair.key));
responseObserver.onNext(QueryContainerUtil.queryResult(QueryContainerUtil.similarityQueryResult(// TODO This assumes that all queries in a temporalquery have the same uuid
uuid, category, results)));
});
}
/* There should be no carry-over from this block since temporal queries are executed independently */
}
/* At this point, all StagedQueries have been executed for this TemporalQuery.
* Since results have always been sent for the final stage or, when appropriate, in intermediate steps, there's nothing left to do.
*/
responseObserver.onCompleted();
mediaSegmentReader.close();
mediaObjectReader.close();
segmentMetadataReader.close();
watch.stop();
LOGGER.debug("Query executed in {} ms", watch.getTime(TimeUnit.MILLISECONDS));
}
use of org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor in project cineast by vitrivr.
the class MetadataLookupMessageHandler method handle.
/**
* Invoked when a Message of type MetadataLookup arrives and requires handling. Looks up the MultimediaMetadataDescriptors of the requested objects, wraps them in a MediaObjectMetadataQueryResult object and writes them to the stream.
*
* @param session WebSocketSession for which the message arrived.
* @param message Message of type a that needs to be handled.
*/
@Override
public void handle(Session session, MetadataLookup message) {
Thread.currentThread().setName("metadata-lookup-handler");
MediaObjectMetadataReader reader = new MediaObjectMetadataReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());
List<MediaObjectMetadataDescriptor> descriptors = reader.lookupMultimediaMetadata(message.getIds());
this.write(session, new MediaObjectMetadataQueryResult("", descriptors));
reader.close();
}
use of org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor in project cineast by vitrivr.
the class EXIFMetadataExtractor method extract.
/**
* Extracts the metadata from the specified path and returns a List of MediaObjectMetadataDescriptor objects (one for each metadata entry).
*
* @param objectId ID of the multimedia object for which metadata will be generated.
* @param path Path to the file for which metadata should be extracted.
* @return List of MultimediaMetadataDescriptors. The list may be empty but must always be returned!
*/
@Override
public List<MediaObjectMetadataDescriptor> extract(String objectId, Path path) {
ExifSubIFDDirectory md = MetadataUtil.getMetadataDirectoryOfType(path, ExifSubIFDDirectory.class);
if (md == null) {
return Collections.emptyList();
}
Set<Entry<String, Object>> set = Maps.transformValues(FIELDS, md::getObject).entrySet();
return set.stream().filter(e -> e.getValue() != null).map(e -> MediaObjectMetadataDescriptor.of(objectId, this.domain(), e.getKey(), e.getValue())).filter(e -> !(e.getValueProvider() instanceof NothingProvider)).collect(Collectors.toList());
}
use of org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor in project cineast by vitrivr.
the class IIIFMetaDataExtractor method extract.
@Override
public List<MediaObjectMetadataDescriptor> extract(String objectId, Path path) {
File file = path.toFile();
File parent = file.getParentFile();
File jsonFile = new File(parent, file.getName() + ".iiif");
if (!jsonFile.exists()) {
jsonFile = new File(parent, com.google.common.io.Files.getNameWithoutExtension(file.getName()) + ".iiif");
}
if (!jsonFile.exists()) {
return new ArrayList<>(0);
}
@SuppressWarnings("unchecked") Map<String, Object> json = jsonProvider.toObject(jsonFile, Map.class);
if (json == null || json.isEmpty()) {
return new ArrayList<>(0);
}
ArrayList<MediaObjectMetadataDescriptor> _return = new ArrayList<>(json.size());
Set<String> keys = json.keySet();
for (String key : keys) {
_return.add(MediaObjectMetadataDescriptor.of(objectId, domain(), key, json.get(key)));
}
return _return;
}
use of org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor in project cineast by vitrivr.
the class TechnicalVideoMetadataExtractor method extract.
/**
* Extracts the technical video metadata from the specified path and returns a List of {@link MediaObjectMetadataDescriptor} objects (one for each metadata entry).
*
* @param objectId ID of the multimedia object for which metadata will be generated.
* @param path Path to the file for which metadata should be extracted.
* @return List of {@link MediaObjectMetadataDescriptor}s or an empty list, if extracting metadata fails.
*/
@Override
public List<MediaObjectMetadataDescriptor> extract(String objectId, Path path) {
final ArrayList<MediaObjectMetadataDescriptor> metadata = new ArrayList<>();
if (!Files.exists(path)) {
LOGGER.warn("File does not exist, returning empty metadata");
return metadata;
}
/* we assume that everythign which can be handled by the ffmpegvideodecoder can also be handled here. Without this safety-guard, extraction will crash with a core-dump */
if (!FFMpegVideoDecoder.supportedFiles.contains(MimeTypeHelper.getContentType(path.toString()))) {
LOGGER.warn("File is not a video, returning empty metadata");
return metadata;
}
/* Initialize the AVFormatContext. */
final AVFormatContext pFormatContext = avformat.avformat_alloc_context();
/* */
if (avformat.avformat_open_input(pFormatContext, path.toString(), null, null) != 0) {
LOGGER.error("Error while accessing file {}. Failed to obtain technical video metadata.", path.toString());
return metadata;
}
/* Retrieve stream information. */
if (avformat.avformat_find_stream_info(pFormatContext, (PointerPointer<?>) null) < 0) {
LOGGER.error("Error, Ccouldn't find stream information. Failed to obtain technical video metadata.");
return metadata;
}
final AVCodec codec = avcodec.av_codec_iterate(new Pointer());
final int videoStreamIdx = avformat.av_find_best_stream(pFormatContext, avutil.AVMEDIA_TYPE_VIDEO, -1, -1, codec, 0);
final AVStream videoStream = pFormatContext.streams(videoStreamIdx);
final AVRational timebase = videoStream.time_base();
/* Allocate new codec-context for codec returned by av_find_best_stream(). */
final AVCodecContext videoCodecContext = avcodec.avcodec_alloc_context3(codec);
avcodec.avcodec_parameters_to_context(videoCodecContext, videoStream.codecpar());
/* Open the code context. */
if (avcodec.avcodec_open2(videoCodecContext, codec, (AVDictionary) null) < 0) {
LOGGER.error("Error, Could not open video codec. Failed to obtain technical video metadata.");
return metadata;
}
/* Extract and add the video metadata to the list. */
metadata.add(new MediaObjectMetadataDescriptor(objectId, this.domain(), KEY_VIDEO_FPS, ((float) videoStream.avg_frame_rate().num() / (float) videoStream.avg_frame_rate().den()), false));
metadata.add(new MediaObjectMetadataDescriptor(objectId, this.domain(), KEY_VIDEO_DURATION, Math.floorDiv(videoStream.duration() * timebase.num() * 1000, timebase.den()), false));
metadata.add(new MediaObjectMetadataDescriptor(objectId, this.domain(), KEY_VIDEO_WIDTH, videoCodecContext.width(), false));
metadata.add(new MediaObjectMetadataDescriptor(objectId, this.domain(), KEY_VIDEO_HEIGHT, videoCodecContext.height(), false));
/* Closes all the resources. */
avcodec.avcodec_free_context(videoCodecContext);
avformat.avformat_close_input(pFormatContext);
/* Return list of results. */
return metadata;
}
Aggregations