Examples with MediaTimePoint org.opencastproject.metadata.mpeg7.MediaTimePoint used on opensource projects

Example 1 with MediaTimePoint

use of org.opencastproject.metadata.mpeg7.MediaTimePoint in project opencast by opencast.

the class Mpeg7CaptionConverter method importCaption.

/**
 * @see org.opencastproject.caption.api.CaptionConverter#importCaption(java.io.InputStream, java.lang.String)
 */
@SuppressWarnings("unchecked")
@Override
public List<Caption> importCaption(InputStream inputStream, String language) throws CaptionConverterException {
    List<Caption> captions = new ArrayList<Caption>();
    Mpeg7Catalog catalog = new Mpeg7CatalogImpl(inputStream);
    Iterator<Audio> audioContentIterator = catalog.audioContent();
    if (audioContentIterator == null)
        return captions;
    content: while (audioContentIterator.hasNext()) {
        Audio audioContent = audioContentIterator.next();
        TemporalDecomposition<AudioSegment> audioSegments = (TemporalDecomposition<AudioSegment>) audioContent.getTemporalDecomposition();
        Iterator<AudioSegment> audioSegmentIterator = audioSegments.segments();
        if (audioSegmentIterator == null)
            continue content;
        while (audioSegmentIterator.hasNext()) {
            AudioSegment segment = audioSegmentIterator.next();
            Iterator<TextAnnotation> annotationIterator = segment.textAnnotations();
            if (annotationIterator == null)
                continue content;
            while (annotationIterator.hasNext()) {
                TextAnnotation annotation = annotationIterator.next();
                if (!annotation.getLanguage().equals(language)) {
                    logger.debug("Skipping audio content '{}' because of language mismatch", audioContent.getId());
                    continue content;
                }
                List<String> captionLines = new ArrayList<String>();
                Iterator<FreeTextAnnotation> freeTextAnnotationIterator = annotation.freeTextAnnotations();
                if (freeTextAnnotationIterator == null)
                    continue;
                while (freeTextAnnotationIterator.hasNext()) {
                    FreeTextAnnotation freeTextAnnotation = freeTextAnnotationIterator.next();
                    captionLines.add(freeTextAnnotation.getText());
                }
                MediaTime segmentTime = segment.getMediaTime();
                MediaTimePoint stp = segmentTime.getMediaTimePoint();
                MediaDuration d = segmentTime.getMediaDuration();
                Calendar startCalendar = Calendar.getInstance();
                int millisAtStart = (int) (stp.getTimeInMilliseconds() - (((stp.getHour() * 60 + stp.getMinutes()) * 60 + stp.getSeconds()) * 1000));
                int millisAtEnd = (int) (d.getDurationInMilliseconds() - (((d.getHours() * 60 + d.getMinutes()) * 60 + d.getSeconds()) * 1000));
                startCalendar.set(Calendar.HOUR, stp.getHour());
                startCalendar.set(Calendar.MINUTE, stp.getMinutes());
                startCalendar.set(Calendar.SECOND, stp.getSeconds());
                startCalendar.set(Calendar.MILLISECOND, millisAtStart);
                startCalendar.add(Calendar.HOUR, d.getHours());
                startCalendar.add(Calendar.MINUTE, d.getMinutes());
                startCalendar.add(Calendar.SECOND, d.getSeconds());
                startCalendar.set(Calendar.MILLISECOND, millisAtEnd);
                try {
                    Time startTime = new TimeImpl(stp.getHour(), stp.getMinutes(), stp.getSeconds(), millisAtStart);
                    Time endTime = new TimeImpl(startCalendar.get(Calendar.HOUR), startCalendar.get(Calendar.MINUTE), startCalendar.get(Calendar.SECOND), startCalendar.get(Calendar.MILLISECOND));
                    Caption caption = new CaptionImpl(startTime, endTime, captionLines.toArray(new String[captionLines.size()]));
                    captions.add(caption);
                } catch (IllegalTimeFormatException e) {
                    logger.warn("Error setting caption time: {}", e.getMessage());
                }
            }
        }
    }
    return captions;
}

Example 2 with MediaTimePoint

use of org.opencastproject.metadata.mpeg7.MediaTimePoint in project opencast by opencast.

the class SolrIndexManager method addMpeg7Metadata.

/**
 * Add the mpeg 7 catalog data to the solr document.
 *
 * @param doc
 *          the input document to the solr index
 * @param mpeg7
 *          the mpeg7 catalog
 */
@SuppressWarnings("unchecked")
static void addMpeg7Metadata(SolrInputDocument doc, MediaPackage mediaPackage, Mpeg7Catalog mpeg7) {
    // Check for multimedia content
    if (!mpeg7.multimediaContent().hasNext()) {
        logger.warn("Mpeg-7 doesn't contain  multimedia content");
        return;
    }
    // Get the content duration by looking at the first content track. This
    // of course assumes that all tracks are equally long.
    MultimediaContent<? extends MultimediaContentType> mc = mpeg7.multimediaContent().next();
    MultimediaContentType mct = mc.elements().next();
    MediaTime mediaTime = mct.getMediaTime();
    Schema.setDcExtent(doc, mediaTime.getMediaDuration().getDurationInMilliseconds());
    // Check if the keywords have been filled by (manually) added dublin
    // core data. If not, look for the most relevant fields in mpeg-7.
    SortedSet<TextAnnotation> sortedAnnotations = null;
    if (!"".equals(Schema.getOcKeywords(doc))) {
        sortedAnnotations = new TreeSet<TextAnnotation>(new Comparator<TextAnnotation>() {

            @Override
            public int compare(TextAnnotation a1, TextAnnotation a2) {
                if ((RELEVANCE_BOOST * a1.getRelevance() + a1.getConfidence()) > (RELEVANCE_BOOST * a2.getRelevance() + a2.getConfidence()))
                    return -1;
                else if ((RELEVANCE_BOOST * a1.getRelevance() + a1.getConfidence()) < (RELEVANCE_BOOST * a2.getRelevance() + a2.getConfidence()))
                    return 1;
                return 0;
            }
        });
    }
    // Iterate over the tracks and extract keywords and hints
    Iterator<MultimediaContent<? extends MultimediaContentType>> mmIter = mpeg7.multimediaContent();
    int segmentCount = 0;
    while (mmIter.hasNext()) {
        MultimediaContent<?> multimediaContent = mmIter.next();
        // We need to process visual segments first, due to the way they are handled in the ui.
        for (Iterator<?> iterator = multimediaContent.elements(); iterator.hasNext(); ) {
            MultimediaContentType type = (MultimediaContentType) iterator.next();
            if (!(type instanceof Video) && !(type instanceof AudioVisual))
                continue;
            // for every segment in the current multimedia content track
            Video video = (Video) type;
            Iterator<VideoSegment> vsegments = (Iterator<VideoSegment>) video.getTemporalDecomposition().segments();
            while (vsegments.hasNext()) {
                VideoSegment segment = vsegments.next();
                StringBuffer segmentText = new StringBuffer();
                StringBuffer hintField = new StringBuffer();
                // Collect the video text elements to a segment text
                SpatioTemporalDecomposition spt = segment.getSpatioTemporalDecomposition();
                if (spt != null) {
                    for (VideoText videoText : spt.getVideoText()) {
                        if (segmentText.length() > 0)
                            segmentText.append(" ");
                        segmentText.append(videoText.getText().getText());
                    // TODO: Add hint on bounding box
                    }
                }
                // Add keyword annotations
                Iterator<TextAnnotation> textAnnotations = segment.textAnnotations();
                while (textAnnotations.hasNext()) {
                    TextAnnotation textAnnotation = textAnnotations.next();
                    Iterator<?> kwIter = textAnnotation.keywordAnnotations();
                    while (kwIter.hasNext()) {
                        KeywordAnnotation keywordAnnotation = (KeywordAnnotation) kwIter.next();
                        if (segmentText.length() > 0)
                            segmentText.append(" ");
                        segmentText.append(keywordAnnotation.getKeyword());
                    }
                }
                // Add free text annotations
                Iterator<TextAnnotation> freeIter = segment.textAnnotations();
                if (freeIter.hasNext()) {
                    Iterator<FreeTextAnnotation> freeTextIter = freeIter.next().freeTextAnnotations();
                    while (freeTextIter.hasNext()) {
                        FreeTextAnnotation freeTextAnnotation = freeTextIter.next();
                        if (segmentText.length() > 0)
                            segmentText.append(" ");
                        segmentText.append(freeTextAnnotation.getText());
                    }
                }
                // add segment text to solr document
                Schema.setSegmentText(doc, new DField<String>(segmentText.toString(), Integer.toString(segmentCount)));
                // get the segments time properties
                MediaTimePoint timepoint = segment.getMediaTime().getMediaTimePoint();
                MediaDuration duration = segment.getMediaTime().getMediaDuration();
                // TODO: define a class with hint field constants
                hintField.append("time=" + timepoint.getTimeInMilliseconds() + "\n");
                hintField.append("duration=" + duration.getDurationInMilliseconds() + "\n");
                // Look for preview images. Their characteristics are that they are
                // attached as attachments with a flavor of preview/<something>.
                String time = timepoint.toString();
                for (Attachment slide : mediaPackage.getAttachments(MediaPackageElements.PRESENTATION_SEGMENT_PREVIEW)) {
                    MediaPackageReference ref = slide.getReference();
                    if (ref != null && time.equals(ref.getProperty("time"))) {
                        hintField.append("preview");
                        hintField.append(".");
                        hintField.append(ref.getIdentifier());
                        hintField.append("=");
                        hintField.append(slide.getURI().toString());
                        hintField.append("\n");
                    }
                }
                logger.trace("Adding segment: " + timepoint.toString());
                Schema.setSegmentHint(doc, new DField<String>(hintField.toString(), Integer.toString(segmentCount)));
                // increase segment counter
                segmentCount++;
            }
        }
    }
    // Put the most important keywords into a special solr field
    if (sortedAnnotations != null) {
        Schema.setOcKeywords(doc, importantKeywordsString(sortedAnnotations).toString());
    }
}

Example 3 with MediaTimePoint

use of org.opencastproject.metadata.mpeg7.MediaTimePoint in project opencast by opencast.

the class SegmentPreviewsWorkflowOperationHandler method createPreviews.

/**
 * Encode tracks from MediaPackage using profiles stored in properties and updates current MediaPackage.
 *
 * @param mediaPackage
 * @param properties
 * @return the operation result containing the updated mediapackage
 * @throws EncoderException
 * @throws ExecutionException
 * @throws InterruptedException
 * @throws IOException
 * @throws NotFoundException
 * @throws WorkflowOperationException
 */
private WorkflowOperationResult createPreviews(final MediaPackage mediaPackage, WorkflowOperationInstance operation) throws EncoderException, InterruptedException, ExecutionException, NotFoundException, MediaPackageException, IOException, WorkflowOperationException {
    long totalTimeInQueue = 0;
    // Read the configuration properties
    String sourceVideoFlavor = StringUtils.trimToNull(operation.getConfiguration("source-flavor"));
    String sourceTags = StringUtils.trimToNull(operation.getConfiguration("source-tags"));
    String targetImageTags = StringUtils.trimToNull(operation.getConfiguration("target-tags"));
    String targetImageFlavor = StringUtils.trimToNull(operation.getConfiguration("target-flavor"));
    String encodingProfileName = StringUtils.trimToNull(operation.getConfiguration("encoding-profile"));
    String referenceFlavor = StringUtils.trimToNull(operation.getConfiguration("reference-flavor"));
    String referenceTags = StringUtils.trimToNull(operation.getConfiguration("reference-tags"));
    // Find the encoding profile
    EncodingProfile profile = composerService.getProfile(encodingProfileName);
    if (profile == null)
        throw new IllegalStateException("Encoding profile '" + encodingProfileName + "' was not found");
    List<String> sourceTagSet = asList(sourceTags);
    // Select the tracks based on the tags and flavors
    Set<Track> videoTrackSet = new HashSet<>();
    for (Track track : mediaPackage.getTracksByTags(sourceTagSet)) {
        if (sourceVideoFlavor == null || (track.getFlavor() != null && sourceVideoFlavor.equals(track.getFlavor().toString()))) {
            if (!track.hasVideo())
                continue;
            videoTrackSet.add(track);
        }
    }
    if (videoTrackSet.size() == 0) {
        logger.debug("Mediapackage {} has no suitable tracks to extract images based on tags {} and flavor {}", mediaPackage, sourceTags, sourceVideoFlavor);
        return createResult(mediaPackage, Action.CONTINUE);
    } else {
        // Determine the tagset for the reference
        List<String> referenceTagSet = asList(referenceTags);
        // Determine the reference master
        for (Track t : videoTrackSet) {
            // Try to load the segments catalog
            MediaPackageReference trackReference = new MediaPackageReferenceImpl(t);
            Catalog[] segmentCatalogs = mediaPackage.getCatalogs(MediaPackageElements.SEGMENTS, trackReference);
            Mpeg7Catalog mpeg7 = null;
            if (segmentCatalogs.length > 0) {
                mpeg7 = loadMpeg7Catalog(segmentCatalogs[0]);
                if (segmentCatalogs.length > 1)
                    logger.warn("More than one segments catalog found for track {}. Resuming with the first one ({})", t, mpeg7);
            } else {
                logger.debug("No segments catalog found for track {}", t);
                continue;
            }
            // Check the catalog's consistency
            if (mpeg7.videoContent() == null || mpeg7.videoContent().next() == null) {
                logger.info("Segments catalog {} contains no video content", mpeg7);
                continue;
            }
            Video videoContent = mpeg7.videoContent().next();
            TemporalDecomposition<? extends Segment> decomposition = videoContent.getTemporalDecomposition();
            // Are there any segments?
            if (decomposition == null || !decomposition.hasSegments()) {
                logger.info("Segments catalog {} contains no video content", mpeg7);
                continue;
            }
            // Is a derived track with the configured reference flavor available?
            MediaPackageElement referenceMaster = getReferenceMaster(mediaPackage, t, referenceFlavor, referenceTagSet);
            // Create the preview images according to the mpeg7 segments
            if (t.hasVideo() && mpeg7 != null) {
                Iterator<? extends Segment> segmentIterator = decomposition.segments();
                List<MediaTimePoint> timePointList = new LinkedList<>();
                while (segmentIterator.hasNext()) {
                    Segment segment = segmentIterator.next();
                    MediaTimePoint tp = segment.getMediaTime().getMediaTimePoint();
                    timePointList.add(tp);
                }
                // convert to time array
                double[] timeArray = new double[timePointList.size()];
                for (int i = 0; i < timePointList.size(); i++) timeArray[i] = (double) timePointList.get(i).getTimeInMilliseconds() / 1000;
                Job job = composerService.image(t, profile.getIdentifier(), timeArray);
                if (!waitForStatus(job).isSuccess()) {
                    throw new WorkflowOperationException("Extracting preview image from " + t + " failed");
                }
                // Get the latest copy
                try {
                    job = serviceRegistry.getJob(job.getId());
                } catch (ServiceRegistryException e) {
                    throw new WorkflowOperationException(e);
                }
                // add this receipt's queue time to the total
                totalTimeInQueue += job.getQueueTime();
                List<? extends MediaPackageElement> composedImages = MediaPackageElementParser.getArrayFromXml(job.getPayload());
                Iterator<MediaTimePoint> it = timePointList.iterator();
                for (MediaPackageElement element : composedImages) {
                    Attachment composedImage = (Attachment) element;
                    if (composedImage == null)
                        throw new IllegalStateException("Unable to compose image");
                    // Add the flavor, either from the operation configuration or from the composer
                    if (targetImageFlavor != null) {
                        composedImage.setFlavor(MediaPackageElementFlavor.parseFlavor(targetImageFlavor));
                        logger.debug("Preview image has flavor '{}'", composedImage.getFlavor());
                    }
                    // Set the mimetype
                    if (profile.getMimeType() != null)
                        composedImage.setMimeType(MimeTypes.parseMimeType(profile.getMimeType()));
                    // Add tags
                    for (String tag : asList(targetImageTags)) {
                        logger.trace("Tagging image with '{}'", tag);
                        composedImage.addTag(tag);
                    }
                    // Refer to the original track including a timestamp
                    MediaPackageReferenceImpl ref = new MediaPackageReferenceImpl(referenceMaster);
                    ref.setProperty("time", it.next().toString());
                    composedImage.setReference(ref);
                    // store new image in the mediaPackage
                    mediaPackage.add(composedImage);
                    String fileName = getFileNameFromElements(t, composedImage);
                    composedImage.setURI(workspace.moveTo(composedImage.getURI(), mediaPackage.getIdentifier().toString(), composedImage.getIdentifier(), fileName));
                }
            }
        }
    }
    return createResult(mediaPackage, Action.CONTINUE, totalTimeInQueue);
}

Example 4 with MediaTimePoint

use of org.opencastproject.metadata.mpeg7.MediaTimePoint in project opencast by opencast.

the class TextAnalysisWorkflowOperationHandler method extractVideoText.

/**
 * Runs the text analysis service on each of the video segments found.
 *
 * @param mediaPackage
 *          the original mediapackage
 * @param operation
 *          the workflow operation
 * @throws ExecutionException
 * @throws InterruptedException
 * @throws NotFoundException
 * @throws WorkflowOperationException
 */
protected WorkflowOperationResult extractVideoText(final MediaPackage mediaPackage, WorkflowOperationInstance operation) throws EncoderException, InterruptedException, ExecutionException, IOException, NotFoundException, MediaPackageException, TextAnalyzerException, WorkflowOperationException, ServiceRegistryException {
    long totalTimeInQueue = 0;
    List<String> sourceTagSet = asList(operation.getConfiguration("source-tags"));
    List<String> targetTagSet = asList(operation.getConfiguration("target-tags"));
    // Select the catalogs according to the tags
    Map<Catalog, Mpeg7Catalog> catalogs = loadSegmentCatalogs(mediaPackage, operation);
    // Was there at least one matching catalog
    if (catalogs.size() == 0) {
        logger.debug("Mediapackage {} has no suitable mpeg-7 catalogs based on tags {} to to run text analysis", mediaPackage, sourceTagSet);
        return createResult(mediaPackage, Action.CONTINUE);
    }
    // Loop over all existing segment catalogs
    for (Entry<Catalog, Mpeg7Catalog> mapEntry : catalogs.entrySet()) {
        Map<VideoSegment, Job> jobs = new HashMap<VideoSegment, Job>();
        List<Attachment> images = new LinkedList<Attachment>();
        Catalog segmentCatalog = mapEntry.getKey();
        try {
            MediaPackageReference catalogRef = segmentCatalog.getReference();
            // Make sure we can figure out the source track
            if (catalogRef == null) {
                logger.info("Skipping catalog {} since we can't determine the source track", segmentCatalog);
            } else if (mediaPackage.getElementByReference(catalogRef) == null) {
                logger.info("Skipping catalog {} since we can't determine the source track", segmentCatalog);
            } else if (!(mediaPackage.getElementByReference(catalogRef) instanceof Track)) {
                logger.info("Skipping catalog {} since it's source was not a track", segmentCatalog);
            }
            logger.info("Analyzing mpeg-7 segments catalog {} for text", segmentCatalog);
            // Create a copy that will contain the segments enriched with the video text elements
            Mpeg7Catalog textCatalog = mapEntry.getValue().clone();
            Track sourceTrack = mediaPackage.getTrack(catalogRef.getIdentifier());
            // Load the temporal decomposition (segments)
            Video videoContent = textCatalog.videoContent().next();
            TemporalDecomposition<? extends Segment> decomposition = videoContent.getTemporalDecomposition();
            Iterator<? extends Segment> segmentIterator = decomposition.segments();
            // For every segment, try to find the still image and run text analysis on it
            List<VideoSegment> videoSegments = new LinkedList<VideoSegment>();
            while (segmentIterator.hasNext()) {
                Segment segment = segmentIterator.next();
                if ((segment instanceof VideoSegment))
                    videoSegments.add((VideoSegment) segment);
            }
            // argument array for image extraction
            long[] times = new long[videoSegments.size()];
            for (int i = 0; i < videoSegments.size(); i++) {
                VideoSegment videoSegment = videoSegments.get(i);
                MediaTimePoint segmentTimePoint = videoSegment.getMediaTime().getMediaTimePoint();
                MediaDuration segmentDuration = videoSegment.getMediaTime().getMediaDuration();
                // Choose a time
                MediaPackageReference reference = null;
                if (catalogRef == null)
                    reference = new MediaPackageReferenceImpl();
                else
                    reference = new MediaPackageReferenceImpl(catalogRef.getType(), catalogRef.getIdentifier());
                reference.setProperty("time", segmentTimePoint.toString());
                // Have the time for ocr image created. To circumvent problems with slowly building slides, we take the image
                // that is
                // almost at the end of the segment, it should contain the most content and is stable as well.
                long startTimeSeconds = segmentTimePoint.getTimeInMilliseconds() / 1000;
                long durationSeconds = segmentDuration.getDurationInMilliseconds() / 1000;
                times[i] = Math.max(startTimeSeconds + durationSeconds - stabilityThreshold + 1, 0);
            }
            // Have the ocr image(s) created.
            // TODO: Note that the way of having one image extracted after the other is suited for
            // the ffmpeg-based encoder. When switching to other encoding engines such as gstreamer, it might be preferable
            // to pass in all timepoints to the image extraction method at once.
            SortedMap<Long, Job> extractImageJobs = new TreeMap<Long, Job>();
            try {
                for (long time : times) {
                    extractImageJobs.put(time, composer.image(sourceTrack, IMAGE_EXTRACTION_PROFILE, time));
                }
                if (!waitForStatus(extractImageJobs.values().toArray(new Job[extractImageJobs.size()])).isSuccess())
                    throw new WorkflowOperationException("Extracting scene image from " + sourceTrack + " failed");
                for (Map.Entry<Long, Job> entry : extractImageJobs.entrySet()) {
                    Job job = serviceRegistry.getJob(entry.getValue().getId());
                    Attachment image = (Attachment) MediaPackageElementParser.getFromXml(job.getPayload());
                    images.add(image);
                    totalTimeInQueue += job.getQueueTime();
                }
            } catch (EncoderException e) {
                logger.error("Error creating still image(s) from {}", sourceTrack);
                throw e;
            }
            // Run text extraction on each of the images
            Iterator<VideoSegment> it = videoSegments.iterator();
            for (MediaPackageElement element : images) {
                Attachment image = (Attachment) element;
                VideoSegment videoSegment = it.next();
                jobs.put(videoSegment, analysisService.extract(image));
            }
            // Wait for all jobs to be finished
            if (!waitForStatus(jobs.values().toArray(new Job[jobs.size()])).isSuccess()) {
                throw new WorkflowOperationException("Text extraction failed on images from " + sourceTrack);
            }
            // Process the text extraction results
            for (Map.Entry<VideoSegment, Job> entry : jobs.entrySet()) {
                Job job = serviceRegistry.getJob(entry.getValue().getId());
                totalTimeInQueue += job.getQueueTime();
                VideoSegment videoSegment = entry.getKey();
                MediaDuration segmentDuration = videoSegment.getMediaTime().getMediaDuration();
                Catalog catalog = (Catalog) MediaPackageElementParser.getFromXml(job.getPayload());
                if (catalog == null) {
                    logger.warn("Text analysis did not return a valid mpeg7 for segment {}", videoSegment);
                    continue;
                }
                Mpeg7Catalog videoTextCatalog = loadMpeg7Catalog(catalog);
                if (videoTextCatalog == null)
                    throw new IllegalStateException("Text analysis service did not return a valid mpeg7");
                // Add the spatiotemporal decompositions from the new catalog to the existing video segments
                Iterator<Video> videoTextContents = videoTextCatalog.videoContent();
                if (videoTextContents == null || !videoTextContents.hasNext()) {
                    logger.debug("Text analysis was not able to extract any text from {}", job.getArguments().get(0));
                    break;
                }
                try {
                    Video textVideoContent = videoTextContents.next();
                    VideoSegment textVideoSegment = (VideoSegment) textVideoContent.getTemporalDecomposition().segments().next();
                    VideoText[] videoTexts = textVideoSegment.getSpatioTemporalDecomposition().getVideoText();
                    SpatioTemporalDecomposition std = videoSegment.createSpatioTemporalDecomposition(true, false);
                    for (VideoText videoText : videoTexts) {
                        MediaTime mediaTime = new MediaTimeImpl(new MediaRelTimePointImpl(0), segmentDuration);
                        SpatioTemporalLocator locator = new SpatioTemporalLocatorImpl(mediaTime);
                        videoText.setSpatioTemporalLocator(locator);
                        std.addVideoText(videoText);
                    }
                } catch (Exception e) {
                    logger.warn("The mpeg-7 structure returned by the text analyzer is not what is expected", e);
                    continue;
                }
            }
            // Put the catalog into the workspace and add it to the media package
            MediaPackageElementBuilder builder = MediaPackageElementBuilderFactory.newInstance().newElementBuilder();
            Catalog catalog = (Catalog) builder.newElement(MediaPackageElement.Type.Catalog, MediaPackageElements.TEXTS);
            catalog.setIdentifier(null);
            catalog.setReference(segmentCatalog.getReference());
            // the catalog now has an ID, so we can store the file properly
            mediaPackage.add(catalog);
            InputStream in = mpeg7CatalogService.serialize(textCatalog);
            String filename = "slidetext.xml";
            URI workspaceURI = workspace.put(mediaPackage.getIdentifier().toString(), catalog.getIdentifier(), filename, in);
            catalog.setURI(workspaceURI);
            // Since we've enriched and stored the mpeg7 catalog, remove the original
            try {
                mediaPackage.remove(segmentCatalog);
                workspace.delete(segmentCatalog.getURI());
            } catch (Exception e) {
                logger.warn("Unable to delete segment catalog {}: {}", segmentCatalog.getURI(), e);
            }
            // Add flavor and target tags
            catalog.setFlavor(MediaPackageElements.TEXTS);
            for (String tag : targetTagSet) {
                catalog.addTag(tag);
            }
        } finally {
            // Remove images that were created for text extraction
            logger.debug("Removing temporary images");
            for (Attachment image : images) {
                try {
                    workspace.delete(image.getURI());
                } catch (Exception e) {
                    logger.warn("Unable to delete temporary image {}: {}", image.getURI(), e);
                }
            }
            // Remove the temporary text
            for (Job j : jobs.values()) {
                Catalog catalog = null;
                try {
                    Job job = serviceRegistry.getJob(j.getId());
                    if (!Job.Status.FINISHED.equals(job.getStatus()))
                        continue;
                    catalog = (Catalog) MediaPackageElementParser.getFromXml(job.getPayload());
                    if (catalog != null)
                        workspace.delete(catalog.getURI());
                } catch (Exception e) {
                    if (catalog != null) {
                        logger.warn("Unable to delete temporary text file {}: {}", catalog.getURI(), e);
                    } else {
                        logger.warn("Unable to parse textextraction payload of job {}", j.getId());
                    }
                }
            }
        }
    }
    logger.debug("Text analysis completed");
    return createResult(mediaPackage, Action.CONTINUE, totalTimeInQueue);
}

Example 5 with MediaTimePoint

use of org.opencastproject.metadata.mpeg7.MediaTimePoint in project opencast by opencast.

the class VideoSegmenterServiceImpl method filterSegmentation.

/**
 * Merges small subsequent segments (with high difference) into a bigger one
 *
 * @param segments list of segments to be filtered
 * @param track the track that is segmented
 * @param segmentsNew will be set to list of new segments (pass null if not required)
 * @param mergeThresh minimum duration for a segment in milliseconds
 * @return Mpeg7Catalog that can later be saved in a Catalog as endresult
 */
protected Mpeg7Catalog filterSegmentation(LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew, int mergeThresh) {
    if (segmentsNew == null) {
        segmentsNew = new LinkedList<Segment>();
    }
    boolean merging = false;
    MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
    MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
    Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
    Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
    int segmentcount = 1;
    MediaTimePoint currentSegStart = new MediaTimePointImpl();
    for (Segment o : segments) {
        // if the current segment is shorter than merge treshold start merging
        if (o.getMediaTime().getMediaDuration().getDurationInMilliseconds() <= mergeThresh) {
            // start merging and save beginning of new segment that will be generated
            if (!merging) {
                currentSegStart = o.getMediaTime().getMediaTimePoint();
                merging = true;
            }
        // current segment is longer than merge threshold
        } else {
            long currentSegDuration = o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
            long currentSegEnd = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds() + currentSegDuration;
            if (merging) {
                long newDuration = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds() - currentSegStart.getTimeInMilliseconds();
                // save new segment that merges all previously skipped short segments
                if (newDuration >= mergeThresh) {
                    Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
                    s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
                    segmentsNew.add(s);
                    // copy the following long segment to new list
                    Segment s2 = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
                    s2.setMediaTime(o.getMediaTime());
                    segmentsNew.add(s2);
                // if too short split new segment in middle and merge halves to
                // previous and following segments
                } else {
                    long followingStartOld = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
                    long newSplit = (currentSegStart.getTimeInMilliseconds() + followingStartOld) / 2;
                    long followingEnd = followingStartOld + o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
                    long followingDuration = followingEnd - newSplit;
                    // if at beginning, don't split, just merge to first large segment
                    if (segmentsNew.isEmpty()) {
                        Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
                        s.setMediaTime(new MediaRelTimeImpl(0, followingEnd));
                        segmentsNew.add(s);
                    } else {
                        long previousStart = segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
                        // adjust end time of previous segment to split time
                        segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(previousStart, newSplit - previousStart));
                        // create new segment starting at split time
                        Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
                        s.setMediaTime(new MediaRelTimeImpl(newSplit, followingDuration));
                        segmentsNew.add(s);
                    }
                }
                merging = false;
            // copy segments that are long enough to new list (with corrected number)
            } else {
                Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
                s.setMediaTime(o.getMediaTime());
                segmentsNew.add(s);
            }
        }
    }
    // if there is an unfinished merging process after going through all segments
    if (merging && !segmentsNew.isEmpty()) {
        long newDuration = track.getDuration() - currentSegStart.getTimeInMilliseconds();
        // if merged segment is long enough, create new segment
        if (newDuration >= mergeThresh) {
            Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount);
            s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
            segmentsNew.add(s);
        // if not long enough, merge with previous segment
        } else {
            newDuration = track.getDuration() - segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
            segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds(), newDuration));
        }
    }
    // segment spanning the whole video
    if (segmentsNew.isEmpty()) {
        Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount);
        s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
        segmentsNew.add(s);
    }
    return mpeg7;
}