use of org.opencastproject.metadata.mpeg7.MediaLocatorImpl in project opencast by opencast.
the class VideoSegmenterServiceImpl method segment.
/**
* Starts segmentation on the video track identified by
* <code>mediapackageId</code> and <code>elementId</code> and returns a
* receipt containing the final result in the form of anMpeg7Catalog.
*
* @param track
* the element to analyze
* @return a receipt containing the resulting mpeg-7 catalog
* @throws VideoSegmenterException
*/
protected Catalog segment(Job job, Track track) throws VideoSegmenterException, MediaPackageException {
// implementation
if (!track.hasVideo()) {
logger.warn("Element {} is not a video track", track);
throw new VideoSegmenterException("Element is not a video track");
}
try {
Mpeg7Catalog mpeg7;
File mediaFile = null;
URL mediaUrl = null;
try {
mediaFile = workspace.get(track.getURI());
mediaUrl = mediaFile.toURI().toURL();
} catch (NotFoundException e) {
throw new VideoSegmenterException("Error finding the video file in the workspace", e);
} catch (IOException e) {
throw new VideoSegmenterException("Error reading the video file in the workspace", e);
}
if (track.getDuration() == null)
throw new MediaPackageException("Track " + track + " does not have a duration");
logger.info("Track {} loaded, duration is {} s", mediaUrl, track.getDuration() / 1000);
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Video videoContent;
logger.debug("changesThreshold: {}, stabilityThreshold: {}", changesThreshold, stabilityThreshold);
logger.debug("prefNumber: {}, maxCycles: {}", prefNumber, maxCycles);
boolean endOptimization = false;
int cycleCount = 0;
LinkedList<Segment> segments;
LinkedList<OptimizationStep> optimizationList = new LinkedList<OptimizationStep>();
LinkedList<OptimizationStep> unusedResultsList = new LinkedList<OptimizationStep>();
OptimizationStep stepBest = new OptimizationStep();
// local copy of changesThreshold, that can safely be changed over optimization iterations
float changesThresholdLocal = changesThreshold;
// local copies of prefNumber, absoluteMin and absoluteMax, to make a dependency on track length possible
int prefNumberLocal = prefNumber;
int absoluteMaxLocal = absoluteMax;
int absoluteMinLocal = absoluteMin;
// absoluteMax and absoluteMin with the duration of the track
if (durationDependent) {
double trackDurationInHours = track.getDuration() / 3600000.0;
prefNumberLocal = (int) Math.round(trackDurationInHours * prefNumberLocal);
absoluteMaxLocal = (int) Math.round(trackDurationInHours * absoluteMax);
absoluteMinLocal = (int) Math.round(trackDurationInHours * absoluteMin);
// make sure prefNumberLocal will never be 0 or negative
if (prefNumberLocal <= 0) {
prefNumberLocal = 1;
}
logger.info("Numbers of segments are set to be relative to track duration. Therefore for {} the preferred " + "number of segments is {}", mediaUrl, prefNumberLocal);
}
logger.info("Starting video segmentation of {}", mediaUrl);
// to the desired number of segments
while (!endOptimization) {
mpeg7 = mpeg7CatalogService.newInstance();
videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
// run the segmentation with FFmpeg
segments = runSegmentationFFmpeg(track, videoContent, mediaFile, changesThresholdLocal);
// calculate errors for "normal" and filtered segmentation
// and compare them to find better optimization.
// "normal"
OptimizationStep currentStep = new OptimizationStep(stabilityThreshold, changesThresholdLocal, segments.size(), prefNumberLocal, mpeg7, segments);
// filtered
LinkedList<Segment> segmentsNew = new LinkedList<Segment>();
OptimizationStep currentStepFiltered = new OptimizationStep(stabilityThreshold, changesThresholdLocal, 0, prefNumberLocal, filterSegmentation(segments, track, segmentsNew, stabilityThreshold * 1000), segments);
currentStepFiltered.setSegmentNumAndRecalcErrors(segmentsNew.size());
logger.info("Segmentation yields {} segments after filtering", segmentsNew.size());
OptimizationStep currentStepBest;
// - and the filtered segmentation is not already better than the maximum error
if (currentStep.getErrorAbs() <= currentStepFiltered.getErrorAbs() || (segmentsNew.size() < prefNumberLocal && currentStep.getSegmentNum() > (track.getDuration() / 1000.0f) / (stabilityThreshold / 2) && !(currentStepFiltered.getErrorAbs() <= maxError))) {
optimizationList.add(currentStep);
Collections.sort(optimizationList);
currentStepBest = currentStep;
unusedResultsList.add(currentStepFiltered);
} else {
optimizationList.add(currentStepFiltered);
Collections.sort(optimizationList);
currentStepBest = currentStepFiltered;
}
cycleCount++;
logger.debug("errorAbs = {}, error = {}", currentStep.getErrorAbs(), currentStep.getError());
logger.debug("changesThreshold = {}", changesThresholdLocal);
logger.debug("cycleCount = {}", cycleCount);
// end optimization if maximum number of cycles is reached or if the segmentation is good enough
if (cycleCount >= maxCycles || currentStepBest.getErrorAbs() <= maxError) {
endOptimization = true;
if (optimizationList.size() > 0) {
if (optimizationList.getFirst().getErrorAbs() <= optimizationList.getLast().getErrorAbs() && optimizationList.getFirst().getError() >= 0) {
stepBest = optimizationList.getFirst();
} else {
stepBest = optimizationList.getLast();
}
}
// just to be sure, check if one of the unused results was better
for (OptimizationStep currentUnusedStep : unusedResultsList) {
if (currentUnusedStep.getErrorAbs() < stepBest.getErrorAbs()) {
stepBest = unusedResultsList.getFirst();
}
}
// continue optimization, calculate new changes threshold for next iteration of optimization
} else {
OptimizationStep first = optimizationList.getFirst();
OptimizationStep last = optimizationList.getLast();
// estimate a new changesThreshold based on the one yielding the smallest error
if (optimizationList.size() == 1 || first.getError() < 0 || last.getError() > 0) {
if (currentStepBest.getError() >= 0) {
// if the error is smaller or equal to 1, increase changes threshold weighted with the error
if (currentStepBest.getError() <= 1) {
changesThresholdLocal += changesThresholdLocal * currentStepBest.getError();
} else {
// to faster reach reasonable segment numbers
if (cycleCount <= 1 && currentStep.getSegmentNum() > 2000) {
changesThresholdLocal = 0.2f;
// if the error is bigger than one, double the changes threshold, because multiplying
// with a large error can yield a much too high changes threshold
} else {
changesThresholdLocal *= 2;
}
}
} else {
changesThresholdLocal /= 2;
}
logger.debug("onesided optimization yields new changesThreshold = {}", changesThresholdLocal);
// if there are already iterations with positive and negative errors, choose a changesThreshold between those
} else {
// for simplicity a linear relationship between the changesThreshold
// and the number of generated segments is assumed and based on that
// the expected correct changesThreshold is calculated
// the new changesThreshold is calculated by averaging the the mean and the mean weighted with errors
// because this seemed to yield better results in several cases
float x = (first.getSegmentNum() - prefNumberLocal) / (float) (first.getSegmentNum() - last.getSegmentNum());
float newX = ((x + 0.5f) * 0.5f);
changesThresholdLocal = first.getChangesThreshold() * (1 - newX) + last.getChangesThreshold() * newX;
logger.debug("doublesided optimization yields new changesThreshold = {}", changesThresholdLocal);
}
}
}
// after optimization of the changes threshold, the minimum duration for a segment
// (stability threshold) is optimized if the result is still not good enough
int threshLow = stabilityThreshold * 1000;
int threshHigh = threshLow + (threshLow / 2);
LinkedList<Segment> tmpSegments;
float smallestError = Float.MAX_VALUE;
int bestI = threshLow;
segments = stepBest.getSegments();
// is smaller than the maximum error, the stability threshold will not be optimized
if (stepBest.getError() <= maxError) {
threshHigh = stabilityThreshold * 1000;
}
for (int i = threshLow; i <= threshHigh; i = i + 1000) {
tmpSegments = new LinkedList<Segment>();
filterSegmentation(segments, track, tmpSegments, i);
float newError = OptimizationStep.calculateErrorAbs(tmpSegments.size(), prefNumberLocal);
if (newError < smallestError) {
smallestError = newError;
bestI = i;
}
}
tmpSegments = new LinkedList<Segment>();
mpeg7 = filterSegmentation(segments, track, tmpSegments, bestI);
// for debugging: output of final segmentation after optimization
logger.debug("result segments:");
for (int i = 0; i < tmpSegments.size(); i++) {
int[] tmpLog2 = new int[7];
tmpLog2[0] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getHour();
tmpLog2[1] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getMinutes();
tmpLog2[2] = tmpSegments.get(i).getMediaTime().getMediaTimePoint().getSeconds();
tmpLog2[3] = tmpSegments.get(i).getMediaTime().getMediaDuration().getHours();
tmpLog2[4] = tmpSegments.get(i).getMediaTime().getMediaDuration().getMinutes();
tmpLog2[5] = tmpSegments.get(i).getMediaTime().getMediaDuration().getSeconds();
Object[] tmpLog1 = { tmpLog2[0], tmpLog2[1], tmpLog2[2], tmpLog2[3], tmpLog2[4], tmpLog2[5], tmpLog2[6] };
tmpLog1[6] = tmpSegments.get(i).getIdentifier();
logger.debug("s:{}:{}:{}, d:{}:{}:{}, {}", tmpLog1);
}
logger.info("Optimized Segmentation yields (after {} iteration" + (cycleCount == 1 ? "" : "s") + ") {} segments", cycleCount, tmpSegments.size());
// if no reasonable segmentation could be found, instead return a uniform segmentation
if (tmpSegments.size() < absoluteMinLocal || tmpSegments.size() > absoluteMaxLocal) {
mpeg7 = uniformSegmentation(track, tmpSegments, prefNumberLocal);
logger.info("Since no reasonable segmentation could be found, a uniform segmentation was created");
}
Catalog mpeg7Catalog = (Catalog) MediaPackageElementBuilderFactory.newInstance().newElementBuilder().newElement(Catalog.TYPE, MediaPackageElements.SEGMENTS);
URI uri;
try {
uri = workspace.putInCollection(COLLECTION_ID, job.getId() + ".xml", mpeg7CatalogService.serialize(mpeg7));
} catch (IOException e) {
throw new VideoSegmenterException("Unable to put the mpeg7 catalog into the workspace", e);
}
mpeg7Catalog.setURI(uri);
logger.info("Finished video segmentation of {}", mediaUrl);
return mpeg7Catalog;
} catch (Exception e) {
logger.warn("Error segmenting " + track, e);
if (e instanceof VideoSegmenterException) {
throw (VideoSegmenterException) e;
} else {
throw new VideoSegmenterException(e);
}
}
}
use of org.opencastproject.metadata.mpeg7.MediaLocatorImpl in project opencast by opencast.
the class VideoSegmenterTest method testAnalyzeSegmentMerging.
@Test
public void testAnalyzeSegmentMerging() {
Mpeg7CatalogService mpeg7catalogService = vsegmenter.mpeg7CatalogService;
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Mpeg7Catalog mpeg7 = mpeg7catalogService.newInstance();
Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
LinkedList<Segment> segments;
LinkedList<Segment> result;
int segmentcount = 1;
track.setDuration(47000L);
// list of segment durations (starttimes can be calculated from those)
int[] segmentArray1 = { 3000, 2000, 8000, 3000, 1000, 6000, 3000, 2000, 4000, 11000, 2000, 2000 };
int[] segmentArray2 = { 1000, 2000, 8000, 3000, 1000, 6000, 3000, 2000, 4000, 11000, 2000, 4000 };
int[] segmentArray3 = { 1000, 2000, 4000, 3000, 1000, 2000, 3000, 2000, 4000, 1000, 2000, 4000 };
int[] segmentArray4 = { 6000, 7000, 13000, 9000, 8000, 11000, 5000, 16000 };
// predicted outcome of filtering the segmentation
int[] prediction1 = { 5000, 10000, 8000, 9000, 15000 };
int[] prediction2 = { 13000, 8000, 9000, 11000, 6000 };
int[] prediction3 = { 29000 };
int[] prediction4 = { 6000, 7000, 13000, 9000, 8000, 11000, 5000, 16000 };
// total duration of respective segment arrays
long duration1 = 47000L;
long duration2 = 47000L;
long duration3 = 29000L;
long duration4 = 75000L;
int[][] segmentArray = { segmentArray1, segmentArray2, segmentArray3, segmentArray4 };
int[][] prediction = { prediction1, prediction2, prediction3, prediction4 };
long[] durations = { duration1, duration2, duration3, duration4 };
// check for all test segmentations if "filterSegmentation" yields the expected result
for (int k = 0; k < segmentArray.length; k++) {
segments = new LinkedList<Segment>();
result = new LinkedList<Segment>();
track.setDuration(durations[k]);
int previous = 0;
for (int i = 0; i < segmentArray[k].length; i++) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(previous, segmentArray[k][i]));
segments.add(s);
previous += segmentArray[k][i];
}
vsegmenter.filterSegmentation(segments, track, result, 5000);
assertEquals("segment merging yields wrong number of segments", prediction[k].length, result.size());
previous = 0;
for (int i = 0; i < prediction[k].length; i++) {
String message = "segment " + i + " in set " + k + " has the wrong start time.";
String message1 = "segment " + i + " in set " + k + " has the wrong duration.";
assertEquals(message, previous, result.get(i).getMediaTime().getMediaTimePoint().getTimeInMilliseconds());
assertEquals(message1, prediction[k][i], result.get(i).getMediaTime().getMediaDuration().getDurationInMilliseconds());
previous += prediction[k][i];
}
}
}
use of org.opencastproject.metadata.mpeg7.MediaLocatorImpl in project opencast by opencast.
the class VideoSegmenterServiceImpl method uniformSegmentation.
/**
* Creates a uniform segmentation for a given track, with prefNumber as the number of segments
* which will all have the same length
*
* @param track the track that is segmented
* @param segmentsNew will be set to list of new segments (pass null if not required)
* @param prefNumber number of generated segments
* @return Mpeg7Catalog that can later be saved in a Catalog as endresult
*/
protected Mpeg7Catalog uniformSegmentation(Track track, LinkedList<Segment> segmentsNew, int prefNumber) {
if (segmentsNew == null) {
segmentsNew = new LinkedList<Segment>();
}
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
long segmentDuration = track.getDuration() / prefNumber;
long currentSegStart = 0;
// create "prefNumber"-many segments that all have the same length
for (int i = 1; i < prefNumber; i++) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + i);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart, segmentDuration));
segmentsNew.add(s);
currentSegStart += segmentDuration;
}
// add last segment separately to make sure the last segment ends exactly at the end of the track
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + prefNumber);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart, track.getDuration() - currentSegStart));
segmentsNew.add(s);
return mpeg7;
}
use of org.opencastproject.metadata.mpeg7.MediaLocatorImpl in project opencast by opencast.
the class VideoSegmenterServiceImpl method filterSegmentation.
/**
* Merges small subsequent segments (with high difference) into a bigger one
*
* @param segments list of segments to be filtered
* @param track the track that is segmented
* @param segmentsNew will be set to list of new segments (pass null if not required)
* @param mergeThresh minimum duration for a segment in milliseconds
* @return Mpeg7Catalog that can later be saved in a Catalog as endresult
*/
protected Mpeg7Catalog filterSegmentation(LinkedList<Segment> segments, Track track, LinkedList<Segment> segmentsNew, int mergeThresh) {
if (segmentsNew == null) {
segmentsNew = new LinkedList<Segment>();
}
boolean merging = false;
MediaTime contentTime = new MediaRelTimeImpl(0, track.getDuration());
MediaLocator contentLocator = new MediaLocatorImpl(track.getURI());
Mpeg7Catalog mpeg7 = mpeg7CatalogService.newInstance();
Video videoContent = mpeg7.addVideoContent("videosegment", contentTime, contentLocator);
int segmentcount = 1;
MediaTimePoint currentSegStart = new MediaTimePointImpl();
for (Segment o : segments) {
// if the current segment is shorter than merge treshold start merging
if (o.getMediaTime().getMediaDuration().getDurationInMilliseconds() <= mergeThresh) {
// start merging and save beginning of new segment that will be generated
if (!merging) {
currentSegStart = o.getMediaTime().getMediaTimePoint();
merging = true;
}
// current segment is longer than merge threshold
} else {
long currentSegDuration = o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
long currentSegEnd = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds() + currentSegDuration;
if (merging) {
long newDuration = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds() - currentSegStart.getTimeInMilliseconds();
// save new segment that merges all previously skipped short segments
if (newDuration >= mergeThresh) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
segmentsNew.add(s);
// copy the following long segment to new list
Segment s2 = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s2.setMediaTime(o.getMediaTime());
segmentsNew.add(s2);
// if too short split new segment in middle and merge halves to
// previous and following segments
} else {
long followingStartOld = o.getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
long newSplit = (currentSegStart.getTimeInMilliseconds() + followingStartOld) / 2;
long followingEnd = followingStartOld + o.getMediaTime().getMediaDuration().getDurationInMilliseconds();
long followingDuration = followingEnd - newSplit;
// if at beginning, don't split, just merge to first large segment
if (segmentsNew.isEmpty()) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(0, followingEnd));
segmentsNew.add(s);
} else {
long previousStart = segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
// adjust end time of previous segment to split time
segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(previousStart, newSplit - previousStart));
// create new segment starting at split time
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s.setMediaTime(new MediaRelTimeImpl(newSplit, followingDuration));
segmentsNew.add(s);
}
}
merging = false;
// copy segments that are long enough to new list (with corrected number)
} else {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount++);
s.setMediaTime(o.getMediaTime());
segmentsNew.add(s);
}
}
}
// if there is an unfinished merging process after going through all segments
if (merging && !segmentsNew.isEmpty()) {
long newDuration = track.getDuration() - currentSegStart.getTimeInMilliseconds();
// if merged segment is long enough, create new segment
if (newDuration >= mergeThresh) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(currentSegStart.getTimeInMilliseconds(), newDuration));
segmentsNew.add(s);
// if not long enough, merge with previous segment
} else {
newDuration = track.getDuration() - segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds();
segmentsNew.getLast().setMediaTime(new MediaRelTimeImpl(segmentsNew.getLast().getMediaTime().getMediaTimePoint().getTimeInMilliseconds(), newDuration));
}
}
// segment spanning the whole video
if (segmentsNew.isEmpty()) {
Segment s = videoContent.getTemporalDecomposition().createSegment("segment-" + segmentcount);
s.setMediaTime(new MediaRelTimeImpl(0, track.getDuration()));
segmentsNew.add(s);
}
return mpeg7;
}
Aggregations