Examples with JobBarrier org.opencastproject.job.api.JobBarrier used on opensource projects

Example 11 with JobBarrier

use of org.opencastproject.job.api.JobBarrier in project opencast by opencast.

the class ServiceRegistryJpaImplTest method testDispatchingJobsHigherMaxLoad.

@Test
public void testDispatchingJobsHigherMaxLoad() throws Exception {
    if (serviceRegistryJpaImpl.scheduledExecutor != null)
        serviceRegistryJpaImpl.scheduledExecutor.shutdown();
    serviceRegistryJpaImpl.scheduledExecutor = Executors.newScheduledThreadPool(1);
    serviceRegistryJpaImpl.activate(null);
    Hashtable<String, String> properties = new Hashtable<>();
    properties.put("dispatchinterval", "1000");
    serviceRegistryJpaImpl.updated(properties);
    registerTestHostAndService();
    Job testJob = serviceRegistryJpaImpl.createJob(TEST_HOST, TEST_SERVICE, TEST_OPERATION, null, null, true, null, 10.0f);
    JobBarrier barrier = new JobBarrier(null, serviceRegistryJpaImpl, testJob);
    try {
        barrier.waitForJobs(2000);
        Assert.fail();
    } catch (Exception e) {
        testJob = serviceRegistryJpaImpl.getJob(testJob.getId());
        Assert.assertEquals(TEST_HOST_OTHER, testJob.getProcessingHost());
    }
}

Example 12 with JobBarrier

use of org.opencastproject.job.api.JobBarrier in project opencast by opencast.

the class SoxServiceTest method testNormalizeIncreaseAudio.

@Test
public void testNormalizeIncreaseAudio() throws Exception {
    if (!soxInstalled)
        return;
    assertTrue(source.isFile());
    String sourceTrackXml = "<track xmlns=\"http://mediapackage.opencastproject.org\" id=\"track-1\" type=\"presentation/source\"><mimetype>audio/flac</mimetype>" + "<url>http://localhost:8080/workflow/samples/camera.mpg</url>" + "<checksum type=\"md5\">43b7d843b02c4a429b2f547a4f230d31</checksum><duration>14546</duration>" + "<audio><device type=\"UFG03\" version=\"30112007\" vendor=\"Unigraf\" />" + "<encoder type=\"H.264\" version=\"7.4\" vendor=\"Apple Inc\" /><channels>2</channels>" + "<bitdepth>16</bitdepth><rmsleveldb>-27.78</rmsleveldb><samplingrate>44100</samplingrate></audio></track>";
    Track sourceTrack = (Track) MediaPackageElementParser.getFromXml(sourceTrackXml);
    List<Job> jobs = new ArrayList<Job>();
    for (int i = 0; i < 10; i++) {
        jobs.add(soxService.normalize(sourceTrack, -25f));
    }
    boolean success = new JobBarrier(null, serviceRegistry, jobs.toArray(new Job[jobs.size()])).waitForJobs().isSuccess();
    assertTrue(success);
    for (Job j : jobs) {
        // Always check the service registry for the latest version of the job
        Job job = serviceRegistry.getJob(j.getId());
        TrackImpl track = (TrackImpl) MediaPackageElementParser.getFromXml(job.getPayload());
        AudioStream audioStream = track.getAudio().get(0);
        assertEquals(-25f, audioStream.getRmsLevDb().floatValue(), 0.9);
        assertEquals(Job.Status.FINISHED, job.getStatus());
    }
}

Example 13 with JobBarrier

use of org.opencastproject.job.api.JobBarrier in project opencast by opencast.

the class ExecuteOnceWorkflowOperationHandler method start.

/**
 * {@inheritDoc}
 *
 * @see org.opencastproject.workflow.api.WorkflowOperationHandler#start(org.opencastproject.workflow.api.WorkflowInstance, JobContext)
 */
@Override
public WorkflowOperationResult start(WorkflowInstance workflowInstance, JobContext context) throws WorkflowOperationException {
    MediaPackage mediaPackage = workflowInstance.getMediaPackage();
    WorkflowOperationInstance operation = workflowInstance.getCurrentOperation();
    logger.debug("Running execute workflow operation with ID {}", operation.getId());
    // Get operation parameters
    String exec = StringUtils.trimToNull(operation.getConfiguration(EXEC_PROPERTY));
    String params = StringUtils.trimToNull(operation.getConfiguration(PARAMS_PROPERTY));
    float load = 1.0f;
    String loadPropertyStr = StringUtils.trimToEmpty(operation.getConfiguration(LOAD_PROPERTY));
    if (StringUtils.isNotBlank(loadPropertyStr)) {
        try {
            load = Float.parseFloat(loadPropertyStr);
        } catch (NumberFormatException e) {
            String description = StringUtils.trimToEmpty(operation.getDescription());
            logger.warn("Ignoring invalid load value '{}' on execute operation with description '{}'", loadPropertyStr, description);
        }
    }
    String targetFlavorStr = StringUtils.trimToNull(operation.getConfiguration(TARGET_FLAVOR_PROPERTY));
    String targetTags = StringUtils.trimToNull(operation.getConfiguration(TARGET_TAGS_PROPERTY));
    String outputFilename = StringUtils.trimToNull(operation.getConfiguration(OUTPUT_FILENAME_PROPERTY));
    String expectedTypeStr = StringUtils.trimToNull(operation.getConfiguration(EXPECTED_TYPE_PROPERTY));
    boolean setWfProps = Boolean.valueOf(StringUtils.trimToNull(operation.getConfiguration(SET_WF_PROPS_PROPERTY)));
    // Unmarshall target flavor
    MediaPackageElementFlavor targetFlavor = null;
    if (targetFlavorStr != null)
        targetFlavor = MediaPackageElementFlavor.parseFlavor(targetFlavorStr);
    // Unmarshall expected mediapackage element type
    MediaPackageElement.Type expectedType = null;
    if (expectedTypeStr != null) {
        for (MediaPackageElement.Type type : MediaPackageElement.Type.values()) if (type.toString().equalsIgnoreCase(expectedTypeStr)) {
            expectedType = type;
            break;
        }
        if (expectedType == null)
            throw new WorkflowOperationException("'" + expectedTypeStr + "' is not a valid element type");
    }
    // Process the result element
    MediaPackageElement resultElement = null;
    try {
        Job job = executeService.execute(exec, params, mediaPackage, outputFilename, expectedType, load);
        WorkflowOperationResult result = null;
        // Wait for all jobs to be finished
        if (!waitForStatus(job).isSuccess())
            throw new WorkflowOperationException("Execute operation failed");
        if (StringUtils.isNotBlank(job.getPayload())) {
            if (setWfProps) {
                // The job payload is a file with set of properties for the workflow
                resultElement = MediaPackageElementParser.getFromXml(job.getPayload());
                final Properties properties = new Properties();
                File propertiesFile = workspace.get(resultElement.getURI());
                try (InputStream is = new FileInputStream(propertiesFile)) {
                    properties.load(is);
                }
                logger.debug("Loaded {} properties from {}", properties.size(), propertiesFile);
                workspace.deleteFromCollection(ExecuteService.COLLECTION, propertiesFile.getName());
                Map<String, String> wfProps = new HashMap<String, String>((Map) properties);
                result = createResult(mediaPackage, wfProps, Action.CONTINUE, job.getQueueTime());
            } else {
                // The job payload is a new element for the MediaPackage
                resultElement = MediaPackageElementParser.getFromXml(job.getPayload());
                if (resultElement.getElementType() == MediaPackageElement.Type.Track) {
                    // Have the track inspected and return the result
                    Job inspectionJob = null;
                    inspectionJob = inspectionService.inspect(resultElement.getURI());
                    JobBarrier barrier = new JobBarrier(job, serviceRegistry, inspectionJob);
                    if (!barrier.waitForJobs().isSuccess()) {
                        throw new ExecuteException("Media inspection of " + resultElement.getURI() + " failed");
                    }
                    resultElement = MediaPackageElementParser.getFromXml(inspectionJob.getPayload());
                }
                // Store new element to mediaPackage
                mediaPackage.add(resultElement);
                URI uri = workspace.moveTo(resultElement.getURI(), mediaPackage.getIdentifier().toString(), resultElement.getIdentifier(), outputFilename);
                resultElement.setURI(uri);
                // Set new flavor
                if (targetFlavor != null)
                    resultElement.setFlavor(targetFlavor);
                // Set new tags
                if (targetTags != null) {
                    // Assume the tags starting with "-" means we want to eliminate such tags form the result element
                    for (String tag : asList(targetTags)) {
                        if (tag.startsWith("-"))
                            // We remove the tag resulting from stripping all the '-' characters at the beginning of the tag
                            resultElement.removeTag(tag.replaceAll("^-+", ""));
                        else
                            resultElement.addTag(tag);
                    }
                }
                result = createResult(mediaPackage, Action.CONTINUE, job.getQueueTime());
            }
        } else {
            // Payload is empty
            result = createResult(mediaPackage, Action.CONTINUE, job.getQueueTime());
        }
        logger.debug("Execute operation {} completed", operation.getId());
        return result;
    } catch (ExecuteException e) {
        throw new WorkflowOperationException(e);
    } catch (MediaPackageException e) {
        throw new WorkflowOperationException("Some result element couldn't be serialized", e);
    } catch (NotFoundException e) {
        throw new WorkflowOperationException("Could not find mediapackage", e);
    } catch (IOException e) {
        throw new WorkflowOperationException("Error unmarshalling a result mediapackage element", e);
    } catch (MediaInspectionException e) {
        throw new WorkflowOperationException("Media inspection of " + resultElement.getURI() + " failed", e);
    }
}

Example 14 with JobBarrier

use of org.opencastproject.job.api.JobBarrier in project opencast by opencast.

the class DownloadDistributionServiceImplTest method testRetractByOriginal.

@Test
public void testRetractByOriginal() throws Exception {
    // Distribute the mediapackage and all of its elements
    Job job1 = service.distribute("engage-player", mp, "track-1");
    Job job2 = service.distribute("engage-player", mp, "catalog-1");
    Job job3 = service.distribute("engage-player", mp, "catalog-2");
    Job job4 = service.distribute("engage-player", mp, "notes");
    Job job5 = service.distribute("oai-pmh", mp, "track-1");
    JobBarrier jobBarrier = new JobBarrier(null, serviceRegistry, 500, job1, job2, job3, job4, job5);
    jobBarrier.waitForJobs();
    // Add the new elements to the mediapackage
    mp.add(MediaPackageElementParser.getFromXml(job1.getPayload()));
    mp.add(MediaPackageElementParser.getFromXml(job2.getPayload()));
    mp.add(MediaPackageElementParser.getFromXml(job3.getPayload()));
    mp.add(MediaPackageElementParser.getFromXml(job4.getPayload()));
    mp.add(MediaPackageElementParser.getFromXml(job5.getPayload()));
    File mpDir = new File(distributionRoot, PathSupport.path(defaultOrganization.getId(), "engage-player", mp.getIdentifier().compact()));
    File mediaDir = new File(mpDir, "track-1");
    File metadata1Dir = new File(mpDir, "catalog-1");
    File metadata2Dir = new File(mpDir, "catalog-2");
    File attachmentsDir = new File(mpDir, "notes");
    Assert.assertTrue(mediaDir.exists());
    Assert.assertTrue(metadata1Dir.exists());
    Assert.assertTrue(metadata2Dir.exists());
    Assert.assertTrue(attachmentsDir.exists());
    // the filenames are changed to reflect the element ID
    Assert.assertTrue(new File(mediaDir, "media.mov").exists());
    Assert.assertTrue(new File(metadata1Dir, "dublincore.xml").exists());
    Assert.assertTrue(new File(metadata2Dir, "mpeg7.xml").exists());
    Assert.assertTrue(new File(attachmentsDir, "attachment.txt").exists());
    // Now retract the mediapackage and ensure that the distributed files have been removed
    Job job6 = service.retract("engage-player", mp, "track-1");
    Job job7 = service.retract("engage-player", mp, "catalog-1");
    Job job8 = service.retract("engage-player", mp, "catalog-2");
    Job job9 = service.retract("engage-player", mp, "notes");
    jobBarrier = new JobBarrier(null, serviceRegistry, 500, job6, job7, job8, job9);
    jobBarrier.waitForJobs();
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("track-1")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("catalog-1")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("catalog-2")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("notes")).isFile());
    Assert.assertTrue(service.getDistributionFile("oai-pmh", mp, mp.getElementById("track-1")).isFile());
}

Example 15 with JobBarrier

use of org.opencastproject.job.api.JobBarrier in project opencast by opencast.

the class DownloadDistributionServiceImplTest method testRetractByDistributed.

@Test
public void testRetractByDistributed() throws Exception {
    int elementCount = mp.getElements().length;
    // Distribute the mediapackage and all of its elements
    Job job1 = service.distribute("engage-player", mp, "track-1");
    Job job2 = service.distribute("engage-player", mp, "catalog-1");
    Job job3 = service.distribute("engage-player", mp, "catalog-2");
    Job job4 = service.distribute("engage-player", mp, "notes");
    Job job5 = service.distribute("oai-pmh", mp, "notes");
    JobBarrier jobBarrier = new JobBarrier(null, serviceRegistry, 500, job1, job2, job3, job4, job5);
    jobBarrier.waitForJobs();
    // Add the new elements to the mediapackage
    MediaPackageElement job1Element = MediaPackageElementParser.getFromXml(job1.getPayload());
    mp.add(job1Element);
    MediaPackageElement job2Element = MediaPackageElementParser.getFromXml(job2.getPayload());
    mp.add(job2Element);
    MediaPackageElement job3Element = MediaPackageElementParser.getFromXml(job3.getPayload());
    mp.add(job3Element);
    MediaPackageElement job4Element = MediaPackageElementParser.getFromXml(job4.getPayload());
    mp.add(job4Element);
    MediaPackageElement job5Element = MediaPackageElementParser.getFromXml(job5.getPayload());
    mp.add(job5Element);
    File mpDir = new File(distributionRoot, PathSupport.path(defaultOrganization.getId(), "engage-player", mp.getIdentifier().compact()));
    File mediaDir = new File(mpDir, "track-1");
    File metadata1Dir = new File(mpDir, "catalog-1");
    File metadata2Dir = new File(mpDir, "catalog-2");
    File attachmentsDir = new File(mpDir, "notes");
    Assert.assertTrue(mediaDir.exists());
    Assert.assertTrue(metadata1Dir.exists());
    Assert.assertTrue(metadata2Dir.exists());
    Assert.assertTrue(attachmentsDir.exists());
    // the filenames are changed to reflect the element ID
    Assert.assertTrue(new File(mediaDir, "media.mov").exists());
    Assert.assertTrue(new File(metadata1Dir, "dublincore.xml").exists());
    Assert.assertTrue(new File(metadata2Dir, "mpeg7.xml").exists());
    Assert.assertTrue(new File(attachmentsDir, "attachment.txt").exists());
    // Now retract the mediapackage and ensure that the distributed files have been removed
    Job job6 = service.retract("engage-player", mp, job1Element.getIdentifier());
    Job job7 = service.retract("engage-player", mp, job2Element.getIdentifier());
    Job job8 = service.retract("engage-player", mp, job3Element.getIdentifier());
    Job job9 = service.retract("engage-player", mp, job4Element.getIdentifier());
    jobBarrier = new JobBarrier(null, serviceRegistry, 500, job6, job7, job8, job9);
    jobBarrier.waitForJobs();
    // Remove the distributed elements from the mediapackage
    mp.remove(MediaPackageElementParser.getFromXml(job6.getPayload()));
    mp.remove(MediaPackageElementParser.getFromXml(job7.getPayload()));
    mp.remove(MediaPackageElementParser.getFromXml(job8.getPayload()));
    mp.remove(MediaPackageElementParser.getFromXml(job9.getPayload()));
    // +1 since the oai-pmh distribution has not been retracted
    Assert.assertEquals(elementCount + 1, mp.getElements().length);
    Assert.assertNotNull(mp.getElementById("track-1"));
    Assert.assertNotNull(mp.getElementById("catalog-1"));
    Assert.assertNotNull(mp.getElementById("catalog-2"));
    Assert.assertNotNull(mp.getElementById("notes"));
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("track-1")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("catalog-1")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("catalog-2")).isFile());
    Assert.assertFalse(service.getDistributionFile("engage-player", mp, mp.getElementById("notes")).isFile());
    Assert.assertTrue(service.getDistributionFile("oai-pmh", mp, mp.getElementById("notes")).isFile());
}