Examples with Job org.pentaho.di.job.Job used on opensource projects

Example 31 with Job

use of org.pentaho.di.job.Job in project pentaho-kettle by pentaho.

the class RegisterPackageServlet method generateBody.

@Override
WebResult generateBody(HttpServletRequest request, HttpServletResponse response, boolean useXML) throws KettleException, IOException {
    FileObject tempFile = KettleVFS.createTempFile("export", ".zip", System.getProperty("java.io.tmpdir"));
    OutputStream out = KettleVFS.getOutputStream(tempFile, false);
    IOUtils.copy(request.getInputStream(), out);
    out.flush();
    IOUtils.closeQuietly(out);
    String archiveUrl = tempFile.getName().toString();
    // the resource to load
    String load = request.getParameter(PARAMETER_LOAD);
    if (!Utils.isEmpty(load)) {
        String fileUrl = MessageFormat.format(ZIP_CONT, archiveUrl, load);
        boolean isJob = TYPE_JOB.equalsIgnoreCase(request.getParameter(PARAMETER_TYPE));
        String resultId;
        if (isJob) {
            Node node = getConfigNodeFromZIP(archiveUrl, Job.CONFIGURATION_IN_EXPORT_FILENAME, JobExecutionConfiguration.XML_TAG);
            JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration(node);
            JobMeta jobMeta = new JobMeta(fileUrl, jobExecutionConfiguration.getRepository());
            JobConfiguration jobConfiguration = new JobConfiguration(jobMeta, jobExecutionConfiguration);
            Job job = createJob(jobConfiguration);
            resultId = job.getContainerObjectId();
        } else {
            Node node = getConfigNodeFromZIP(archiveUrl, Trans.CONFIGURATION_IN_EXPORT_FILENAME, TransExecutionConfiguration.XML_TAG);
            TransExecutionConfiguration transExecutionConfiguration = new TransExecutionConfiguration(node);
            TransMeta transMeta = new TransMeta(fileUrl, transExecutionConfiguration.getRepository());
            TransConfiguration transConfiguration = new TransConfiguration(transMeta, transExecutionConfiguration);
            Trans trans = createTrans(transConfiguration);
            resultId = trans.getContainerObjectId();
        }
        return new WebResult(WebResult.STRING_OK, fileUrl, resultId);
    }
    return null;
}

Example 32 with Job

use of org.pentaho.di.job.Job in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Trans trans) {
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
            // Do nothing
            } catch (ExecutionException e) {
                log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
                log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
            }
        }
        IExecutionProfile executionProfile = holder.getExecutionProfile();
        if (executionProfile == null) {
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            // TODO: Beware duplicate profiles!
            executionProfile = new ExecutionProfile();
            populateExecutionProfile(executionProfile, trans);
        }
        ExecutionData executionData = (ExecutionData) executionProfile.getExecutionData();
        Result result = trans.getResult();
        if (result != null) {
            executionData.setFailureCount(result.getNrErrors());
        }
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        // lineage information into its own graph
        try {
            Job parentJob = trans.getParentJob();
            Trans parentTrans = trans.getParentTrans();
            if (parentJob == null && parentTrans == null) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", trans.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
}

Example 33 with Job

use of org.pentaho.di.job.Job in project pentaho-metaverse by pentaho.

the class JobLineageHolderMapTest method testGetMetaverseBuilder.

@Test
public void testGetMetaverseBuilder() throws Exception {
    assertNull(jobLineageHolderMap.getMetaverseBuilder(null));
    Job job = mock(Job.class);
    when(job.getParentJob()).thenReturn(null);
    when(job.getParentTrans()).thenReturn(null);
    mockHolder.setMetaverseBuilder(mockBuilder);
    jobLineageHolderMap.putLineageHolder(job, mockHolder);
    IMetaverseBuilder builder = jobLineageHolderMap.getMetaverseBuilder(job);
    assertNotNull(builder);
    Job parentJob = mock(Job.class);
    when(parentJob.getParentJob()).thenReturn(null);
    when(parentJob.getParentTrans()).thenReturn(null);
    when(job.getParentJob()).thenReturn(parentJob);
    LineageHolder mockParentHolder = spy(new LineageHolder());
    IMetaverseBuilder mockParentBuilder = mock(IMetaverseBuilder.class);
    jobLineageHolderMap.putLineageHolder(parentJob, mockParentHolder);
    mockParentHolder.setMetaverseBuilder(null);
    assertEquals(defaultBuilder, jobLineageHolderMap.getMetaverseBuilder(job));
    mockParentHolder.setMetaverseBuilder(mockParentBuilder);
    builder = jobLineageHolderMap.getMetaverseBuilder(job);
    assertNotNull(builder);
}

Example 34 with Job

use of org.pentaho.di.job.Job in project pentaho-metaverse by pentaho.

the class JobLineageHolderMapTest method testGetPutLineageHolder.

@Test
public void testGetPutLineageHolder() throws Exception {
    Job job = mock(Job.class);
    LineageHolder holder = jobLineageHolderMap.getLineageHolder(job);
    // We always get a (perhaps empty) holder
    assertNotNull(holder);
    assertFalse(holder == mockHolder);
    assertNull(holder.getExecutionProfile());
    assertNull(holder.getMetaverseBuilder());
    mockHolder.setMetaverseBuilder(mockBuilder);
    mockHolder.setExecutionProfile(mockExecutionProfile);
    jobLineageHolderMap.putLineageHolder(job, mockHolder);
    holder = jobLineageHolderMap.getLineageHolder(job);
    // We always get a (perhaps empty) holder
    assertNotNull(holder);
    assertTrue(holder == mockHolder);
    assertTrue(holder.getExecutionProfile() == mockExecutionProfile);
    assertTrue(holder.getMetaverseBuilder() == mockBuilder);
}

Example 35 with Job

use of org.pentaho.di.job.Job in project pentaho-metaverse by pentaho.

the class JobEntryExternalResourceConsumerListenerTest method testCallJobEntryAddExternalResources.

@Test
public void testCallJobEntryAddExternalResources() {
    JobEntryExternalResourceConsumerListener stepExtensionPoint = new JobEntryExternalResourceConsumerListener();
    stepExtensionPoint.addExternalResources(null, null);
    JobEntryInterface mockJobEntry = mock(JobEntryInterface.class);
    Job mockJob = mock(Job.class);
    when(mockJobEntry.getParentJob()).thenReturn(mockJob);
    IExecutionProfile executionProfile = mock(IExecutionProfile.class);
    IExecutionData executionData = mock(IExecutionData.class);
    when(executionProfile.getExecutionData()).thenReturn(executionData);
    JobLineageHolderMap.getInstance().getLineageHolder(mockJob).setExecutionProfile(executionProfile);
    Collection<IExternalResourceInfo> externalResources = new ArrayList<IExternalResourceInfo>();
    stepExtensionPoint.addExternalResources(externalResources, mockJobEntry);
    IExternalResourceInfo externalResource = mock(IExternalResourceInfo.class);
    externalResources.add(externalResource);
    stepExtensionPoint.addExternalResources(externalResources, mockJobEntry);
}