Examples with LineageHolder org.pentaho.metaverse.api.model.LineageHolder used on opensource projects

Example 11 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class JobRuntimeExtensionPoint method callExtensionPoint.

/**
 * Callback when a job is about to be started
 *
 * @param logChannelInterface A reference to the log in this context (the Job object's log)
 * @param o                   The object being operated on (Job in this case)
 * @throws org.pentaho.di.core.exception.KettleException
 */
@Override
public void callExtensionPoint(LogChannelInterface logChannelInterface, Object o) throws KettleException {
    // Job Started listeners get called after the extension point is invoked, so just add a job listener
    if (o instanceof Job) {
        Job job = ((Job) o);
        // If runtime lineage collection is disabled, don't run any lineage processes/methods
        if (!isRuntimeEnabled()) {
            return;
        }
        // Create and populate an execution profile with what we know so far
        ExecutionProfile executionProfile = new ExecutionProfile();
        populateExecutionProfile(executionProfile, job);
        IMetaverseBuilder builder = JobLineageHolderMap.getInstance().getMetaverseBuilder(job);
        // Add the job finished listener
        job.addJobListener(this);
        // Analyze the current transformation
        if (documentAnalyzer != null) {
            documentAnalyzer.setMetaverseBuilder(builder);
            // Create a document for the Trans
            final String clientName = executionProfile.getExecutionEngine().getName();
            final INamespace namespace = new Namespace(clientName);
            final IMetaverseNode designNode = builder.getMetaverseObjectFactory().createNodeObject(clientName, clientName, DictionaryConst.NODE_TYPE_LOCATOR);
            builder.addNode(designNode);
            final JobMeta jobMeta = job.getJobMeta();
            // The variables and parameters in the Job may not have been set on the meta, so we do it here
            // to ensure the job analyzer will have access to the parameter values.
            jobMeta.copyParametersFrom(job);
            jobMeta.activateParameters();
            job.copyVariablesFrom(jobMeta);
            if (job.getRep() != null) {
                jobMeta.setRepository(job.getRep());
            }
            String id = getFilename(jobMeta);
            if (!id.endsWith(jobMeta.getDefaultExtension())) {
                id += "." + jobMeta.getDefaultExtension();
            }
            IDocument metaverseDocument = builder.getMetaverseObjectFactory().createDocumentObject();
            metaverseDocument.setNamespace(namespace);
            metaverseDocument.setContent(jobMeta);
            metaverseDocument.setStringID(id);
            metaverseDocument.setName(jobMeta.getName());
            metaverseDocument.setExtension("kjb");
            metaverseDocument.setMimeType(URLConnection.getFileNameMap().getContentTypeFor("job.kjb"));
            metaverseDocument.setContext(new AnalysisContext(DictionaryConst.CONTEXT_RUNTIME));
            String normalizedPath;
            try {
                normalizedPath = KettleAnalyzerUtil.normalizeFilePath(id);
            } catch (MetaverseException e) {
                normalizedPath = id;
            }
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAME, job.getName());
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_PATH, normalizedPath);
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAMESPACE, namespace.getNamespaceId());
            Runnable analyzerRunner = MetaverseUtil.getAnalyzerRunner(documentAnalyzer, metaverseDocument);
            MetaverseCompletionService.getInstance().submit(analyzerRunner, id);
        }
        // Save the lineage objects for later
        LineageHolder holder = JobLineageHolderMap.getInstance().getLineageHolder(job);
        holder.setExecutionProfile(executionProfile);
        holder.setMetaverseBuilder(builder);
    }
}

Example 12 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class JobRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Job job) {
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = JobLineageHolderMap.getInstance().getLineageHolder(job);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
                // TODO logger?
                e.printStackTrace();
            } catch (ExecutionException e) {
                // TODO logger?
                e.printStackTrace();
            }
        }
        // Get the current execution profile for this job
        IExecutionProfile executionProfile = JobLineageHolderMap.getInstance().getLineageHolder(job).getExecutionProfile();
        if (executionProfile == null) {
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            executionProfile = new ExecutionProfile();
            populateExecutionProfile(executionProfile, job);
        }
        ExecutionData executionData = (ExecutionData) executionProfile.getExecutionData();
        Result result = job.getResult();
        if (result != null) {
            executionData.setFailureCount(result.getNrErrors());
        }
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", job.getName(), e.getMessage()));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        // lineage information into its own graph
        try {
            Job parentJob = job.getParentJob();
            Trans parentTrans = job.getParentTrans();
            if (parentJob == null && parentTrans == null) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", job.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", job.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
}

Example 13 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class TransLineageHolderMap method getLineageHolder.

public LineageHolder getLineageHolder(Trans t) {
    LineageHolder holder = lineageHolderMap.get(t);
    if (holder == null) {
        holder = new LineageHolder();
        putLineageHolder(t, holder);
    }
    return holder;
}

Example 14 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class TransLineageHolderMapTest method testGetMetaverseBuilder.

@Test
public void testGetMetaverseBuilder() throws Exception {
    assertNull(jobLineageHolderMap.getMetaverseBuilder(null));
    Trans trans = mock(Trans.class);
    when(trans.getParentJob()).thenReturn(null);
    when(trans.getParentTrans()).thenReturn(null);
    mockHolder.setMetaverseBuilder(mockBuilder);
    jobLineageHolderMap.putLineageHolder(trans, mockHolder);
    IMetaverseBuilder builder = jobLineageHolderMap.getMetaverseBuilder(trans);
    assertNotNull(builder);
    Trans parentTrans = mock(Trans.class);
    when(parentTrans.getParentJob()).thenReturn(null);
    when(parentTrans.getParentTrans()).thenReturn(null);
    when(trans.getParentTrans()).thenReturn(parentTrans);
    LineageHolder mockParentHolder = spy(new LineageHolder());
    IMetaverseBuilder mockParentBuilder = mock(IMetaverseBuilder.class);
    jobLineageHolderMap.putLineageHolder(parentTrans, mockParentHolder);
    mockParentHolder.setMetaverseBuilder(null);
    assertEquals(defaultBuilder, jobLineageHolderMap.getMetaverseBuilder(trans));
    mockParentHolder.setMetaverseBuilder(mockParentBuilder);
    builder = jobLineageHolderMap.getMetaverseBuilder(trans);
    assertNotNull(builder);
}

Example 15 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class StepExternalResourceConsumerListenerTest method testCallStepAddExternalResources.

@Test
public void testCallStepAddExternalResources() {
    StepExternalResourceConsumerListener stepExtensionPoint = new StepExternalResourceConsumerListener();
    stepExtensionPoint.addExternalResources(null, null);
    StepInterface mockStep = mock(StepInterface.class);
    Trans mockTrans = mock(Trans.class);
    when(mockStep.getTrans()).thenReturn(mockTrans);
    IExecutionProfile executionProfile = mock(IExecutionProfile.class);
    IExecutionData executionData = mock(IExecutionData.class);
    when(executionProfile.getExecutionData()).thenReturn(executionData);
    LineageHolder holder = new LineageHolder();
    holder.setExecutionProfile(executionProfile);
    TransLineageHolderMap.getInstance().putLineageHolder(mockTrans, holder);
    Collection<IExternalResourceInfo> externalResources = new ArrayList<IExternalResourceInfo>();
    stepExtensionPoint.addExternalResources(externalResources, mockStep);
    IExternalResourceInfo externalResource = mock(IExternalResourceInfo.class);
    externalResources.add(externalResource);
    stepExtensionPoint.addExternalResources(externalResources, mockStep);
}