Examples with LineageHolder org.pentaho.metaverse.api.model.LineageHolder used on opensource projects

Example 1 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class JobLineageHolderMap method getLineageHolder.

public LineageHolder getLineageHolder(Job job) {
    LineageHolder holder = lineageHolderMap.get(job);
    if (holder == null) {
        holder = new LineageHolder();
        lineageHolderMap.put(job, holder);
    }
    return holder;
}

Example 2 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Trans trans) {
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
            // Do nothing
            } catch (ExecutionException e) {
                log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
                log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
            }
        }
        IExecutionProfile executionProfile = holder.getExecutionProfile();
        if (executionProfile == null) {
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            // TODO: Beware duplicate profiles!
            executionProfile = new ExecutionProfile();
            populateExecutionProfile(executionProfile, trans);
        }
        ExecutionData executionData = (ExecutionData) executionProfile.getExecutionData();
        Result result = trans.getResult();
        if (result != null) {
            executionData.setFailureCount(result.getNrErrors());
        }
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        // lineage information into its own graph
        try {
            Job parentJob = trans.getParentJob();
            Trans parentTrans = trans.getParentTrans();
            if (parentJob == null && parentTrans == null) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", trans.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
}

Example 3 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method startAnalyzer.

protected void startAnalyzer(Trans trans) throws KettleException {
    if (trans == null) {
        return;
    }
    // Create and populate an execution profile with what we know so far
    ExecutionProfile executionProfile = new ExecutionProfile();
    populateExecutionProfile(executionProfile, trans);
    IMetaverseBuilder builder = TransLineageHolderMap.getInstance().getMetaverseBuilder(trans);
    // Analyze the current transformation
    if (documentAnalyzer != null) {
        documentAnalyzer.setMetaverseBuilder(builder);
        // Create a document for the Trans
        final String clientName = executionProfile.getExecutionEngine().getName();
        final INamespace namespace = new Namespace(clientName);
        final IMetaverseNode designNode = builder.getMetaverseObjectFactory().createNodeObject(clientName, clientName, DictionaryConst.NODE_TYPE_LOCATOR);
        builder.addNode(designNode);
        final TransMeta transMeta = trans.getTransMeta();
        String id = TransExtensionPointUtil.getFilename(transMeta);
        IDocument metaverseDocument = builder.getMetaverseObjectFactory().createDocumentObject();
        metaverseDocument.setNamespace(namespace);
        metaverseDocument.setContent(transMeta);
        metaverseDocument.setStringID(id);
        metaverseDocument.setName(transMeta.getName());
        metaverseDocument.setExtension("ktr");
        metaverseDocument.setMimeType(URLConnection.getFileNameMap().getContentTypeFor("trans.ktr"));
        metaverseDocument.setContext(new AnalysisContext(DictionaryConst.CONTEXT_RUNTIME));
        String normalizedPath;
        try {
            normalizedPath = KettleAnalyzerUtil.normalizeFilePath(id);
        } catch (MetaverseException e) {
            normalizedPath = id;
        }
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAME, trans.getName());
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_PATH, normalizedPath);
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAMESPACE, namespace.getNamespaceId());
        Runnable analyzerRunner = MetaverseUtil.getAnalyzerRunner(documentAnalyzer, metaverseDocument);
        MetaverseCompletionService.getInstance().submit(analyzerRunner, id);
    }
    // Save the lineage objects for later
    LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
    holder.setExecutionProfile(executionProfile);
    holder.setMetaverseBuilder(builder);
}

Example 4 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class BaseRuntimeExtensionPointTest method testWriteLineageHolder.

@Test
public void testWriteLineageHolder() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    assertEquals(0, baos.size());
    PrintStream stringStream = new PrintStream(baos);
    IGraphWriter graphWriter = mock(IGraphWriter.class);
    IExecutionProfile executionProfile = new ExecutionProfile();
    executionProfile.setName("testName");
    IMetaverseBuilder builder = mock(IMetaverseBuilder.class);
    LineageHolder holder = new LineageHolder(executionProfile, builder);
    LineageWriter lineageWriter = new LineageWriter();
    lineageWriter.setProfileOutputStream(stringStream);
    lineageWriter.setGraphOutputStream(System.out);
    lineageWriter.setOutputStrategy("all");
    lineageWriter.setGraphWriter(graphWriter);
    extensionPoint.setLineageWriter(lineageWriter);
    extensionPoint.writeLineageInfo(holder);
    assertNotEquals(0, baos.size());
    assertTrue(baos.toString().contains("testName"));
    verify(graphWriter, times(1)).outputGraph(any(Graph.class), any(OutputStream.class));
}

Example 5 with LineageHolder

use of org.pentaho.metaverse.api.model.LineageHolder in project pentaho-metaverse by pentaho.

the class JobLineageHolderMapTest method testGetMetaverseBuilder.

@Test
public void testGetMetaverseBuilder() throws Exception {
    assertNull(jobLineageHolderMap.getMetaverseBuilder(null));
    Job job = mock(Job.class);
    when(job.getParentJob()).thenReturn(null);
    when(job.getParentTrans()).thenReturn(null);
    mockHolder.setMetaverseBuilder(mockBuilder);
    jobLineageHolderMap.putLineageHolder(job, mockHolder);
    IMetaverseBuilder builder = jobLineageHolderMap.getMetaverseBuilder(job);
    assertNotNull(builder);
    Job parentJob = mock(Job.class);
    when(parentJob.getParentJob()).thenReturn(null);
    when(parentJob.getParentTrans()).thenReturn(null);
    when(job.getParentJob()).thenReturn(parentJob);
    LineageHolder mockParentHolder = spy(new LineageHolder());
    IMetaverseBuilder mockParentBuilder = mock(IMetaverseBuilder.class);
    jobLineageHolderMap.putLineageHolder(parentJob, mockParentHolder);
    mockParentHolder.setMetaverseBuilder(null);
    assertEquals(defaultBuilder, jobLineageHolderMap.getMetaverseBuilder(job));
    mockParentHolder.setMetaverseBuilder(mockParentBuilder);
    builder = jobLineageHolderMap.getMetaverseBuilder(job);
    assertNotNull(builder);
}