Examples with ExecutionProfile org.pentaho.metaverse.impl.model.ExecutionProfile used on opensource projects

Example 6 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class StepExternalConsumerRowListenerTest method testStepExternalConsumerRowListener.

@Test
public void testStepExternalConsumerRowListener() throws Exception {
    BaseStep mockStep = mock(BaseStep.class, withSettings().extraInterfaces(StepInterface.class));
    when(mockStep.getStepname()).thenReturn("my_step");
    StepMeta mockStepMeta = mock(StepMeta.class);
    BaseStepMeta bsm = mock(BaseStepMeta.class, withSettings().extraInterfaces(StepMetaInterface.class));
    StepMetaInterface stepMetaInterface = (StepMetaInterface) bsm;
    when(mockStep.getStepMeta()).thenReturn(mockStepMeta);
    when(mockStepMeta.getStepMetaInterface()).thenReturn(stepMetaInterface);
    Trans mockTrans = mock(Trans.class);
    when(mockStep.getTrans()).thenReturn(mockTrans);
    LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(mockStep.getTrans());
    IExecutionProfile executionProfile = new ExecutionProfile();
    holder.setExecutionProfile(executionProfile);
    TransLineageHolderMap.getInstance().putLineageHolder(mockTrans, holder);
    IStepExternalResourceConsumer consumer = mock(IStepExternalResourceConsumer.class);
    RowMetaInterface rmi = mock(RowMetaInterface.class);
    Object[] row1 = new String[] { "val1a", "val1b" };
    Object[] row2 = new String[] { "val21a", "val2b" };
    StepExternalConsumerRowListener listener = new StepExternalConsumerRowListener(consumer, mockStep);
    IExternalResourceInfo resource1 = mock(IExternalResourceInfo.class);
    IExternalResourceInfo resource2 = mock(IExternalResourceInfo.class);
    IExternalResourceInfo resource3 = mock(IExternalResourceInfo.class);
    when(consumer.getResourcesFromRow(mockStep, rmi, row1)).thenReturn(Arrays.asList(new IExternalResourceInfo[] { resource1, resource2 }));
    when(consumer.getResourcesFromRow(mockStep, rmi, row2)).thenReturn(Arrays.asList(new IExternalResourceInfo[] { resource2, resource3 }));
    listener.rowReadEvent(rmi, row1);
    Map resourceMap = TransLineageHolderMap.getInstance().getLineageHolder(mockTrans).getExecutionProfile().getExecutionData().getExternalResources();
    Assert.assertNotNull(resourceMap);
    Assert.assertEquals(1, resourceMap.size());
    Assert.assertNotNull(resourceMap.get("my_step"));
    Assert.assertTrue(resourceMap.get("my_step") instanceof List);
    List resources = (List) resourceMap.get("my_step");
    Assert.assertNotNull(resources);
    Assert.assertEquals(2, resources.size());
    Assert.assertTrue(resources.contains(resource1));
    Assert.assertTrue(resources.contains(resource2));
    listener.rowReadEvent(rmi, row2);
    resourceMap = TransLineageHolderMap.getInstance().getLineageHolder(mockTrans).getExecutionProfile().getExecutionData().getExternalResources();
    Assert.assertNotNull(resourceMap);
    Assert.assertEquals(1, resourceMap.size());
    Assert.assertNotNull(resourceMap.get("my_step"));
    Assert.assertTrue(resourceMap.get("my_step") instanceof List);
    resources = (List) resourceMap.get("my_step");
    Assert.assertNotNull(resources);
    Assert.assertEquals(3, resources.size());
    Assert.assertTrue(resources.contains(resource1));
    Assert.assertTrue(resources.contains(resource2));
    Assert.assertTrue(resources.contains(resource3));
}

Example 7 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class BaseRuntimeExtensionPointTest method testWriteLineageHolder.

@Test
public void testWriteLineageHolder() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    assertEquals(0, baos.size());
    PrintStream stringStream = new PrintStream(baos);
    IGraphWriter graphWriter = mock(IGraphWriter.class);
    IExecutionProfile executionProfile = new ExecutionProfile();
    executionProfile.setName("testName");
    IMetaverseBuilder builder = mock(IMetaverseBuilder.class);
    LineageHolder holder = new LineageHolder(executionProfile, builder);
    LineageWriter lineageWriter = new LineageWriter();
    lineageWriter.setProfileOutputStream(stringStream);
    lineageWriter.setGraphOutputStream(System.out);
    lineageWriter.setOutputStrategy("all");
    lineageWriter.setGraphWriter(graphWriter);
    extensionPoint.setLineageWriter(lineageWriter);
    extensionPoint.writeLineageInfo(holder);
    assertNotEquals(0, baos.size());
    assertTrue(baos.toString().contains("testName"));
    verify(graphWriter, times(1)).outputGraph(any(Graph.class), any(OutputStream.class));
}

Example 8 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class FileSystemLineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    FileSystemLineageWriter fslw = new FileSystemLineageWriter();
    writer = spy(fslw);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
    writer.setOutputFolder("target/outputfiles");
}

Example 9 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class JobRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Job job) {
    log.info(Messages.getString("INFO.WrittingGraphForJob", job.getJobname()));
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = JobLineageHolderMap.getInstance().getLineageHolder(job);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
                // TODO logger?
                e.printStackTrace();
            } catch (ExecutionException e) {
                // TODO logger?
                e.printStackTrace();
            }
        }
        // Get the current execution profile for this job
        IExecutionProfile executionProfile = JobLineageHolderMap.getInstance().getLineageHolder(job).getExecutionProfile();
        if (executionProfile == null) {
            // Note that this should NEVER happen, this is purely a preventative measure...
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            executionProfile = new ExecutionProfile();
        }
        populateExecutionProfile(executionProfile, job);
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", job.getName(), e.getMessage()));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        try {
            if (shouldCreateGraph(job)) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                    // lineage has been written - call the appropriate extension point
                    ExtensionPointHandler.callExtensionPoint(job.getLogChannel(), MetaverseExtensionPoint.JobLineageWriteEnd.id, job);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", job.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", job.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
    // cleanup to prevent unnecessary memory usage - we no longer need this Job in the JobLineageHolderMap
    JobLineageHolderMap.getInstance().removeLineageHolder(job);
    log.warn(Messages.getString("INFO.JobAnalyzeFinished", job.getJobname()));
    logMinimal(Messages.getString("INFO.JobAnalyzeFinished", job.getJobname()));
}