Examples with ExecutionData org.pentaho.metaverse.impl.model.ExecutionData used on opensource projects

Example 1 with ExecutionData

use of org.pentaho.metaverse.impl.model.ExecutionData in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Trans trans) {
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
            // Do nothing
            } catch (ExecutionException e) {
                log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
                log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
            }
        }
        IExecutionProfile executionProfile = holder.getExecutionProfile();
        if (executionProfile == null) {
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            // TODO: Beware duplicate profiles!
            executionProfile = new ExecutionProfile();
            populateExecutionProfile(executionProfile, trans);
        }
        ExecutionData executionData = (ExecutionData) executionProfile.getExecutionData();
        Result result = trans.getResult();
        if (result != null) {
            executionData.setFailureCount(result.getNrErrors());
        }
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        // lineage information into its own graph
        try {
            Job parentJob = trans.getParentJob();
            Trans parentTrans = trans.getParentTrans();
            if (parentJob == null && parentTrans == null) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", trans.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
}

Example 2 with ExecutionData

use of org.pentaho.metaverse.impl.model.ExecutionData in project pentaho-metaverse by pentaho.

the class LineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    LineageWriter fslw = new LineageWriter();
    writer = spy(fslw);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
}

Example 3 with ExecutionData

use of org.pentaho.metaverse.impl.model.ExecutionData in project pentaho-metaverse by pentaho.

the class VfsLineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    String basePath = new File(".").getCanonicalPath();
    writer = new VfsLineageWriter();
    writer = spy(writer);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
    BAD_OUTPUT_FOLDER = FilenameUtils.separatorsToSystem("file://" + basePath + BAD_OUTPUT_FOLDER_DEFAULT + random.nextInt());
    GOOD_OUTPUT_FOLDER = FilenameUtils.separatorsToSystem("file://" + basePath + GOOD_OUTPUT_FOLDER_DEFAULT + random.nextInt());
    writer.setOutputFolder(GOOD_OUTPUT_FOLDER);
}

Example 4 with ExecutionData

use of org.pentaho.metaverse.impl.model.ExecutionData in project pentaho-metaverse by pentaho.

the class JobRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Job job) {
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = JobLineageHolderMap.getInstance().getLineageHolder(job);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
                // TODO logger?
                e.printStackTrace();
            } catch (ExecutionException e) {
                // TODO logger?
                e.printStackTrace();
            }
        }
        // Get the current execution profile for this job
        IExecutionProfile executionProfile = JobLineageHolderMap.getInstance().getLineageHolder(job).getExecutionProfile();
        if (executionProfile == null) {
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            executionProfile = new ExecutionProfile();
            populateExecutionProfile(executionProfile, job);
        }
        ExecutionData executionData = (ExecutionData) executionProfile.getExecutionData();
        Result result = job.getResult();
        if (result != null) {
            executionData.setFailureCount(result.getNrErrors());
        }
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", job.getName(), e.getMessage()));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        // lineage information into its own graph
        try {
            Job parentJob = job.getParentJob();
            Trans parentTrans = job.getParentTrans();
            if (parentJob == null && parentTrans == null) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", job.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", job.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
}

Example 5 with ExecutionData

use of org.pentaho.metaverse.impl.model.ExecutionData in project pentaho-metaverse by pentaho.

the class FileSystemLineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    FileSystemLineageWriter fslw = new FileSystemLineageWriter();
    writer = spy(fslw);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
    writer.setOutputFolder("target/outputfiles");
}