Examples with IExecutionProfile org.pentaho.metaverse.api.model.IExecutionProfile used on opensource projects

Example 1 with IExecutionProfile

use of org.pentaho.metaverse.api.model.IExecutionProfile in project pentaho-metaverse by pentaho.

the class JobEntryExternalResourceListener method afterExecution.

@Override
public void afterExecution(Job job, JobEntryCopy jobEntryCopy, JobEntryInterface jobEntryInterface, Result result) {
    IExecutionProfile executionProfile = JobLineageHolderMap.getInstance().getLineageHolder(job).getExecutionProfile();
    IExecutionData executionData = executionProfile.getExecutionData();
    // Get input files (aka Resource Dependencies)
    JobMeta jobMeta = job.getJobMeta();
    if (jobMeta != null) {
        List<ResourceReference> dependencies = jobEntryInterface.getResourceDependencies(jobMeta);
        if (dependencies != null) {
            for (ResourceReference ref : dependencies) {
                List<ResourceEntry> resourceEntries = ref.getEntries();
                if (resourceEntries != null) {
                    for (ResourceEntry entry : resourceEntries) {
                        executionData.addExternalResource(jobEntryInterface.getName(), ExternalResourceInfoFactory.createResource(entry, true));
                    }
                }
            }
        }
    }
    // Get output files (aka result files)
    if (result != null) {
        List<ResultFile> resultFiles = result.getResultFilesList();
        if (resultFiles != null) {
            for (ResultFile resultFile : resultFiles) {
                executionData.addExternalResource(jobEntryInterface.getName(), ExternalResourceInfoFactory.createFileResource(resultFile.getFile(), false));
            }
        }
    }
}

Example 2 with IExecutionProfile

use of org.pentaho.metaverse.api.model.IExecutionProfile in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Trans trans) {
    log.info(Messages.getString("INFO.WrittingGraphForTransformation", trans.getName()));
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
            // Do nothing
            } catch (ExecutionException e) {
                log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
                log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
            }
        }
        IExecutionProfile executionProfile = holder.getExecutionProfile();
        if (executionProfile == null) {
            // Note that this should NEVER happen, this is purely a preventative measure...
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            // TODO: Beware duplicate profiles!
            executionProfile = new ExecutionProfile();
        }
        populateExecutionProfile(executionProfile, trans);
        removeSensitiveDataFromHolder(holder);
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        try {
            if (shouldCreateGraph(trans)) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                    // lineage has been written - call the appropriate extension point
                    ExtensionPointHandler.callExtensionPoint(trans.getLogChannel(), MetaverseExtensionPoint.TransLineageWriteEnd.id, trans);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", trans.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
    // cleanup to prevent unnecessary memory usage - we no longer need this Trans in the TransLineageHolderMap
    TransLineageHolderMap.getInstance().removeLineageHolder(trans);
    log.warn(Messages.getString("INFO.TransformationAnalyzeFinished", trans.getName()));
    logMinimal(Messages.getString("INFO.TransformationAnalyzeFinished", trans.getName()));
}

Example 3 with IExecutionProfile

use of org.pentaho.metaverse.api.model.IExecutionProfile in project pentaho-metaverse by pentaho.

the class StepExternalResourceConsumerListener method addExternalResources.

protected void addExternalResources(Collection<IExternalResourceInfo> resources, StepInterface step) {
    if (resources != null) {
        // Add the resources to the execution profile
        IExecutionProfile executionProfile = TransLineageHolderMap.getInstance().getLineageHolder(step.getTrans()).getExecutionProfile();
        if (executionProfile != null) {
            String stepName = step.getStepname();
            Map<String, List<IExternalResourceInfo>> resourceMap = executionProfile.getExecutionData().getExternalResources();
            List<IExternalResourceInfo> externalResources = resourceMap.get(stepName);
            if (externalResources == null) {
                externalResources = new LinkedList<IExternalResourceInfo>();
            }
            externalResources.addAll(resources);
            resourceMap.put(stepName, externalResources);
        }
    }
}

Example 4 with IExecutionProfile

use of org.pentaho.metaverse.api.model.IExecutionProfile in project pentaho-metaverse by pentaho.

the class FileSystemLineageWriter method getDateFolder.

protected File getDateFolder(String parentDir, LineageHolder holder) {
    String dir = (parentDir == null) ? "" : parentDir + File.separator;
    if (holder != null && holder.getExecutionProfile() != null) {
        IExecutionProfile profile = holder.getExecutionProfile();
        dir += dateFolderFormat.format(profile.getExecutionData().getStartTime());
    } else {
        dir += dateFolderFormat.format(new Date());
    }
    return new File(dir);
}

Example 5 with IExecutionProfile

use of org.pentaho.metaverse.api.model.IExecutionProfile in project pentaho-metaverse by pentaho.

the class StepExternalConsumerRowListener method rowReadEvent.

/**
 * Called when rows are read by the step to which this listener is attached
 *
 * @param rowMeta The metadata (value types, e.g.) of the associated row data
 * @param row     An array of Objects corresponding to the row data
 * @see org.pentaho.di.trans.step.RowListener#rowReadEvent(org.pentaho.di.core.row.RowMetaInterface,
 * Object[])
 */
@Override
@SuppressWarnings("unchecked")
public void rowReadEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
    Collection<IExternalResourceInfo> resources = stepExternalResourceConsumer.getResourcesFromRow((BaseStep) step, rowMeta, row);
    if (resources != null) {
        // Add the resources to the execution profile
        IExecutionProfile executionProfile = TransLineageHolderMap.getInstance().getLineageHolder(step.getTrans()).getExecutionProfile();
        if (executionProfile != null) {
            String stepName = step.getStepname();
            Map<String, List<IExternalResourceInfo>> resourceMap = executionProfile.getExecutionData().getExternalResources();
            final List<IExternalResourceInfo> existingResources = resourceMap.get(stepName);
            Set<IExternalResourceInfo> externalResources = existingResources == null ? new HashSet() : new HashSet(resourceMap.get(stepName));
            if (externalResources == null) {
                externalResources = new HashSet();
            }
            // avoid adding duplicates
            externalResources.addAll(new HashSet<>(resources));
            resourceMap.put(stepName, new ArrayList(externalResources));
        }
    }
}