Examples with ExecutionProfile org.pentaho.metaverse.impl.model.ExecutionProfile used on opensource projects

Example 1 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method startAnalyzer.

protected void startAnalyzer(Trans trans) throws KettleException {
    if (trans == null) {
        return;
    }
    // Create and populate an execution profile with what we know so far
    ExecutionProfile executionProfile = new ExecutionProfile();
    populateExecutionProfile(executionProfile, trans);
    IMetaverseBuilder builder = TransLineageHolderMap.getInstance().getMetaverseBuilder(trans);
    // Analyze the current transformation
    if (documentAnalyzer != null) {
        documentAnalyzer.setMetaverseBuilder(builder);
        // Create a document for the Trans
        final String clientName = executionProfile.getExecutionEngine().getName();
        final INamespace namespace = new Namespace(clientName);
        final IMetaverseNode designNode = builder.getMetaverseObjectFactory().createNodeObject(clientName, clientName, DictionaryConst.NODE_TYPE_LOCATOR);
        builder.addNode(designNode);
        final TransMeta transMeta = trans.getTransMeta();
        String id = TransExtensionPointUtil.getFilename(transMeta);
        IDocument metaverseDocument = builder.getMetaverseObjectFactory().createDocumentObject();
        metaverseDocument.setNamespace(namespace);
        metaverseDocument.setContent(transMeta);
        metaverseDocument.setStringID(id);
        metaverseDocument.setName(transMeta.getName());
        metaverseDocument.setExtension("ktr");
        metaverseDocument.setMimeType(URLConnection.getFileNameMap().getContentTypeFor("trans.ktr"));
        metaverseDocument.setContext(new AnalysisContext(DictionaryConst.CONTEXT_RUNTIME));
        String normalizedPath;
        try {
            normalizedPath = KettleAnalyzerUtil.normalizeFilePath(id);
        } catch (MetaverseException e) {
            normalizedPath = id;
        }
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAME, trans.getName());
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_PATH, normalizedPath);
        metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAMESPACE, namespace.getNamespaceId());
        Runnable analyzerRunner = MetaverseUtil.getAnalyzerRunner(documentAnalyzer, metaverseDocument);
        MetaverseCompletionService.getInstance().submit(analyzerRunner, id);
    }
    // Save the lineage objects for later
    LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
    holder.setExecutionProfile(executionProfile);
    holder.setMetaverseBuilder(builder);
}

Example 2 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class TransformationRuntimeExtensionPoint method createLineGraph.

protected void createLineGraph(final Trans trans) {
    log.info(Messages.getString("INFO.WrittingGraphForTransformation", trans.getName()));
    try {
        // Get the current execution profile for this transformation
        LineageHolder holder = TransLineageHolderMap.getInstance().getLineageHolder(trans);
        Future lineageTask = holder.getLineageTask();
        if (lineageTask != null) {
            try {
                lineageTask.get();
            } catch (InterruptedException e) {
            // Do nothing
            } catch (ExecutionException e) {
                log.warn(Messages.getString("ERROR.CouldNotWriteLineageGraph", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
                log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
            }
        }
        IExecutionProfile executionProfile = holder.getExecutionProfile();
        if (executionProfile == null) {
            // Note that this should NEVER happen, this is purely a preventative measure...
            // Something's wrong here, the transStarted method didn't properly store the execution profile. We should know
            // the same info, so populate a new ExecutionProfile using the current Trans
            // TODO: Beware duplicate profiles!
            executionProfile = new ExecutionProfile();
        }
        populateExecutionProfile(executionProfile, trans);
        removeSensitiveDataFromHolder(holder);
        // Export the lineage info (execution profile, lineage graph, etc.)
        try {
            if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                // clearOutput right before the first call to outputXYZ().
                if ("latest".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.cleanOutput(holder);
                }
                lineageWriter.outputExecutionProfile(holder);
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
        try {
            if (shouldCreateGraph(trans)) {
                // Add the execution profile information to the lineage graph
                addRuntimeLineageInfo(holder);
                if (lineageWriter != null && !"none".equals(lineageWriter.getOutputStrategy())) {
                    lineageWriter.outputLineageGraph(holder);
                    // lineage has been written - call the appropriate extension point
                    ExtensionPointHandler.callExtensionPoint(trans.getLogChannel(), MetaverseExtensionPoint.TransLineageWriteEnd.id, trans);
                }
            }
        } catch (IOException e) {
            log.warn(Messages.getString("ERROR.CouldNotWriteExecutionProfile", trans.getName(), Const.NVL(e.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), e);
        }
    } catch (Throwable t) {
        log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", trans.getName(), Const.NVL(t.getLocalizedMessage(), "Unspecified")));
        log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), t);
    }
    // cleanup to prevent unnecessary memory usage - we no longer need this Trans in the TransLineageHolderMap
    TransLineageHolderMap.getInstance().removeLineageHolder(trans);
    log.warn(Messages.getString("INFO.TransformationAnalyzeFinished", trans.getName()));
    logMinimal(Messages.getString("INFO.TransformationAnalyzeFinished", trans.getName()));
}

Example 3 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class LineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    LineageWriter fslw = new LineageWriter();
    writer = spy(fslw);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
}

Example 4 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class VfsLineageWriterTest method setUp.

@Before
public void setUp() throws Exception {
    String basePath = new File(".").getCanonicalPath();
    writer = new VfsLineageWriter();
    writer = spy(writer);
    holder = new LineageHolder();
    IExecutionProfile profile = new ExecutionProfile();
    profile.setName("test");
    IExecutionData data = new ExecutionData();
    data.setStartTime(now);
    profile.setExecutionData(data);
    holder.setExecutionProfile(profile);
    BAD_OUTPUT_FOLDER = FilenameUtils.separatorsToSystem("file://" + basePath + "/target/outputfiles/doesnt_exist" + random.nextInt());
    GOOD_OUTPUT_FOLDER = FilenameUtils.separatorsToSystem("file://" + basePath + "/target/outputfiles" + random.nextInt());
    writer.setOutputFolder(GOOD_OUTPUT_FOLDER);
}

Example 5 with ExecutionProfile

use of org.pentaho.metaverse.impl.model.ExecutionProfile in project pentaho-metaverse by pentaho.

the class BaseRuntimeExtensionPoint method createExecutionProfile.

protected IExecutionProfile createExecutionProfile(final LogChannelInterface logChannelInterface, final Object o) {
    // create the Execution profile and store within the LineageHolder so that it's available to any
    // potential row listeners
    final LineageHolder holder = getLineageHolder(o);
    IExecutionProfile executionProfile = holder.getExecutionProfile();
    if (executionProfile == null) {
        executionProfile = new ExecutionProfile();
        executionProfile.getExecutionData().setStartTime(new Timestamp(new Date().getTime()));
        if (logChannelInterface != null) {
            executionProfile.getExecutionData().setLoggingChannelId(logChannelInterface.getLogChannelId());
        }
        holder.setExecutionProfile(executionProfile);
    }
    return executionProfile;
}