Examples with INamespace org.pentaho.metaverse.api.INamespace used on opensource projects

Example 6 with INamespace

use of org.pentaho.metaverse.api.INamespace in project pentaho-metaverse by pentaho.

the class BaseKettleMetaverseComponentTest method testCreateNodeFromDescriptor.

@Test
public void testCreateNodeFromDescriptor() throws Exception {
    String namespaceId = "{namespace: 'my namespace', name: 'my name', type: 'my type'}";
    ILogicalIdGenerator idGenerator = DictionaryConst.LOGICAL_ID_GENERATOR_DEFAULT;
    IComponentDescriptor descriptor = mock(IComponentDescriptor.class);
    INamespace ns = mock(INamespace.class);
    when(descriptor.getParentNamespace()).thenReturn(mock(INamespace.class));
    when(descriptor.getNamespace()).thenReturn(ns);
    when(ns.getNamespaceId()).thenReturn(namespaceId);
    component.metaverseObjectFactory = spy(new MetaverseObjectFactory());
    IMetaverseNode node = component.createNodeFromDescriptor(descriptor, idGenerator);
    assertNotNull(node);
    assertTrue(node.getLogicalId().contains(namespaceId));
}

Example 7 with INamespace

use of org.pentaho.metaverse.api.INamespace in project pentaho-metaverse by pentaho.

the class JobAnalyzer method analyze.

@Override
public synchronized IMetaverseNode analyze(IComponentDescriptor descriptor, IDocument document) throws MetaverseAnalyzerException {
    validateState(document);
    Object repoObject = document.getContent();
    JobMeta jobMeta = null;
    if (repoObject instanceof String) {
        // hydrate the job
        try {
            String content = (String) repoObject;
            ByteArrayInputStream xmlStream = new ByteArrayInputStream(content.getBytes());
            jobMeta = new JobMeta(xmlStream, null, null);
        } catch (KettleXMLException e) {
            throw new MetaverseAnalyzerException(e);
        }
    } else if (repoObject instanceof JobMeta) {
        jobMeta = (JobMeta) repoObject;
    }
    // construct a dummy job based on our JobMeta so we get out VariableSpace set properly
    jobMeta.setFilename(document.getStringID());
    Job j = new Job(null, jobMeta);
    j.setInternalKettleVariables(jobMeta);
    IComponentDescriptor documentDescriptor = new MetaverseComponentDescriptor(document.getStringID(), DictionaryConst.NODE_TYPE_JOB, new Namespace(descriptor.getLogicalId()), descriptor.getContext());
    // Create a metaverse node and start filling in details
    IMetaverseNode node = metaverseObjectFactory.createNodeObject(document.getNamespace(), jobMeta.getName(), DictionaryConst.NODE_TYPE_JOB);
    node.setLogicalIdGenerator(DictionaryConst.LOGICAL_ID_GENERATOR_DOCUMENT);
    // pull out the standard fields
    String description = jobMeta.getDescription();
    if (description != null) {
        node.setProperty(DictionaryConst.PROPERTY_DESCRIPTION, description);
    }
    String extendedDescription = jobMeta.getExtendedDescription();
    if (extendedDescription != null) {
        node.setProperty("extendedDescription", extendedDescription);
    }
    Date createdDate = jobMeta.getCreatedDate();
    if (createdDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED, Long.toString(createdDate.getTime()));
    }
    String createdUser = jobMeta.getCreatedUser();
    if (createdUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED_BY, createdUser);
    }
    Date lastModifiedDate = jobMeta.getModifiedDate();
    if (lastModifiedDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED, Long.toString(lastModifiedDate.getTime()));
    }
    String lastModifiedUser = jobMeta.getModifiedUser();
    if (lastModifiedUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED_BY, lastModifiedUser);
    }
    String version = jobMeta.getJobversion();
    if (version != null) {
        node.setProperty(DictionaryConst.PROPERTY_ARTIFACT_VERSION, version);
    }
    String status = Messages.getString("INFO.JobOrTrans.Status_" + Integer.toString(jobMeta.getJobstatus()));
    if (status != null && !status.startsWith("!")) {
        node.setProperty(DictionaryConst.PROPERTY_STATUS, status);
    }
    node.setProperty(DictionaryConst.PROPERTY_PATH, document.getProperty(DictionaryConst.PROPERTY_PATH));
    // Process job parameters
    String[] parameters = jobMeta.listParameters();
    if (parameters != null) {
        for (String parameter : parameters) {
            try {
                // Determine parameter properties and add them to a map, then the map to the list
                String defaultParameterValue = jobMeta.getParameterDefault(parameter);
                String parameterValue = jobMeta.getParameterValue(parameter);
                String parameterDescription = jobMeta.getParameterDescription(parameter);
                PropertiesHolder paramProperties = new PropertiesHolder();
                paramProperties.setProperty("defaultValue", defaultParameterValue);
                paramProperties.setProperty("value", parameterValue);
                paramProperties.setProperty("description", parameterDescription);
                node.setProperty("parameter_" + parameter, paramProperties.toString());
            } catch (UnknownParamException upe) {
                // This shouldn't happen as we're using the list provided by the meta
                throw new MetaverseAnalyzerException(upe);
            }
        }
    }
    // handle the entries
    for (int i = 0; i < jobMeta.nrJobEntries(); i++) {
        JobEntryCopy entry = jobMeta.getJobEntry(i);
        try {
            if (entry != null) {
                entry.getEntry().setParentJob(j);
                IMetaverseNode jobEntryNode = null;
                JobEntryInterface jobEntryInterface = entry.getEntry();
                IComponentDescriptor entryDescriptor = new MetaverseComponentDescriptor(entry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY, node, descriptor.getContext());
                Set<IJobEntryAnalyzer> jobEntryAnalyzers = getJobEntryAnalyzers(jobEntryInterface);
                if (jobEntryAnalyzers != null && !jobEntryAnalyzers.isEmpty()) {
                    for (IJobEntryAnalyzer jobEntryAnalyzer : jobEntryAnalyzers) {
                        jobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                        jobEntryNode = (IMetaverseNode) jobEntryAnalyzer.analyze(entryDescriptor, entry.getEntry());
                    }
                } else {
                    GenericJobEntryMetaAnalyzer defaultJobEntryAnalyzer = new GenericJobEntryMetaAnalyzer();
                    defaultJobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                    jobEntryNode = defaultJobEntryAnalyzer.analyze(entryDescriptor, jobEntryInterface);
                }
                if (jobEntryNode != null) {
                    metaverseBuilder.addLink(node, DictionaryConst.LINK_CONTAINS, jobEntryNode);
                }
            }
        } catch (Throwable mae) {
            // Don't throw an exception, just log and carry on
            log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", entry.getName(), Const.NVL(mae.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), mae);
        }
    }
    // Model the hops between steps
    int numHops = jobMeta.nrJobHops();
    for (int i = 0; i < numHops; i++) {
        JobHopMeta hop = jobMeta.getJobHop(i);
        JobEntryCopy fromEntry = hop.getFromEntry();
        JobEntryCopy toEntry = hop.getToEntry();
        INamespace childNs = new Namespace(node.getLogicalId());
        // process legitimate hops
        if (fromEntry != null && toEntry != null) {
            IMetaverseNode fromEntryNode = metaverseObjectFactory.createNodeObject(childNs, fromEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            IMetaverseNode toEntryNode = metaverseObjectFactory.createNodeObject(childNs, toEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            metaverseBuilder.addLink(fromEntryNode, DictionaryConst.LINK_HOPSTO, toEntryNode);
        }
    }
    metaverseBuilder.addNode(node);
    addParentLink(documentDescriptor, node);
    return node;
}

Example 8 with INamespace

use of org.pentaho.metaverse.api.INamespace in project pentaho-metaverse by pentaho.

the class TransformationAnalyzer method analyze.

@Override
public synchronized IMetaverseNode analyze(IComponentDescriptor descriptor, IDocument document) throws MetaverseAnalyzerException {
    validateState(document);
    Object repoObject = document.getContent();
    TransMeta transMeta = null;
    if (repoObject instanceof String) {
        // hydrate the transformation
        try {
            String content = (String) repoObject;
            ByteArrayInputStream xmlStream = new ByteArrayInputStream(content.getBytes());
            transMeta = new TransMeta(xmlStream, null, false, null, null);
            transMeta.setFilename(document.getStringID());
            if (transMeta.hasMissingPlugins()) {
                throw new MetaverseAnalyzerException(Messages.getErrorString("ERROR.MissingPlugin"));
            }
        } catch (KettleException e) {
            throw new MetaverseAnalyzerException(e);
        }
    } else if (repoObject instanceof TransMeta) {
        transMeta = (TransMeta) repoObject;
    }
    Trans t = new Trans(transMeta);
    t.setInternalKettleVariables(transMeta);
    IComponentDescriptor documentDescriptor = new MetaverseComponentDescriptor(document.getStringID(), DictionaryConst.NODE_TYPE_TRANS, new Namespace(descriptor.getLogicalId()), descriptor.getContext());
    // Create a metaverse node and start filling in details
    IMetaverseNode node = metaverseObjectFactory.createNodeObject(document.getNamespace(), transMeta.getName(), DictionaryConst.NODE_TYPE_TRANS);
    node.setLogicalIdGenerator(DictionaryConst.LOGICAL_ID_GENERATOR_DOCUMENT);
    // pull out the standard fields
    String description = transMeta.getDescription();
    if (description != null) {
        node.setProperty(DictionaryConst.PROPERTY_DESCRIPTION, description);
    }
    String extendedDescription = transMeta.getExtendedDescription();
    if (extendedDescription != null) {
        node.setProperty("extendedDescription", extendedDescription);
    }
    Date createdDate = transMeta.getCreatedDate();
    if (createdDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED, Long.toString(createdDate.getTime()));
    }
    String createdUser = transMeta.getCreatedUser();
    if (createdUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED_BY, createdUser);
    }
    Date lastModifiedDate = transMeta.getModifiedDate();
    if (lastModifiedDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED, Long.toString(lastModifiedDate.getTime()));
    }
    String lastModifiedUser = transMeta.getModifiedUser();
    if (lastModifiedUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED_BY, lastModifiedUser);
    }
    String version = transMeta.getTransversion();
    if (version != null) {
        node.setProperty(DictionaryConst.PROPERTY_ARTIFACT_VERSION, version);
    }
    String status = Messages.getString("INFO.JobOrTrans.Status_" + Integer.toString(transMeta.getTransstatus()));
    if (status != null && !status.startsWith("!")) {
        node.setProperty(DictionaryConst.PROPERTY_STATUS, status);
    }
    node.setProperty(DictionaryConst.PROPERTY_PATH, document.getProperty(DictionaryConst.PROPERTY_PATH));
    String[] parameters = transMeta.listParameters();
    if (parameters != null) {
        for (String parameter : parameters) {
            try {
                // Determine parameter properties and add them to a map, then the map to the list
                String defaultParameterValue = transMeta.getParameterDefault(parameter);
                String parameterValue = transMeta.getParameterValue(parameter);
                String parameterDescription = transMeta.getParameterDescription(parameter);
                PropertiesHolder paramProperties = new PropertiesHolder();
                paramProperties.setProperty("defaultValue", defaultParameterValue);
                paramProperties.setProperty("value", parameterValue);
                paramProperties.setProperty("description", parameterDescription);
                node.setProperty("parameter_" + parameter, paramProperties.toString());
            } catch (UnknownParamException upe) {
                // This shouldn't happen as we're using the list provided by the meta
                throw new MetaverseAnalyzerException(upe);
            }
        }
    }
    // handle the step
    for (int stepNr = 0; stepNr < transMeta.nrSteps(); stepNr++) {
        StepMeta stepMeta = transMeta.getStep(stepNr);
        try {
            if (stepMeta != null) {
                if (stepMeta.getParentTransMeta() == null) {
                    stepMeta.setParentTransMeta(transMeta);
                }
                IMetaverseNode stepNode = null;
                IComponentDescriptor stepDescriptor = new MetaverseComponentDescriptor(stepMeta.getName(), DictionaryConst.NODE_TYPE_TRANS_STEP, node, documentDescriptor.getContext());
                Set<IStepAnalyzer> stepAnalyzers = getStepAnalyzers(stepMeta);
                if (stepAnalyzers != null && !stepAnalyzers.isEmpty()) {
                    for (IStepAnalyzer stepAnalyzer : stepAnalyzers) {
                        stepAnalyzer.setMetaverseBuilder(metaverseBuilder);
                        stepNode = (IMetaverseNode) stepAnalyzer.analyze(stepDescriptor, getBaseStepMetaFromStepMeta(stepMeta));
                    }
                } else {
                    GenericStepMetaAnalyzer defaultStepAnalyzer = new GenericStepMetaAnalyzer();
                    defaultStepAnalyzer.setMetaverseBuilder(metaverseBuilder);
                    stepNode = defaultStepAnalyzer.analyze(stepDescriptor, getBaseStepMetaFromStepMeta(stepMeta));
                }
                if (stepNode != null) {
                    metaverseBuilder.addLink(node, DictionaryConst.LINK_CONTAINS, stepNode);
                }
            }
        } catch (Throwable mae) {
            // Don't throw an exception, just log and carry on
            log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", stepMeta.getName(), Const.NVL(mae.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), mae);
        }
    }
    // Model the hops between steps
    int numHops = transMeta.nrTransHops();
    for (int i = 0; i < numHops; i++) {
        TransHopMeta hop = transMeta.getTransHop(i);
        StepMeta fromStep = hop.getFromStep();
        StepMeta toStep = hop.getToStep();
        INamespace childNs = new Namespace(node.getLogicalId());
        // process legitimate hops
        if (fromStep != null && toStep != null) {
            IMetaverseNode fromStepNode = metaverseObjectFactory.createNodeObject(childNs, fromStep.getName(), DictionaryConst.NODE_TYPE_TRANS_STEP);
            IMetaverseNode toStepNode = metaverseObjectFactory.createNodeObject(childNs, toStep.getName(), DictionaryConst.NODE_TYPE_TRANS_STEP);
            // Create and decorate the link between the steps
            IMetaverseLink link = metaverseObjectFactory.createLinkObject();
            link.setFromNode(fromStepNode);
            link.setLabel(DictionaryConst.LINK_HOPSTO);
            link.setToNode(toStepNode);
            // Is this hop enabled?
            link.setProperty(DictionaryConst.PROPERTY_ENABLED, hop.isEnabled());
            // Add metadata about the type of stream (target, error, info) it is. Default to "target".
            String linkType = "target";
            if (fromStep.isSendingErrorRowsToStep(toStep)) {
                linkType = "error";
            } else {
                String[] infoStepnames = toStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
                // If the "from" step is the source of an info stream to the "to" step, it's an "info" hop
                if (Const.indexOfString(fromStep.getName(), infoStepnames) >= 0) {
                    linkType = "info";
                }
            }
            link.setProperty(DictionaryConst.PROPERTY_TYPE, linkType);
            metaverseBuilder.addLink(link);
        }
    }
    metaverseBuilder.addNode(node);
    addParentLink(documentDescriptor, node);
    return node;
}

Example 9 with INamespace

use of org.pentaho.metaverse.api.INamespace in project pentaho-metaverse by pentaho.

the class JobRuntimeExtensionPoint method callExtensionPoint.

/**
 * Callback when a job is about to be started
 *
 * @param logChannelInterface A reference to the log in this context (the Job object's log)
 * @param o                   The object being operated on (Job in this case)
 * @throws org.pentaho.di.core.exception.KettleException
 */
@Override
public void callExtensionPoint(LogChannelInterface logChannelInterface, Object o) throws KettleException {
    // Job Started listeners get called after the extension point is invoked, so just add a job listener
    if (o instanceof Job) {
        Job job = ((Job) o);
        // If runtime lineage collection is disabled, don't run any lineage processes/methods
        if (!isRuntimeEnabled()) {
            return;
        }
        // Create and populate an execution profile with what we know so far
        ExecutionProfile executionProfile = new ExecutionProfile();
        populateExecutionProfile(executionProfile, job);
        IMetaverseBuilder builder = JobLineageHolderMap.getInstance().getMetaverseBuilder(job);
        // Add the job finished listener
        job.addJobListener(this);
        // Analyze the current transformation
        if (documentAnalyzer != null) {
            documentAnalyzer.setMetaverseBuilder(builder);
            // Create a document for the Trans
            final String clientName = executionProfile.getExecutionEngine().getName();
            final INamespace namespace = new Namespace(clientName);
            final IMetaverseNode designNode = builder.getMetaverseObjectFactory().createNodeObject(clientName, clientName, DictionaryConst.NODE_TYPE_LOCATOR);
            builder.addNode(designNode);
            final JobMeta jobMeta = job.getJobMeta();
            // The variables and parameters in the Job may not have been set on the meta, so we do it here
            // to ensure the job analyzer will have access to the parameter values.
            jobMeta.copyParametersFrom(job);
            jobMeta.activateParameters();
            job.copyVariablesFrom(jobMeta);
            if (job.getRep() != null) {
                jobMeta.setRepository(job.getRep());
            }
            String id = getFilename(jobMeta);
            if (!id.endsWith(jobMeta.getDefaultExtension())) {
                id += "." + jobMeta.getDefaultExtension();
            }
            IDocument metaverseDocument = builder.getMetaverseObjectFactory().createDocumentObject();
            metaverseDocument.setNamespace(namespace);
            metaverseDocument.setContent(jobMeta);
            metaverseDocument.setStringID(id);
            metaverseDocument.setName(jobMeta.getName());
            metaverseDocument.setExtension("kjb");
            metaverseDocument.setMimeType(URLConnection.getFileNameMap().getContentTypeFor("job.kjb"));
            metaverseDocument.setContext(new AnalysisContext(DictionaryConst.CONTEXT_RUNTIME));
            String normalizedPath;
            try {
                normalizedPath = KettleAnalyzerUtil.normalizeFilePath(id);
            } catch (MetaverseException e) {
                normalizedPath = id;
            }
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAME, job.getName());
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_PATH, normalizedPath);
            metaverseDocument.setProperty(DictionaryConst.PROPERTY_NAMESPACE, namespace.getNamespaceId());
            Runnable analyzerRunner = MetaverseUtil.getAnalyzerRunner(documentAnalyzer, metaverseDocument);
            MetaverseCompletionService.getInstance().submit(analyzerRunner, id);
        }
        // Save the lineage objects for later
        LineageHolder holder = JobLineageHolderMap.getInstance().getLineageHolder(job);
        holder.setExecutionProfile(executionProfile);
        holder.setMetaverseBuilder(builder);
    }
}

Example 10 with INamespace

use of org.pentaho.metaverse.api.INamespace in project pentaho-metaverse by pentaho.

the class MetaverseUtilTest method testCreateDocument.

@Test
public void testCreateDocument() throws Exception {
    Object content = new Object();
    INamespace namespace = mock(INamespace.class);
    IDocument document = MetaverseUtil.createDocument(namespace, content, "myID", "myName", "myExtension", "application/text");
    assertEquals(document.getNamespace(), namespace);
    assertEquals(document.getContent(), content);
    assertEquals(document.getStringID(), "myID");
    assertEquals(document.getName(), "myName");
    assertEquals(document.getMimeType(), "application/text");
}