Examples with IJobEntryAnalyzer org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer used on opensource projects

Example 1 with IJobEntryAnalyzer

use of org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer in project pentaho-metaverse by pentaho.

the class JobEntryAnalyzerProviderTest method TestAddAndSetAnalyzers.

@Test
public void TestAddAndSetAnalyzers() {
    final IJobEntryAnalyzer analyzer1 = mock(IJobEntryAnalyzer.class);
    final IJobEntryAnalyzer analyzer2 = mock(IJobEntryAnalyzer.class);
    List<IJobEntryAnalyzer> analyzers = new ArrayList();
    analyzers.add(analyzer1);
    analyzers.add(analyzer2);
    provider.setJobEntryAnalyzers(analyzers);
    assertEquals(2, provider.getAnalyzers().size());
    // verify that duplicate analyzers aren't added to the list
    provider.setJobEntryAnalyzers(analyzers);
    assertEquals(2, provider.getAnalyzers().size());
    provider.setJobEntryAnalyzers(null);
    assertNull(provider.getAnalyzers());
    // verify that "clonable" analyzers are added to the main analyzers list
    provider.setClonableJobEntryAnalyzers(analyzers);
    assertEquals(2, provider.getAnalyzers().size());
    // and that duplicate clonable analyzers aren't added to the list
    provider.setClonableJobEntryAnalyzers(analyzers);
    assertEquals(2, provider.getAnalyzers().size());
    provider.setClonableJobEntryAnalyzers(null);
    assertNull(provider.getAnalyzers());
}

Example 2 with IJobEntryAnalyzer

use of org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer in project pentaho-metaverse by pentaho.

the class JobEntryAnalyzerProviderTest method testRemoveAnalyzer.

@Test
public void testRemoveAnalyzer() throws Exception {
    IJobEntryAnalyzer baseStepAnalyzer = mock(IJobEntryAnalyzer.class);
    when(baseStepAnalyzer.getSupportedEntries()).thenReturn(Sets.newSet(JobEntryInterface.class));
    IJobEntryAnalyzer jobEntryTransAnalyzer = mock(IJobEntryAnalyzer.class);
    when(jobEntryTransAnalyzer.getSupportedEntries()).thenReturn(Sets.newSet(JobEntryTrans.class));
    provider.setJobEntryAnalyzers(Lists.newArrayList(baseStepAnalyzer, jobEntryTransAnalyzer));
    Set<IJobEntryAnalyzer> tableOutputStepAnalyzers = provider.analyzerTypeMap.get(JobEntryTrans.class);
    assertNotNull(tableOutputStepAnalyzers);
    assertEquals(tableOutputStepAnalyzers.size(), 1);
    provider.removeAnalyzer(jobEntryTransAnalyzer);
    tableOutputStepAnalyzers = provider.analyzerTypeMap.get(JobEntryTrans.class);
    assertNull(tableOutputStepAnalyzers);
}

Example 3 with IJobEntryAnalyzer

use of org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer in project pentaho-metaverse by pentaho.

the class JobEntryAnalyzerProviderTest method testLoadAnalyzerTypeMap.

@SuppressWarnings("unchecked")
@Test
public void testLoadAnalyzerTypeMap() throws Exception {
    IJobEntryAnalyzer baseStepAnalyzer = mock(IJobEntryAnalyzer.class);
    when(baseStepAnalyzer.getSupportedEntries()).thenReturn(Sets.newSet(JobEntryInterface.class));
    IJobEntryAnalyzer tableOutputStepAnalyzer = mock(IJobEntryAnalyzer.class);
    when(tableOutputStepAnalyzer.getSupportedEntries()).thenReturn(Sets.newSet(JobEntryTrans.class));
    IJobEntryAnalyzer tableOutputStepAnalyzer2 = mock(IJobEntryAnalyzer.class);
    when(tableOutputStepAnalyzer2.getSupportedEntries()).thenReturn(Sets.newSet(JobEntryTrans.class));
    provider.jobEntryAnalyzers = Lists.newArrayList(baseStepAnalyzer, tableOutputStepAnalyzer, tableOutputStepAnalyzer2);
    // Method under test
    provider.loadAnalyzerTypeMap();
    Set<IJobEntryAnalyzer> baseStepAnalyzers = provider.analyzerTypeMap.get(JobEntryInterface.class);
    assertNotNull(baseStepAnalyzers);
    assertEquals(baseStepAnalyzers.size(), 1);
    Set<IJobEntryAnalyzer> tableOutputStepAnalyzers = provider.analyzerTypeMap.get(JobEntryTrans.class);
    assertNotNull(tableOutputStepAnalyzers);
    assertEquals(tableOutputStepAnalyzers.size(), 2);
}

Example 4 with IJobEntryAnalyzer

use of org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer in project pentaho-metaverse by pentaho.

the class JobAnalyzer method analyze.

@Override
public synchronized IMetaverseNode analyze(final IComponentDescriptor documentDescriptor, final AbstractMeta meta, final IMetaverseNode node, final String documentPath) throws MetaverseAnalyzerException {
    final JobMeta jobMeta = (JobMeta) meta;
    Job j = new Job(null, jobMeta);
    j.setInternalKettleVariables(jobMeta);
    // pull out the standard fields
    String description = jobMeta.getDescription();
    if (description != null) {
        node.setProperty(DictionaryConst.PROPERTY_DESCRIPTION, description);
    }
    String extendedDescription = jobMeta.getExtendedDescription();
    if (extendedDescription != null) {
        node.setProperty("extendedDescription", extendedDescription);
    }
    Date createdDate = jobMeta.getCreatedDate();
    if (createdDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED, Long.toString(createdDate.getTime()));
    }
    String createdUser = jobMeta.getCreatedUser();
    if (createdUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED_BY, createdUser);
    }
    Date lastModifiedDate = jobMeta.getModifiedDate();
    if (lastModifiedDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED, Long.toString(lastModifiedDate.getTime()));
    }
    String lastModifiedUser = jobMeta.getModifiedUser();
    if (lastModifiedUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED_BY, lastModifiedUser);
    }
    String version = jobMeta.getJobversion();
    if (version != null) {
        node.setProperty(DictionaryConst.PROPERTY_ARTIFACT_VERSION, version);
    }
    String status = Messages.getString("INFO.JobOrTrans.Status_" + Integer.toString(jobMeta.getJobstatus()));
    if (status != null && !status.startsWith("!")) {
        node.setProperty(DictionaryConst.PROPERTY_STATUS, status);
    }
    node.setProperty(DictionaryConst.PROPERTY_PATH, documentPath);
    // Process job parameters
    String[] parameters = jobMeta.listParameters();
    if (parameters != null) {
        for (String parameter : parameters) {
            try {
                // Determine parameter properties and add them to a map, then the map to the list
                String defaultParameterValue = jobMeta.getParameterDefault(parameter);
                String parameterValue = jobMeta.getParameterValue(parameter);
                String parameterDescription = jobMeta.getParameterDescription(parameter);
                PropertiesHolder paramProperties = new PropertiesHolder();
                paramProperties.setProperty("defaultValue", defaultParameterValue);
                paramProperties.setProperty("value", parameterValue);
                paramProperties.setProperty("description", parameterDescription);
                node.setProperty("parameter_" + parameter, paramProperties.toString());
            } catch (UnknownParamException upe) {
                // This shouldn't happen as we're using the list provided by the meta
                throw new MetaverseAnalyzerException(upe);
            }
        }
    }
    // handle the entries
    for (int i = 0; i < jobMeta.nrJobEntries(); i++) {
        JobEntryCopy entry = jobMeta.getJobEntry(i);
        try {
            if (entry != null) {
                entry.getEntry().setParentJob(j);
                IMetaverseNode jobEntryNode = null;
                JobEntryInterface jobEntryInterface = entry.getEntry();
                IComponentDescriptor entryDescriptor = new MetaverseComponentDescriptor(entry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY, node, documentDescriptor.getContext());
                Set<IJobEntryAnalyzer> jobEntryAnalyzers = getJobEntryAnalyzers(jobEntryInterface);
                if (jobEntryAnalyzers != null && !jobEntryAnalyzers.isEmpty()) {
                    for (IJobEntryAnalyzer jobEntryAnalyzer : jobEntryAnalyzers) {
                        // change while the job is being analyzed
                        if (jobEntryAnalyzer instanceof IClonableJobEntryAnalyzer) {
                            jobEntryAnalyzer = ((IClonableJobEntryAnalyzer) jobEntryAnalyzer).cloneAnalyzer();
                            ((IClonableJobEntryAnalyzer) jobEntryAnalyzer).setDocumentAnalyzer(this);
                            ((IClonableJobEntryAnalyzer) jobEntryAnalyzer).setDocumentDescriptor(documentDescriptor);
                            ((IClonableJobEntryAnalyzer) jobEntryAnalyzer).setDocumentPath(documentPath);
                        } else {
                            log.debug(Messages.getString("WARNING.CannotCloneAnalyzer"), jobEntryAnalyzer);
                        }
                        jobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                        jobEntryNode = (IMetaverseNode) jobEntryAnalyzer.analyze(entryDescriptor, entry.getEntry());
                    }
                } else if (new AnnotatedClassFields(jobEntryInterface, jobEntryInterface.getParentJobMeta()).hasMetaverseAnnotations()) {
                    AnnotationDrivenJobAnalyzer annotationDrivenJobAnalyzer = new AnnotationDrivenJobAnalyzer(jobEntryInterface);
                    annotationDrivenJobAnalyzer.setMetaverseBuilder(metaverseBuilder);
                    annotationDrivenJobAnalyzer.setDocumentAnalyzer(this);
                    annotationDrivenJobAnalyzer.setDocumentDescriptor(documentDescriptor);
                    annotationDrivenJobAnalyzer.setDocumentPath(documentPath);
                    jobEntryNode = annotationDrivenJobAnalyzer.analyze(entryDescriptor, jobEntryInterface);
                } else {
                    GenericJobEntryMetaAnalyzer defaultJobEntryAnalyzer = new GenericJobEntryMetaAnalyzer();
                    defaultJobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                    jobEntryNode = defaultJobEntryAnalyzer.analyze(entryDescriptor, jobEntryInterface);
                }
                if (jobEntryNode != null) {
                    metaverseBuilder.addLink(node, DictionaryConst.LINK_CONTAINS, jobEntryNode);
                }
            }
        } catch (Exception mae) {
            // Don't throw an exception, just log and carry on
            log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", entry.getName(), Const.NVL(mae.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), mae);
        }
    }
    // Model the hops between steps
    int numHops = jobMeta.nrJobHops();
    for (int i = 0; i < numHops; i++) {
        JobHopMeta hop = jobMeta.getJobHop(i);
        JobEntryCopy fromEntry = hop.getFromEntry();
        JobEntryCopy toEntry = hop.getToEntry();
        INamespace childNs = new Namespace(node.getLogicalId());
        // process legitimate hops
        if (fromEntry != null && toEntry != null) {
            IMetaverseNode fromEntryNode = metaverseObjectFactory.createNodeObject(childNs, fromEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            IMetaverseNode toEntryNode = metaverseObjectFactory.createNodeObject(childNs, toEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            metaverseBuilder.addLink(fromEntryNode, DictionaryConst.LINK_HOPSTO, toEntryNode);
        }
    }
    metaverseBuilder.addNode(node);
    addParentLink(documentDescriptor, node);
    return node;
}

Example 5 with IJobEntryAnalyzer

use of org.pentaho.metaverse.api.analyzer.kettle.jobentry.IJobEntryAnalyzer in project pentaho-metaverse by pentaho.

the class AnalyzerInfoService method getSupportedJobEntries.

/**
 * Gets a list of all implementations of {@link JobEntryInterface} with a custom {@link IJobEntryAnalyzer}
 * that produces lineage. Any step not found in this list will fall back to using
 * {@link org.pentaho.metaverse.analyzer.kettle.jobentry.GenericJobEntryMetaAnalyzer}.
 *
 * <p><b>Example Request:</b><br />
 *    GET pentaho-di/osgi/cxf/lineage/info/entries
 * </p>
 *
 * @return List of {@link AnalyzerInfo}
 *
 * <p><b>Example Response:</b></p>
 *    <pre function="syntax.js">
 *      [ { meta: "JobEntryTrans" } ]
 *    </pre>
 */
@GET
@Path("/entries")
@Produces({ MediaType.APPLICATION_JSON })
@StatusCodes({ @ResponseCode(code = OK, condition = "Successfully listed the supported job entries"), @ResponseCode(code = SERVER_ERROR, condition = "Server Error.") })
public Response getSupportedJobEntries() {
    List<AnalyzerInfo> analyzers = new ArrayList<>();
    for (IJobEntryAnalyzer analyzer : getJobEntryAnalyzerProvider().getAnalyzers()) {
        Set<Class<? extends JobEntryInterface>> supportedEntries = analyzer.getSupportedEntries();
        for (Class<? extends JobEntryInterface> supportedEntry : supportedEntries) {
            AnalyzerInfo info = new AnalyzerInfo(supportedEntry.getSimpleName());
            analyzers.add(info);
        }
    }
    Collections.sort(analyzers, new AnalyzerInfoComparator());
    return Response.ok(analyzers).build();
}