Examples with JobEntryCopy org.pentaho.di.job.entry.JobEntryCopy used on opensource projects

Example 76 with JobEntryCopy

use of org.pentaho.di.job.entry.JobEntryCopy in project pentaho-metaverse by pentaho.

the class JobEntryExternalResourceListenerTest method testBeforeAfterExecution.

@Test
public void testBeforeAfterExecution() throws Exception {
    IJobEntryExternalResourceConsumer consumer = mock(IJobEntryExternalResourceConsumer.class);
    JobMeta mockJobMeta = mock(JobMeta.class);
    Job job = mock(Job.class);
    when(job.getJobMeta()).thenReturn(mockJobMeta);
    JobEntryInterface jobEntryInterface = mock(JobEntryInterface.class);
    when(jobEntryInterface.getParentJob()).thenReturn(job);
    when(jobEntryInterface.getResourceDependencies(mockJobMeta)).thenReturn(Collections.singletonList(new ResourceReference(null, Collections.singletonList(new ResourceEntry("myFile", ResourceEntry.ResourceType.FILE)))));
    JobEntryCopy jobEntryCopy = mock(JobEntryCopy.class);
    IExecutionProfile executionProfile = mock(IExecutionProfile.class);
    IExecutionData executionData = mock(IExecutionData.class);
    when(executionProfile.getExecutionData()).thenReturn(executionData);
    JobLineageHolderMap.getInstance().getLineageHolder(job).setExecutionProfile(executionProfile);
    JobEntryExternalResourceListener listener = new JobEntryExternalResourceListener(consumer);
    FileObject mockFile = mock(FileObject.class);
    FileName mockFilename = mock(FileName.class);
    when(mockFilename.getPath()).thenReturn("/path/to/file");
    when(mockFile.getName()).thenReturn(mockFilename);
    ResultFile resultFile = mock(ResultFile.class);
    when(resultFile.getFile()).thenReturn(mockFile);
    List<ResultFile> resultFiles = Collections.singletonList(resultFile);
    Result result = mock(Result.class);
    when(result.getResultFilesList()).thenReturn(resultFiles);
    // Call beforeExecution for coverage
    listener.beforeExecution(null, null, null);
    listener.afterExecution(job, jobEntryCopy, jobEntryInterface, result);
}

Example 77 with JobEntryCopy

use of org.pentaho.di.job.entry.JobEntryCopy in project pentaho-metaverse by pentaho.

the class JobMetaJsonSerializerTest method testSerializeHops.

@Test
public void testSerializeHops() throws Exception {
    JobHopMeta jobHopMeta = mock(JobHopMeta.class);
    JobEntryCopy fromJobEntry = mock(JobEntryCopy.class);
    JobEntryCopy toJobEntry = mock(JobEntryCopy.class);
    when(meta.nrJobHops()).thenReturn(2);
    when(meta.getJobHop(anyInt())).thenReturn(jobHopMeta);
    when(jobHopMeta.getFromEntry()).thenReturn(fromJobEntry);
    when(jobHopMeta.getToEntry()).thenReturn(toJobEntry);
    when(jobHopMeta.isEnabled()).thenReturn(true);
    when(fromJobEntry.getName()).thenReturn("from");
    when(toJobEntry.getName()).thenReturn("to");
    serializer.serializeHops(meta, json);
    verify(json, times(2)).writeObject(any(HopInfo.class));
}

Example 78 with JobEntryCopy

use of org.pentaho.di.job.entry.JobEntryCopy in project pentaho-metaverse by pentaho.

the class JobEntryAnalyzerTest method setUp.

/**
 * @throws Exception
 */
@Before
public void setUp() throws Exception {
    IMetaverseObjectFactory factory = MetaverseTestUtils.getMetaverseObjectFactory();
    when(mockBuilder.getMetaverseObjectFactory()).thenReturn(factory);
    JobEntryAnalyzer baseAnalyzer = new JobEntryAnalyzer() {

        @Override
        public Set<Class<? super JobEntryCopy>> getSupportedEntries() {
            return null;
        }

        @Override
        protected void customAnalyze(JobEntryInterface entry, IMetaverseNode rootNode) throws MetaverseAnalyzerException {
        // TODO Auto-generated method stub
        }
    };
    analyzer = spy(baseAnalyzer);
    analyzer.setMetaverseBuilder(mockBuilder);
    when(mockEntry.getEntry()).thenReturn(mockJobEntryInterface);
    when(mockJobEntryInterface.getPluginId()).thenReturn("Base job entry");
    when(mockJobEntryInterface.getParentJob()).thenReturn(mockJob);
    when(mockJob.getJobMeta()).thenReturn(mockJobMeta);
}

Example 79 with JobEntryCopy

use of org.pentaho.di.job.entry.JobEntryCopy in project pentaho-metaverse by pentaho.

the class JobMetaJsonSerializer method serializeSteps.

@Override
protected void serializeSteps(JobMeta meta, JsonGenerator json) throws IOException {
    json.writeArrayFieldStart(JSON_PROPERTY_STEPS);
    int numberOfEntries = meta.nrJobEntries();
    for (int i = 0; i < numberOfEntries; i++) {
        JobEntryCopy jobEntry = meta.getJobEntry(i);
        LineageRepository repo = getLineageRepository();
        ObjectId jobId = meta.getObjectId() == null ? new StringObjectId(meta.getName()) : meta.getObjectId();
        ObjectId entryId = jobEntry.getObjectId() == null ? new StringObjectId(jobEntry.getName()) : jobEntry.getObjectId();
        JobEntryInterface jobEntryInterface = jobEntry.getEntry();
        JobEntryBase jobEntryBase = getJobEntryBase(jobEntryInterface);
        Job job = new Job(null, meta);
        jobEntryBase.setParentJob(job);
        jobEntryInterface.setObjectId(entryId);
        try {
            jobEntryInterface.saveRep(repo, null, jobId);
        } catch (KettleException e) {
            LOGGER.warn(Messages.getString("INFO.Serialization.Trans.Step", jobEntry.getName()), e);
        }
        json.writeObject(jobEntryBase);
    }
    json.writeEndArray();
}

Example 80 with JobEntryCopy

use of org.pentaho.di.job.entry.JobEntryCopy in project pentaho-metaverse by pentaho.

the class JobAnalyzer method analyze.

@Override
public synchronized IMetaverseNode analyze(IComponentDescriptor descriptor, IDocument document) throws MetaverseAnalyzerException {
    validateState(document);
    Object repoObject = document.getContent();
    JobMeta jobMeta = null;
    if (repoObject instanceof String) {
        // hydrate the job
        try {
            String content = (String) repoObject;
            ByteArrayInputStream xmlStream = new ByteArrayInputStream(content.getBytes());
            jobMeta = new JobMeta(xmlStream, null, null);
        } catch (KettleXMLException e) {
            throw new MetaverseAnalyzerException(e);
        }
    } else if (repoObject instanceof JobMeta) {
        jobMeta = (JobMeta) repoObject;
    }
    // construct a dummy job based on our JobMeta so we get out VariableSpace set properly
    jobMeta.setFilename(document.getStringID());
    Job j = new Job(null, jobMeta);
    j.setInternalKettleVariables(jobMeta);
    IComponentDescriptor documentDescriptor = new MetaverseComponentDescriptor(document.getStringID(), DictionaryConst.NODE_TYPE_JOB, new Namespace(descriptor.getLogicalId()), descriptor.getContext());
    // Create a metaverse node and start filling in details
    IMetaverseNode node = metaverseObjectFactory.createNodeObject(document.getNamespace(), jobMeta.getName(), DictionaryConst.NODE_TYPE_JOB);
    node.setLogicalIdGenerator(DictionaryConst.LOGICAL_ID_GENERATOR_DOCUMENT);
    // pull out the standard fields
    String description = jobMeta.getDescription();
    if (description != null) {
        node.setProperty(DictionaryConst.PROPERTY_DESCRIPTION, description);
    }
    String extendedDescription = jobMeta.getExtendedDescription();
    if (extendedDescription != null) {
        node.setProperty("extendedDescription", extendedDescription);
    }
    Date createdDate = jobMeta.getCreatedDate();
    if (createdDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED, Long.toString(createdDate.getTime()));
    }
    String createdUser = jobMeta.getCreatedUser();
    if (createdUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_CREATED_BY, createdUser);
    }
    Date lastModifiedDate = jobMeta.getModifiedDate();
    if (lastModifiedDate != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED, Long.toString(lastModifiedDate.getTime()));
    }
    String lastModifiedUser = jobMeta.getModifiedUser();
    if (lastModifiedUser != null) {
        node.setProperty(DictionaryConst.PROPERTY_LAST_MODIFIED_BY, lastModifiedUser);
    }
    String version = jobMeta.getJobversion();
    if (version != null) {
        node.setProperty(DictionaryConst.PROPERTY_ARTIFACT_VERSION, version);
    }
    String status = Messages.getString("INFO.JobOrTrans.Status_" + Integer.toString(jobMeta.getJobstatus()));
    if (status != null && !status.startsWith("!")) {
        node.setProperty(DictionaryConst.PROPERTY_STATUS, status);
    }
    node.setProperty(DictionaryConst.PROPERTY_PATH, document.getProperty(DictionaryConst.PROPERTY_PATH));
    // Process job parameters
    String[] parameters = jobMeta.listParameters();
    if (parameters != null) {
        for (String parameter : parameters) {
            try {
                // Determine parameter properties and add them to a map, then the map to the list
                String defaultParameterValue = jobMeta.getParameterDefault(parameter);
                String parameterValue = jobMeta.getParameterValue(parameter);
                String parameterDescription = jobMeta.getParameterDescription(parameter);
                PropertiesHolder paramProperties = new PropertiesHolder();
                paramProperties.setProperty("defaultValue", defaultParameterValue);
                paramProperties.setProperty("value", parameterValue);
                paramProperties.setProperty("description", parameterDescription);
                node.setProperty("parameter_" + parameter, paramProperties.toString());
            } catch (UnknownParamException upe) {
                // This shouldn't happen as we're using the list provided by the meta
                throw new MetaverseAnalyzerException(upe);
            }
        }
    }
    // handle the entries
    for (int i = 0; i < jobMeta.nrJobEntries(); i++) {
        JobEntryCopy entry = jobMeta.getJobEntry(i);
        try {
            if (entry != null) {
                entry.getEntry().setParentJob(j);
                IMetaverseNode jobEntryNode = null;
                JobEntryInterface jobEntryInterface = entry.getEntry();
                IComponentDescriptor entryDescriptor = new MetaverseComponentDescriptor(entry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY, node, descriptor.getContext());
                Set<IJobEntryAnalyzer> jobEntryAnalyzers = getJobEntryAnalyzers(jobEntryInterface);
                if (jobEntryAnalyzers != null && !jobEntryAnalyzers.isEmpty()) {
                    for (IJobEntryAnalyzer jobEntryAnalyzer : jobEntryAnalyzers) {
                        jobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                        jobEntryNode = (IMetaverseNode) jobEntryAnalyzer.analyze(entryDescriptor, entry.getEntry());
                    }
                } else {
                    GenericJobEntryMetaAnalyzer defaultJobEntryAnalyzer = new GenericJobEntryMetaAnalyzer();
                    defaultJobEntryAnalyzer.setMetaverseBuilder(metaverseBuilder);
                    jobEntryNode = defaultJobEntryAnalyzer.analyze(entryDescriptor, jobEntryInterface);
                }
                if (jobEntryNode != null) {
                    metaverseBuilder.addLink(node, DictionaryConst.LINK_CONTAINS, jobEntryNode);
                }
            }
        } catch (Throwable mae) {
            // Don't throw an exception, just log and carry on
            log.warn(Messages.getString("ERROR.ErrorDuringAnalysis", entry.getName(), Const.NVL(mae.getLocalizedMessage(), "Unspecified")));
            log.debug(Messages.getString("ERROR.ErrorDuringAnalysisStackTrace"), mae);
        }
    }
    // Model the hops between steps
    int numHops = jobMeta.nrJobHops();
    for (int i = 0; i < numHops; i++) {
        JobHopMeta hop = jobMeta.getJobHop(i);
        JobEntryCopy fromEntry = hop.getFromEntry();
        JobEntryCopy toEntry = hop.getToEntry();
        INamespace childNs = new Namespace(node.getLogicalId());
        // process legitimate hops
        if (fromEntry != null && toEntry != null) {
            IMetaverseNode fromEntryNode = metaverseObjectFactory.createNodeObject(childNs, fromEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            IMetaverseNode toEntryNode = metaverseObjectFactory.createNodeObject(childNs, toEntry.getName(), DictionaryConst.NODE_TYPE_JOB_ENTRY);
            metaverseBuilder.addLink(fromEntryNode, DictionaryConst.LINK_HOPSTO, toEntryNode);
        }
    }
    metaverseBuilder.addNode(node);
    addParentLink(documentDescriptor, node);
    return node;
}