Examples with StepIOMeta org.pentaho.di.trans.step.StepIOMeta used on opensource projects

Example 1 with StepIOMeta

use of org.pentaho.di.trans.step.StepIOMeta in project pentaho-metaverse by pentaho.

the class AnnotationDrivenStepMetaAnalyzerTest method getTestStepMeta.

// private Map<String, String> typeCategoryMap = new HashMap<>();
// @Mock AnnotationDrivenStepMetaAnalyzer.EntityRegister entityRegister;
// private RowMeta inputRowMeta;
// private RowMeta outputRowMeta;
// 
// @Mock SubtransAnalyzer subtransAnalyzer;
// 
// private IMetaverseNode root = new MetaverseTransientNode( "rootNode" );
// private IMetaverseNode outputMessageNode = new MetaverseTransientNode( "messageOutputNode" );
// private IMetaverseNode outputTopicNode = new MetaverseTransientNode( "topicOutputNode" );
// private StepMetaInterface meta;
// private StepMetaInterface mappingMeta;
// private TestableAnnotationDrivenAnalyzer analyzer;
// private static IMetaverseNode subTransRoot = new MetaverseTransientNode( "subTransRootNode" );
// 
// 
// @BeforeClass
// public static void setupClass() throws Exception {
// KettleClientEnvironment.init();
// PluginRegistry.addPluginType( ValueMetaPluginType.getInstance() );
// PluginRegistry.init();
// }
// 
// @Before
// public void before() throws KettleException {
// inputRowMeta = new RowMeta();
// outputRowMeta = new RowMeta();
// 
// this.meta = getTestStepMeta( inputRowMeta );
// outputMessageNode.setName( "messageOutputNode" );
// outputMessageNode.setType( NODE_TYPE_TRANS_FIELD );
// outputTopicNode.setName( "topicOutputNode" );
// outputTopicNode.setType( NODE_TYPE_TRANS_FIELD );
// 
// analyzer = new TestableAnnotationDrivenAnalyzer( new TestStepMeta(), typeCategoryMap, entityRegister );
// //doNothing().when( subtransAnalyzer ).linkResultFieldToSubTrans( any(), any(), any(), any() );
// }
// 
// 
// @After
// public void after() {
// // the act of creating a metaverse builder registers entities.  Clear them out so
// // no state is carried forward.
// DictionaryHelper.clearEntityRegistry();
// }
// 
// @Test
// public void testGetUsedFields() {
// Set<StepField> usedFields = analyzer.getUsedFields( (BaseStepMeta) meta );
// assertThat( usedFields.size(), equalTo( 5 ) );
// List<String> usedList = usedFields.stream()
// .map( StepField::getFieldName )
// .sorted()
// .collect( Collectors.toList() );
// List<String> inputSteps = usedFields.stream()
// .map( StepField::getStepName )
// .sorted()
// .collect( Collectors.toList() );
// assertThat( usedList, equalTo( Arrays.asList( "messageField", "substitutedField",
// "usedField", "usedField1", "usedField2" ) ) );
// assertThat( inputSteps, equalTo( Arrays.asList( "Step1", "Step1", "Step1", "Step1", "Step1" ) ) );
// }
// 
// @Test
// public void testCustomAnalyze() throws MetaverseAnalyzerException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// analyzer.loadInputAndOutputStreamFields( (BaseStepMeta) meta );
// analyzer.customAnalyze( (BaseStepMeta) meta, root );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_READBY );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// Edge e = edges.get( 0 );
// 
// Vertex externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "server" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "port" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "isSsl" ), equalTo( "true" ) );
// assertThat( externalResourceV.getProperty( "name" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "type" ), equalTo( TEST_TYPE ) );
// 
// // nested properties in referenced class
// assertThat( externalResourceV.getProperty( "SUBPROP" ), equalTo( "subProp1" ) );
// assertThat( externalResourceV.getProperty( "subProp2" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "method-property" ), equalTo( "blah" ) );
// 
// edges = getEdgesWithLabel( graph, LINK_DEPENDENCYOF );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// e = edges.get( 0 );
// 
// externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "dbaddress" ), equalTo( "127.0.0.1:3363" ) );
// }
// 
// @Test
// public void testCustomAnalyzeSubtrans() throws MetaverseAnalyzerException, KettleException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// this.mappingMeta = getTestStepMappingMeta( inputRowMeta, outputRowMeta );
// 
// analyzer.loadInputAndOutputStreamFields( (BaseStepMeta) mappingMeta );
// analyzer.customAnalyze( (BaseStepMeta) mappingMeta, root );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_READBY );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// Edge e = edges.get( 0 );
// 
// Vertex externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "server" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "port" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "isSsl" ), equalTo( "true" ) );
// assertThat( externalResourceV.getProperty( "name" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "type" ), equalTo( TEST_TYPE ) );
// 
// //verify server node "contains" the two input resources
// List<Vertex> resourceNodes = new ArrayList<>();
// externalResourceV.getVertices( Direction.OUT ).forEach( v -> resourceNodes.add( v ) );
// assertThat( resourceNodes.size(), equalTo( 3 ) );
// Vertex messageVertex = resourceNodes.stream().filter( v -> v.getProperty( PROPERTY_NAME ).equals( "message" ) ).findFirst().get();
// Vertex topicVertex = resourceNodes.stream().filter( v -> v.getProperty( PROPERTY_NAME ).equals( "topic" ) ).findFirst().get();
// 
// assertThat( messageVertex.getProperty( PROPERTY_TYPE ), equalTo( RESOURCE ) );
// assertThat( topicVertex.getProperty( PROPERTY_TYPE ), equalTo( RESOURCE ) );
// 
// // nested properties in referenced class
// assertThat( externalResourceV.getProperty( "SUBPROP" ), equalTo( "subProp1" ) );
// assertThat( externalResourceV.getProperty( "subProp2" ), equalTo( "123" ) );
// 
// edges = getEdgesWithLabel( graph, LINK_DEPENDENCYOF );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// e = edges.get( 0 );
// 
// externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "dbaddress" ), equalTo( "127.0.0.1:3363" ) );
// 
// doNothing().when( subtransAnalyzer ).linkUsedFieldToSubTrans( any(), any(), any(), any(), any() );
// doNothing().when( subtransAnalyzer ).linkResultFieldToSubTrans(
// any( IMetaverseNode.class ), any( TransMeta.class ), any( IMetaverseNode.class ),
// any( IComponentDescriptor.class ), any( String.class ) );
// 
// // verify subtrans analyzer was called on the expected nodes for input to subtrans
// ArgumentCaptor<IMetaverseNode> usedFieldsNodeCaptor = ArgumentCaptor.forClass( IMetaverseNode.class );
// ArgumentCaptor<IMetaverseNode> outputFieldsNodeCaptor = ArgumentCaptor.forClass( IMetaverseNode.class );
// 
// verify( subtransAnalyzer, times( 2 ) )
// .linkResultFieldToSubTrans( outputFieldsNodeCaptor.capture(), any(), any(), any(), any() );
// verify( subtransAnalyzer, times( 2 ) )
// .linkUsedFieldToSubTrans( usedFieldsNodeCaptor.capture(), any(), any(), any(), any() );
// 
// IMetaverseNode messageNode = usedFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "message" ) ).findFirst().get();
// IMetaverseNode topicNode = usedFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "topic" ) ).findFirst().get();
// 
// assertThat( usedFieldsNodeCaptor.getAllValues().size(), equalTo( 2 ) );
// assertThat( topicNode.getType(), equalTo( RESOURCE ) );
// assertThat( messageNode.getType(), equalTo( RESOURCE ) );
// 
// // verify subtrans analyzer was called on the expected nodes for output from subtrans
// IMetaverseNode messageOutputNode = outputFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "messageOutputNode" ) ).findFirst().get();
// IMetaverseNode topicOutputNode = outputFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "topicOutputNode" ) ).findFirst().get();
// 
// assertThat( outputFieldsNodeCaptor.getAllValues().size(), equalTo( 2 ) );
// assertThat( topicOutputNode.getType(), equalTo( NODE_TYPE_TRANS_FIELD ) );
// assertThat( messageOutputNode.getType(), equalTo( NODE_TYPE_TRANS_FIELD ) );
// }
// 
// @Test
// public void testResourceLinkage() throws MetaverseAnalyzerException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// MetaverseComponentDescriptor descriptor = new MetaverseComponentDescriptor(
// "name", "type",
// root, analyzer.getDescriptor().getContext() );
// 
// analyzer.analyze( descriptor, (BaseStepMeta) meta );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_WRITESTO );
// 
// assertThat( edges.size(), equalTo( 2 ) );
// edges.sort( Comparator.comparing( e -> e.getVertex( Direction.IN ).getProperty( "name" ) ) );
// assertEdge( edges.get( 0 ), "Bora Bora", "name" );
// assertEdge( edges.get( 1 ), "message", "ServernameOrWhatever" );
// 
// edges = getEdgesWithLabel( graph, LINK_CONTAINS );
// assertThat( edges.size(), equalTo( 1 ) );
// assertEdge( edges.get( 0 ), "name", "private-field-value" );
// 
// }
// 
// private void assertEdge( Edge edge, String from, String to ) {
// assertThat( edge.getVertex( Direction.IN ).getProperty( "name" ), equalTo( from ) );
// assertThat( edge.getVertex( Direction.OUT ).getProperty( "name" ), equalTo( to ) );
// }
// 
// @Test
// public void testCategoryLinks() {
// assertThat( typeCategoryMap.get( CONN_TYPE ),
// equalTo( CATEGORY_DATASOURCE ) );
// assertThat( typeCategoryMap.get( TEST_TYPE ),
// equalTo( CATEGORY_OTHER ) );
// }
// 
// @Test
// public void testEntityLinks() {
// 
// verify( entityRegister ).registerEntityTypes( LINK_PARENT_CONCEPT, CONN_TYPE, NODE_TYPE_EXTERNAL_CONNECTION );
// verify( entityRegister ).registerEntityTypes( LINK_CONTAINS_CONCEPT, TEST_TYPE, CONN_TYPE );
// verify( entityRegister ).registerEntityTypes( LINK_PARENT_CONCEPT, TEST_TYPE, null );
// }
// 
// @Test
// public void testGetSupportedSteps() {
// assertThat( analyzer.getSupportedSteps(), equalTo( singleton( meta.getClass() ) ) );
// }
// 
// 
// private IMetaverseBuilder getBuilder( IMetaverseNode rootNode ) {
// IComponentDescriptor descriptor = new MetaverseComponentDescriptor( "descriptor", "someType", rootNode );
// analyzer.setDescriptor( descriptor );
// IMetaverseBuilder builder = new BaseMetaverseBuilder( new TinkerGraph() );
// analyzer.setMetaverseBuilder( builder );
// return builder;
// }
// 
// private List<Edge> getEdgesWithLabel( Graph graph, String edgeName ) {
// return StreamSupport.stream( graph.getEdges().spliterator(), false )
// .filter( edge -> edge.getLabel().equals( edgeName ) )
// .collect( Collectors.toList() );
// }
// 
// /**
// * Used to stub out some values that would normally be set during analyze() of a full transformation.
// */
// class TestableAnnotationDrivenAnalyzer extends AnnotationDrivenStepMetaAnalyzer {
// 
// TestableAnnotationDrivenAnalyzer( BaseStepMeta meta, Map<String, String> typeCategoryMap,
// EntityRegister register ) {
// super( meta, typeCategoryMap, register, new Variables() );
// }
// 
// {
// setMetaverseObjectFactory( new MetaverseObjectFactory() );
// rootNode = root;
// }
// 
// protected SubtransAnalyzer<BaseStepMeta> getSubtransAnalyzer() {
// return subtransAnalyzer;
// }
// 
// public Map<String, RowMetaInterface> getInputFields( final TransMeta parentTransMeta,
// final StepMeta parentStepMeta ) {
// 
// return ImmutableMap.of( "Step1", inputRowMeta,
// "Step2", new RowMeta() );
// }
// 
// public StepNodes getOutputs() {
// StepNodes outputs = new StepNodes();
// outputs.addNode( "SomeNextStep", "messageFieldOut", outputMessageNode );
// outputs.addNode( "SomeNextStep", "topicFieldOut", outputTopicNode );
// return outputs;
// }
// }
// 
static TestStepMeta getTestStepMeta(RowMeta inputRowMeta) {
    // step meta with metaverse annotations
    TestStepMeta meta = new TestStepMeta();
    inputRowMeta.addValueMeta(new ValueMetaString("usedField"));
    inputRowMeta.addValueMeta(new ValueMetaString("usedField1"));
    inputRowMeta.addValueMeta(new ValueMetaString("usedField2"));
    inputRowMeta.addValueMeta(new ValueMetaString("messageField"));
    inputRowMeta.addValueMeta(new ValueMetaString("substitutedField"));
    StepMeta parentStepMeta = new StepMeta();
    final Variables variables = new Variables();
    variables.setVariable("substitute", "substitutedField");
    TransMeta transMeta = spy(new TransMeta(variables));
    // stub out previous step input
    try {
        when(transMeta.getPrevStepFields(any(), any(), any())).thenReturn(inputRowMeta);
    } catch (KettleStepException e) {
        throw new IllegalStateException(e);
    }
    meta.setParentStepMeta(parentStepMeta);
    parentStepMeta.setStepMetaInterface(meta);
    StepIOMeta stepIOMeta = new StepIOMeta(true, true, true, false, false, false);
    meta.setStepIOMeta(stepIOMeta);
    transMeta.addStep(parentStepMeta);
    return meta;
}

Example 2 with StepIOMeta

use of org.pentaho.di.trans.step.StepIOMeta in project pentaho-kettle by pentaho.

the class FilterRowsMeta method getStepIOMeta.

/**
 * Returns the Input/Output metadata for this step.
 */
public StepIOMetaInterface getStepIOMeta() {
    StepIOMetaInterface ioMeta = super.getStepIOMeta(false);
    if (ioMeta == null) {
        ioMeta = new StepIOMeta(true, true, false, false, false, false);
        ioMeta.addStream(new Stream(StreamType.TARGET, null, BaseMessages.getString(PKG, "FilterRowsMeta.InfoStream.True.Description"), StreamIcon.TRUE, null));
        ioMeta.addStream(new Stream(StreamType.TARGET, null, BaseMessages.getString(PKG, "FilterRowsMeta.InfoStream.False.Description"), StreamIcon.FALSE, null));
        setStepIOMeta(ioMeta);
    }
    return ioMeta;
}

Example 3 with StepIOMeta

use of org.pentaho.di.trans.step.StepIOMeta in project pentaho-kettle by pentaho.

the class TransExecutorMeta method getStepIOMeta.

@Override
public StepIOMetaInterface getStepIOMeta() {
    StepIOMetaInterface ioMeta = super.getStepIOMeta(false);
    if (ioMeta == null) {
        ioMeta = new StepIOMeta(true, true, true, false, true, false);
        ioMeta.addStream(new Stream(StreamType.TARGET, executionResultTargetStepMeta, BaseMessages.getString(PKG, "TransExecutorMeta.ResultStream.Description"), StreamIcon.TARGET, null));
        ioMeta.addStream(new Stream(StreamType.TARGET, outputRowsSourceStepMeta, BaseMessages.getString(PKG, "TransExecutorMeta.ResultRowsStream.Description"), StreamIcon.TARGET, null));
        ioMeta.addStream(new Stream(StreamType.TARGET, resultFilesTargetStepMeta, BaseMessages.getString(PKG, "TransExecutorMeta.ResultFilesStream.Description"), StreamIcon.TARGET, null));
        ioMeta.addStream(new Stream(StreamType.TARGET, executorsOutputStepMeta, BaseMessages.getString(PKG, "TransExecutorMeta.ExecutorOutputStream.Description"), StreamIcon.OUTPUT, null));
        setStepIOMeta(ioMeta);
    }
    return ioMeta;
}

Example 4 with StepIOMeta

use of org.pentaho.di.trans.step.StepIOMeta in project pentaho-kettle by pentaho.

the class TableInputMeta method getStepIOMeta.

/**
 * Returns the Input/Output metadata for this step. The generator step only produces output, does not accept input!
 */
public StepIOMetaInterface getStepIOMeta() {
    StepIOMetaInterface ioMeta = super.getStepIOMeta(false);
    if (ioMeta == null) {
        ioMeta = new StepIOMeta(true, true, false, false, false, false);
        StreamInterface stream = new Stream(StreamType.INFO, null, BaseMessages.getString(PKG, "TableInputMeta.InfoStream.Description"), StreamIcon.INFO, null);
        ioMeta.addStream(stream);
        setStepIOMeta(ioMeta);
    }
    return ioMeta;
}

Example 5 with StepIOMeta

use of org.pentaho.di.trans.step.StepIOMeta in project pentaho-kettle by pentaho.

the class AppendMeta method getStepIOMeta.

/**
 * Returns the Input/Output metadata for this step.
 */
public StepIOMetaInterface getStepIOMeta() {
    StepIOMetaInterface ioMeta = super.getStepIOMeta(false);
    if (ioMeta == null) {
        ioMeta = new StepIOMeta(true, true, false, false, false, false);
        ioMeta.addStream(new Stream(StreamType.INFO, null, BaseMessages.getString(PKG, "AppendMeta.InfoStream.FirstStream.Description"), StreamIcon.INFO, null));
        ioMeta.addStream(new Stream(StreamType.INFO, null, BaseMessages.getString(PKG, "AppendMeta.InfoStream.SecondStream.Description"), StreamIcon.INFO, null));
        setStepIOMeta(ioMeta);
    }
    return ioMeta;
}