Examples with BaseStepMeta org.pentaho.di.trans.step.BaseStepMeta used on opensource projects

Example 1 with BaseStepMeta

use of org.pentaho.di.trans.step.BaseStepMeta in project pentaho-metaverse by pentaho.

the class AnnotationDrivenStepMetaAnalyzerTest method getTestStepMeta.

// private Map<String, String> typeCategoryMap = new HashMap<>();
// @Mock AnnotationDrivenStepMetaAnalyzer.EntityRegister entityRegister;
// private RowMeta inputRowMeta;
// private RowMeta outputRowMeta;
// 
// @Mock SubtransAnalyzer subtransAnalyzer;
// 
// private IMetaverseNode root = new MetaverseTransientNode( "rootNode" );
// private IMetaverseNode outputMessageNode = new MetaverseTransientNode( "messageOutputNode" );
// private IMetaverseNode outputTopicNode = new MetaverseTransientNode( "topicOutputNode" );
// private StepMetaInterface meta;
// private StepMetaInterface mappingMeta;
// private TestableAnnotationDrivenAnalyzer analyzer;
// private static IMetaverseNode subTransRoot = new MetaverseTransientNode( "subTransRootNode" );
// 
// 
// @BeforeClass
// public static void setupClass() throws Exception {
// KettleClientEnvironment.init();
// PluginRegistry.addPluginType( ValueMetaPluginType.getInstance() );
// PluginRegistry.init();
// }
// 
// @Before
// public void before() throws KettleException {
// inputRowMeta = new RowMeta();
// outputRowMeta = new RowMeta();
// 
// this.meta = getTestStepMeta( inputRowMeta );
// outputMessageNode.setName( "messageOutputNode" );
// outputMessageNode.setType( NODE_TYPE_TRANS_FIELD );
// outputTopicNode.setName( "topicOutputNode" );
// outputTopicNode.setType( NODE_TYPE_TRANS_FIELD );
// 
// analyzer = new TestableAnnotationDrivenAnalyzer( new TestStepMeta(), typeCategoryMap, entityRegister );
// //doNothing().when( subtransAnalyzer ).linkResultFieldToSubTrans( any(), any(), any(), any() );
// }
// 
// 
// @After
// public void after() {
// // the act of creating a metaverse builder registers entities.  Clear them out so
// // no state is carried forward.
// DictionaryHelper.clearEntityRegistry();
// }
// 
// @Test
// public void testGetUsedFields() {
// Set<StepField> usedFields = analyzer.getUsedFields( (BaseStepMeta) meta );
// assertThat( usedFields.size(), equalTo( 5 ) );
// List<String> usedList = usedFields.stream()
// .map( StepField::getFieldName )
// .sorted()
// .collect( Collectors.toList() );
// List<String> inputSteps = usedFields.stream()
// .map( StepField::getStepName )
// .sorted()
// .collect( Collectors.toList() );
// assertThat( usedList, equalTo( Arrays.asList( "messageField", "substitutedField",
// "usedField", "usedField1", "usedField2" ) ) );
// assertThat( inputSteps, equalTo( Arrays.asList( "Step1", "Step1", "Step1", "Step1", "Step1" ) ) );
// }
// 
// @Test
// public void testCustomAnalyze() throws MetaverseAnalyzerException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// analyzer.loadInputAndOutputStreamFields( (BaseStepMeta) meta );
// analyzer.customAnalyze( (BaseStepMeta) meta, root );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_READBY );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// Edge e = edges.get( 0 );
// 
// Vertex externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "server" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "port" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "isSsl" ), equalTo( "true" ) );
// assertThat( externalResourceV.getProperty( "name" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "type" ), equalTo( TEST_TYPE ) );
// 
// // nested properties in referenced class
// assertThat( externalResourceV.getProperty( "SUBPROP" ), equalTo( "subProp1" ) );
// assertThat( externalResourceV.getProperty( "subProp2" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "method-property" ), equalTo( "blah" ) );
// 
// edges = getEdgesWithLabel( graph, LINK_DEPENDENCYOF );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// e = edges.get( 0 );
// 
// externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "dbaddress" ), equalTo( "127.0.0.1:3363" ) );
// }
// 
// @Test
// public void testCustomAnalyzeSubtrans() throws MetaverseAnalyzerException, KettleException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// this.mappingMeta = getTestStepMappingMeta( inputRowMeta, outputRowMeta );
// 
// analyzer.loadInputAndOutputStreamFields( (BaseStepMeta) mappingMeta );
// analyzer.customAnalyze( (BaseStepMeta) mappingMeta, root );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_READBY );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// Edge e = edges.get( 0 );
// 
// Vertex externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "server" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "port" ), equalTo( "123" ) );
// assertThat( externalResourceV.getProperty( "isSsl" ), equalTo( "true" ) );
// assertThat( externalResourceV.getProperty( "name" ), equalTo( "ServernameOrWhatever" ) );
// assertThat( externalResourceV.getProperty( "type" ), equalTo( TEST_TYPE ) );
// 
// //verify server node "contains" the two input resources
// List<Vertex> resourceNodes = new ArrayList<>();
// externalResourceV.getVertices( Direction.OUT ).forEach( v -> resourceNodes.add( v ) );
// assertThat( resourceNodes.size(), equalTo( 3 ) );
// Vertex messageVertex = resourceNodes.stream().filter( v -> v.getProperty( PROPERTY_NAME ).equals( "message" ) ).findFirst().get();
// Vertex topicVertex = resourceNodes.stream().filter( v -> v.getProperty( PROPERTY_NAME ).equals( "topic" ) ).findFirst().get();
// 
// assertThat( messageVertex.getProperty( PROPERTY_TYPE ), equalTo( RESOURCE ) );
// assertThat( topicVertex.getProperty( PROPERTY_TYPE ), equalTo( RESOURCE ) );
// 
// // nested properties in referenced class
// assertThat( externalResourceV.getProperty( "SUBPROP" ), equalTo( "subProp1" ) );
// assertThat( externalResourceV.getProperty( "subProp2" ), equalTo( "123" ) );
// 
// edges = getEdgesWithLabel( graph, LINK_DEPENDENCYOF );
// 
// assertThat( edges.size(), equalTo( 1 ) );
// e = edges.get( 0 );
// 
// externalResourceV = e.getVertex( Direction.OUT );
// assertThat( externalResourceV.getProperty( "dbaddress" ), equalTo( "127.0.0.1:3363" ) );
// 
// doNothing().when( subtransAnalyzer ).linkUsedFieldToSubTrans( any(), any(), any(), any(), any() );
// doNothing().when( subtransAnalyzer ).linkResultFieldToSubTrans(
// any( IMetaverseNode.class ), any( TransMeta.class ), any( IMetaverseNode.class ),
// any( IComponentDescriptor.class ), any( String.class ) );
// 
// // verify subtrans analyzer was called on the expected nodes for input to subtrans
// ArgumentCaptor<IMetaverseNode> usedFieldsNodeCaptor = ArgumentCaptor.forClass( IMetaverseNode.class );
// ArgumentCaptor<IMetaverseNode> outputFieldsNodeCaptor = ArgumentCaptor.forClass( IMetaverseNode.class );
// 
// verify( subtransAnalyzer, times( 2 ) )
// .linkResultFieldToSubTrans( outputFieldsNodeCaptor.capture(), any(), any(), any(), any() );
// verify( subtransAnalyzer, times( 2 ) )
// .linkUsedFieldToSubTrans( usedFieldsNodeCaptor.capture(), any(), any(), any(), any() );
// 
// IMetaverseNode messageNode = usedFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "message" ) ).findFirst().get();
// IMetaverseNode topicNode = usedFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "topic" ) ).findFirst().get();
// 
// assertThat( usedFieldsNodeCaptor.getAllValues().size(), equalTo( 2 ) );
// assertThat( topicNode.getType(), equalTo( RESOURCE ) );
// assertThat( messageNode.getType(), equalTo( RESOURCE ) );
// 
// // verify subtrans analyzer was called on the expected nodes for output from subtrans
// IMetaverseNode messageOutputNode = outputFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "messageOutputNode" ) ).findFirst().get();
// IMetaverseNode topicOutputNode = outputFieldsNodeCaptor.getAllValues().stream()
// .filter( node -> node.getName().equals( "topicOutputNode" ) ).findFirst().get();
// 
// assertThat( outputFieldsNodeCaptor.getAllValues().size(), equalTo( 2 ) );
// assertThat( topicOutputNode.getType(), equalTo( NODE_TYPE_TRANS_FIELD ) );
// assertThat( messageOutputNode.getType(), equalTo( NODE_TYPE_TRANS_FIELD ) );
// }
// 
// @Test
// public void testResourceLinkage() throws MetaverseAnalyzerException {
// IMetaverseBuilder builder = getBuilder( root );
// 
// MetaverseComponentDescriptor descriptor = new MetaverseComponentDescriptor(
// "name", "type",
// root, analyzer.getDescriptor().getContext() );
// 
// analyzer.analyze( descriptor, (BaseStepMeta) meta );
// Graph graph = builder.getGraph();
// 
// List<Edge> edges = getEdgesWithLabel( graph, LINK_WRITESTO );
// 
// assertThat( edges.size(), equalTo( 2 ) );
// edges.sort( Comparator.comparing( e -> e.getVertex( Direction.IN ).getProperty( "name" ) ) );
// assertEdge( edges.get( 0 ), "Bora Bora", "name" );
// assertEdge( edges.get( 1 ), "message", "ServernameOrWhatever" );
// 
// edges = getEdgesWithLabel( graph, LINK_CONTAINS );
// assertThat( edges.size(), equalTo( 1 ) );
// assertEdge( edges.get( 0 ), "name", "private-field-value" );
// 
// }
// 
// private void assertEdge( Edge edge, String from, String to ) {
// assertThat( edge.getVertex( Direction.IN ).getProperty( "name" ), equalTo( from ) );
// assertThat( edge.getVertex( Direction.OUT ).getProperty( "name" ), equalTo( to ) );
// }
// 
// @Test
// public void testCategoryLinks() {
// assertThat( typeCategoryMap.get( CONN_TYPE ),
// equalTo( CATEGORY_DATASOURCE ) );
// assertThat( typeCategoryMap.get( TEST_TYPE ),
// equalTo( CATEGORY_OTHER ) );
// }
// 
// @Test
// public void testEntityLinks() {
// 
// verify( entityRegister ).registerEntityTypes( LINK_PARENT_CONCEPT, CONN_TYPE, NODE_TYPE_EXTERNAL_CONNECTION );
// verify( entityRegister ).registerEntityTypes( LINK_CONTAINS_CONCEPT, TEST_TYPE, CONN_TYPE );
// verify( entityRegister ).registerEntityTypes( LINK_PARENT_CONCEPT, TEST_TYPE, null );
// }
// 
// @Test
// public void testGetSupportedSteps() {
// assertThat( analyzer.getSupportedSteps(), equalTo( singleton( meta.getClass() ) ) );
// }
// 
// 
// private IMetaverseBuilder getBuilder( IMetaverseNode rootNode ) {
// IComponentDescriptor descriptor = new MetaverseComponentDescriptor( "descriptor", "someType", rootNode );
// analyzer.setDescriptor( descriptor );
// IMetaverseBuilder builder = new BaseMetaverseBuilder( new TinkerGraph() );
// analyzer.setMetaverseBuilder( builder );
// return builder;
// }
// 
// private List<Edge> getEdgesWithLabel( Graph graph, String edgeName ) {
// return StreamSupport.stream( graph.getEdges().spliterator(), false )
// .filter( edge -> edge.getLabel().equals( edgeName ) )
// .collect( Collectors.toList() );
// }
// 
// /**
// * Used to stub out some values that would normally be set during analyze() of a full transformation.
// */
// class TestableAnnotationDrivenAnalyzer extends AnnotationDrivenStepMetaAnalyzer {
// 
// TestableAnnotationDrivenAnalyzer( BaseStepMeta meta, Map<String, String> typeCategoryMap,
// EntityRegister register ) {
// super( meta, typeCategoryMap, register, new Variables() );
// }
// 
// {
// setMetaverseObjectFactory( new MetaverseObjectFactory() );
// rootNode = root;
// }
// 
// protected SubtransAnalyzer<BaseStepMeta> getSubtransAnalyzer() {
// return subtransAnalyzer;
// }
// 
// public Map<String, RowMetaInterface> getInputFields( final TransMeta parentTransMeta,
// final StepMeta parentStepMeta ) {
// 
// return ImmutableMap.of( "Step1", inputRowMeta,
// "Step2", new RowMeta() );
// }
// 
// public StepNodes getOutputs() {
// StepNodes outputs = new StepNodes();
// outputs.addNode( "SomeNextStep", "messageFieldOut", outputMessageNode );
// outputs.addNode( "SomeNextStep", "topicFieldOut", outputTopicNode );
// return outputs;
// }
// }
// 
static TestStepMeta getTestStepMeta(RowMeta inputRowMeta) {
    // step meta with metaverse annotations
    TestStepMeta meta = new TestStepMeta();
    inputRowMeta.addValueMeta(new ValueMetaString("usedField"));
    inputRowMeta.addValueMeta(new ValueMetaString("usedField1"));
    inputRowMeta.addValueMeta(new ValueMetaString("usedField2"));
    inputRowMeta.addValueMeta(new ValueMetaString("messageField"));
    inputRowMeta.addValueMeta(new ValueMetaString("substitutedField"));
    StepMeta parentStepMeta = new StepMeta();
    final Variables variables = new Variables();
    variables.setVariable("substitute", "substitutedField");
    TransMeta transMeta = spy(new TransMeta(variables));
    // stub out previous step input
    try {
        when(transMeta.getPrevStepFields(any(), any(), any())).thenReturn(inputRowMeta);
    } catch (KettleStepException e) {
        throw new IllegalStateException(e);
    }
    meta.setParentStepMeta(parentStepMeta);
    parentStepMeta.setStepMetaInterface(meta);
    StepIOMeta stepIOMeta = new StepIOMeta(true, true, true, false, false, false);
    meta.setStepIOMeta(stepIOMeta);
    transMeta.addStep(parentStepMeta);
    return meta;
}

Example 2 with BaseStepMeta

use of org.pentaho.di.trans.step.BaseStepMeta in project pentaho-metaverse by pentaho.

the class BaseStepExternalResourceConsumerTest method testGetResourcesFromMeta.

@Test
public void testGetResourcesFromMeta() throws Exception {
    Collection<IExternalResourceConsumer> resources = null;
    BaseStepMeta meta = null;
    // null meta
    resources = consumer.getResourcesFromMeta(meta);
    assertNotNull(resources);
    assertTrue(resources.isEmpty());
    // non-file meta
    meta = Mockito.mock(BaseStepMeta.class);
    resources = consumer.getResourcesFromMeta(meta);
    assertNotNull(resources);
    assertTrue(resources.isEmpty());
    Mockito.verify(meta, Mockito.times(0)).getParentStepMeta();
    // file meta
    meta = Mockito.mock(BaseFileMeta.class);
    TransMeta transMeta = Mockito.mock(TransMeta.class);
    StepMeta stepMeta = Mockito.mock(StepMeta.class);
    Mockito.when(((BaseFileMeta) meta).writesToFile()).thenReturn(true);
    Mockito.when(meta.getParentStepMeta()).thenReturn(stepMeta);
    Mockito.when(stepMeta.getParentTransMeta()).thenReturn(transMeta);
    consumer.getResourcesFromMeta(meta);
    Mockito.verify(meta, Mockito.times(2)).getParentStepMeta();
    Mockito.verify((BaseFileMeta) meta, Mockito.times(1)).getFilePaths(false);
}

Example 3 with BaseStepMeta

use of org.pentaho.di.trans.step.BaseStepMeta in project pentaho-metaverse by pentaho.

the class ConnectionExternalResourceStepAnalyzerTest method setUp.

@Before
public void setUp() throws Exception {
    analyzer = spy(new ConnectionExternalResourceStepAnalyzer<BaseStepMeta>() {

        @Override
        protected IMetaverseNode createTableNode(IExternalResourceInfo resource) throws MetaverseAnalyzerException {
            return tableNode;
        }

        @Override
        public String getResourceInputNodeType() {
            return "INPUT_TYPE";
        }

        @Override
        public String getResourceOutputNodeType() {
            return "OUTPUT_TYPE";
        }

        @Override
        public boolean isOutput() {
            return false;
        }

        @Override
        public boolean isInput() {
            return true;
        }

        @Override
        protected Set<StepField> getUsedFields(BaseStepMeta meta) {
            Set<StepField> stepFields = new HashSet<>();
            stepFields.add(new StepField("prevStep", "filename"));
            return stepFields;
        }

        @Override
        public Set<Class<? extends BaseStepMeta>> getSupportedSteps() {
            return null;
        }
    });
    when(analyzer.getMetaverseBuilder()).thenReturn(builder);
    analyzer.descriptor = descriptor;
    when(descriptor.getContext()).thenReturn(context);
    analyzer.parentTransMeta = parentTransMeta;
    analyzer.parentStepMeta = parentStepMeta;
    analyzer.setMetaverseObjectFactory(new MetaverseObjectFactory());
}

Example 4 with BaseStepMeta

use of org.pentaho.di.trans.step.BaseStepMeta in project pentaho-metaverse by pentaho.

the class StepAnalyzerTest method testGetInputFieldsWithException.

@Test
public void testGetInputFieldsWithException() {
    analyzer = new StepAnalyzer() {

        @Override
        public void validateState(IComponentDescriptor descriptor, BaseStepMeta object) throws MetaverseAnalyzerException {
            throw new MetaverseAnalyzerException("expected exception");
        }

        @Override
        public Set<Class<? extends BaseStepMeta>> getSupportedSteps() {
            return null;
        }

        @Override
        protected Set getUsedFields(BaseStepMeta meta) {
            // TODO Auto-generated method stub
            return null;
        }

        @Override
        protected void customAnalyze(BaseStepMeta meta, IMetaverseNode rootNode) throws MetaverseAnalyzerException {
        // TODO Auto-generated method stub
        }
    };
    assertNull(analyzer.getInputFields(null));
}

Example 5 with BaseStepMeta

use of org.pentaho.di.trans.step.BaseStepMeta in project pentaho-metaverse by pentaho.

the class StepAnalyzerTest method setUp.

@Before
public void setUp() throws Exception {
    StepAnalyzer stepAnalyzer = new StepAnalyzer() {

        @Override
        protected Set<StepField> getUsedFields(BaseStepMeta meta) {
            return null;
        }

        @Override
        protected void customAnalyze(BaseStepMeta meta, IMetaverseNode rootNode) throws MetaverseAnalyzerException {
        }

        @Override
        public Set<Class<? extends BaseStepMeta>> getSupportedSteps() {
            return null;
        }
    };
    analyzer = spy(stepAnalyzer);
    analyzer.rootNode = rootNode;
    analyzer.baseStepMeta = baseStepMeta;
    analyzer.descriptor = descriptor;
    analyzer.parentTransMeta = parentTransMeta;
    analyzer.parentStepMeta = parentStepMeta;
    usedFields = new HashSet<>();
    inputs = new StepNodes();
    inputs.addNode("prevStep", "name", fieldNode);
    inputs.addNode("prevStep", "address", fieldNode);
    inputs.addNode("prevStep", "email", fieldNode);
    inputs.addNode("prevStep", "birthday", fieldNode);
    inputs.addNode("prevStep2", "employeeNumber", fieldNode);
    inputs.addNode("prevStep2", "occupation", fieldNode);
    outputs = new StepNodes();
    outputs.addNode("nextStep", "full name", fieldNode);
    outputs.addNode("nextStep", "address", fieldNode);
    outputs.addNode("nextStep", "email", fieldNode);
    outputs.addNode("nextStep", "date of birth", fieldNode);
    outputs.addNode("nextStep", "ID", fieldNode);
    outputs.addNode("nextStep", "occupation", fieldNode);
    doReturn(builder).when(analyzer).getMetaverseBuilder();
    MetaverseObjectFactory factory = new MetaverseObjectFactory();
    doReturn(factory).when(analyzer).getMetaverseObjectFactory();
    when(descriptor.getContext()).thenReturn(context);
    when(parentStepMeta.getName()).thenReturn("STEP NAME");
}