Examples with Variables org.pentaho.di.core.variables.Variables used on opensource projects

Example 26 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class JsonInputTest method testJsonInputMetaInputFieldsNotOverwritten.

@Test
public void testJsonInputMetaInputFieldsNotOverwritten() throws Exception {
    JsonInputField inputField = new JsonInputField();
    final String PATH = "$..book[?(@.category=='${category}')].price";
    inputField.setPath(PATH);
    inputField.setType(ValueMetaInterface.TYPE_STRING);
    final JsonInputMeta inputMeta = createSimpleMeta("json", inputField);
    VariableSpace variables = new Variables();
    variables.setVariable("category", "fiction");
    JsonInput jsonInput = createJsonInput("json", inputMeta, variables, new Object[] { getBasicTestJson() });
    processRows(jsonInput, 2);
    assertEquals("Meta input fields paths should be the same after processRows", PATH, inputMeta.getInputFields()[0].getPath());
}

Example 27 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class MemoryGroupByAggregationTest method setUp.

@Before
public void setUp() throws Exception {
    rowMeta = new RowMeta();
    data = TreeBasedTable.create();
    variables = new Variables();
    aggregates = Maps.newHashMap(default_aggregates);
}

Example 28 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class MemoryGroupByMetaTest method testGetFields.

@Test
public void testGetFields() {
    final String stepName = "this step name";
    MemoryGroupByMeta meta = new MemoryGroupByMeta();
    meta.setDefault();
    meta.allocate(1, 17);
    // Declare input fields
    RowMetaInterface rm = getInputRowMeta();
    String[] groupFields = new String[2];
    groupFields[0] = "myGroupField1";
    groupFields[1] = "myGroupField2";
    String[] aggregateFields = new String[24];
    String[] subjectFields = new String[24];
    int[] aggregateTypes = new int[24];
    String[] valueFields = new String[24];
    subjectFields[0] = "myString";
    aggregateTypes[0] = MemoryGroupByMeta.TYPE_GROUP_CONCAT_COMMA;
    aggregateFields[0] = "ConcatComma";
    valueFields[0] = null;
    subjectFields[1] = "myString";
    aggregateTypes[1] = MemoryGroupByMeta.TYPE_GROUP_CONCAT_STRING;
    aggregateFields[1] = "ConcatString";
    valueFields[1] = "|";
    subjectFields[2] = "myString";
    aggregateTypes[2] = MemoryGroupByMeta.TYPE_GROUP_COUNT_ALL;
    aggregateFields[2] = "CountAll";
    valueFields[2] = null;
    subjectFields[3] = "myString";
    aggregateTypes[3] = MemoryGroupByMeta.TYPE_GROUP_COUNT_ANY;
    aggregateFields[3] = "CountAny";
    valueFields[3] = null;
    subjectFields[4] = "myString";
    aggregateTypes[4] = MemoryGroupByMeta.TYPE_GROUP_COUNT_DISTINCT;
    aggregateFields[4] = "CountDistinct";
    valueFields[4] = null;
    subjectFields[5] = "myString";
    aggregateTypes[5] = MemoryGroupByMeta.TYPE_GROUP_FIRST;
    aggregateFields[5] = "First(String)";
    valueFields[5] = null;
    subjectFields[6] = "myInteger";
    aggregateTypes[6] = MemoryGroupByMeta.TYPE_GROUP_FIRST;
    aggregateFields[6] = "First(Integer)";
    valueFields[6] = null;
    subjectFields[7] = "myNumber";
    aggregateTypes[7] = MemoryGroupByMeta.TYPE_GROUP_FIRST_INCL_NULL;
    aggregateFields[7] = "FirstInclNull(Number)";
    valueFields[7] = null;
    subjectFields[8] = "myBigNumber";
    aggregateTypes[8] = MemoryGroupByMeta.TYPE_GROUP_FIRST_INCL_NULL;
    aggregateFields[8] = "FirstInclNull(BigNumber)";
    valueFields[8] = null;
    subjectFields[9] = "myBinary";
    aggregateTypes[9] = MemoryGroupByMeta.TYPE_GROUP_LAST;
    aggregateFields[9] = "Last(Binary)";
    valueFields[9] = null;
    subjectFields[10] = "myBoolean";
    aggregateTypes[10] = MemoryGroupByMeta.TYPE_GROUP_LAST;
    aggregateFields[10] = "Last(Boolean)";
    valueFields[10] = null;
    subjectFields[11] = "myDate";
    aggregateTypes[11] = MemoryGroupByMeta.TYPE_GROUP_LAST_INCL_NULL;
    aggregateFields[11] = "LastInclNull(Date)";
    valueFields[11] = null;
    subjectFields[12] = "myTimestamp";
    aggregateTypes[12] = MemoryGroupByMeta.TYPE_GROUP_LAST_INCL_NULL;
    aggregateFields[12] = "LastInclNull(Timestamp)";
    valueFields[12] = null;
    subjectFields[13] = "myInternetAddress";
    aggregateTypes[13] = MemoryGroupByMeta.TYPE_GROUP_MAX;
    aggregateFields[13] = "Max(InternetAddress)";
    valueFields[13] = null;
    subjectFields[14] = "myString";
    aggregateTypes[14] = MemoryGroupByMeta.TYPE_GROUP_MAX;
    aggregateFields[14] = "Max(String)";
    valueFields[14] = null;
    subjectFields[15] = "myInteger";
    // Always returns Number
    aggregateTypes[15] = MemoryGroupByMeta.TYPE_GROUP_MEDIAN;
    aggregateFields[15] = "Median(Integer)";
    valueFields[15] = null;
    subjectFields[16] = "myNumber";
    aggregateTypes[16] = MemoryGroupByMeta.TYPE_GROUP_MIN;
    aggregateFields[16] = "Min(Number)";
    valueFields[16] = null;
    subjectFields[17] = "myBigNumber";
    aggregateTypes[17] = MemoryGroupByMeta.TYPE_GROUP_MIN;
    aggregateFields[17] = "Min(BigNumber)";
    valueFields[17] = null;
    subjectFields[18] = "myBinary";
    aggregateTypes[18] = MemoryGroupByMeta.TYPE_GROUP_PERCENTILE;
    aggregateFields[18] = "Percentile(Binary)";
    valueFields[18] = "0.5";
    subjectFields[19] = "myBoolean";
    aggregateTypes[19] = MemoryGroupByMeta.TYPE_GROUP_STANDARD_DEVIATION;
    aggregateFields[19] = "StandardDeviation(Boolean)";
    valueFields[19] = null;
    subjectFields[20] = "myDate";
    aggregateTypes[20] = MemoryGroupByMeta.TYPE_GROUP_SUM;
    aggregateFields[20] = "Sum(Date)";
    valueFields[20] = null;
    subjectFields[21] = "myInteger";
    aggregateTypes[21] = MemoryGroupByMeta.TYPE_GROUP_SUM;
    aggregateFields[21] = "Sum(Integer)";
    valueFields[21] = null;
    subjectFields[22] = "myInteger";
    aggregateTypes[22] = MemoryGroupByMeta.TYPE_GROUP_AVERAGE;
    aggregateFields[22] = "Average(Integer)";
    valueFields[22] = null;
    subjectFields[23] = "myDate";
    aggregateTypes[23] = MemoryGroupByMeta.TYPE_GROUP_AVERAGE;
    aggregateFields[23] = "Average(Date)";
    valueFields[23] = null;
    meta.setGroupField(groupFields);
    meta.setSubjectField(subjectFields);
    meta.setAggregateType(aggregateTypes);
    meta.setAggregateField(aggregateFields);
    meta.setValueField(valueFields);
    Variables vars = new Variables();
    meta.getFields(rm, stepName, null, null, vars, null, null);
    assertNotNull(rm);
    assertEquals(26, rm.size());
    assertTrue(rm.indexOfValue("myGroupField1") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("myGroupField1")).getType());
    assertTrue(rm.indexOfValue("myGroupField2") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("myGroupField2")).getType());
    assertTrue(rm.indexOfValue("myGroupField2") > rm.indexOfValue("myGroupField1"));
    assertTrue(rm.indexOfValue("ConcatComma") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("ConcatComma")).getType());
    assertTrue(rm.indexOfValue("ConcatString") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("ConcatString")).getType());
    assertTrue(rm.indexOfValue("CountAll") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("CountAll")).getType());
    assertTrue(rm.indexOfValue("CountAny") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("CountAny")).getType());
    assertTrue(rm.indexOfValue("CountDistinct") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("CountDistinct")).getType());
    assertTrue(rm.indexOfValue("First(String)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("First(String)")).getType());
    assertTrue(rm.indexOfValue("First(Integer)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("First(Integer)")).getType());
    assertTrue(rm.indexOfValue("FirstInclNull(Number)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("FirstInclNull(Number)")).getType());
    assertTrue(rm.indexOfValue("FirstInclNull(BigNumber)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_BIGNUMBER, rm.getValueMeta(rm.indexOfValue("FirstInclNull(BigNumber)")).getType());
    assertTrue(rm.indexOfValue("Last(Binary)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_BINARY, rm.getValueMeta(rm.indexOfValue("Last(Binary)")).getType());
    assertTrue(rm.indexOfValue("Last(Boolean)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_BOOLEAN, rm.getValueMeta(rm.indexOfValue("Last(Boolean)")).getType());
    assertTrue(rm.indexOfValue("LastInclNull(Date)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_DATE, rm.getValueMeta(rm.indexOfValue("LastInclNull(Date)")).getType());
    assertTrue(rm.indexOfValue("LastInclNull(Timestamp)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_TIMESTAMP, rm.getValueMeta(rm.indexOfValue("LastInclNull(Timestamp)")).getType());
    assertTrue(rm.indexOfValue("Max(InternetAddress)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INET, rm.getValueMeta(rm.indexOfValue("Max(InternetAddress)")).getType());
    assertTrue(rm.indexOfValue("Max(String)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_STRING, rm.getValueMeta(rm.indexOfValue("Max(String)")).getType());
    assertTrue(rm.indexOfValue("Median(Integer)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Median(Integer)")).getType());
    assertTrue(rm.indexOfValue("Min(Number)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Min(Number)")).getType());
    assertTrue(rm.indexOfValue("Min(BigNumber)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_BIGNUMBER, rm.getValueMeta(rm.indexOfValue("Min(BigNumber)")).getType());
    assertTrue(rm.indexOfValue("Percentile(Binary)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Percentile(Binary)")).getType());
    assertTrue(rm.indexOfValue("StandardDeviation(Boolean)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("StandardDeviation(Boolean)")).getType());
    assertTrue(rm.indexOfValue("Sum(Date)") >= 0);
    // Force changed to Numeric
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Sum(Date)")).getType());
    assertTrue(rm.indexOfValue("Sum(Integer)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("Sum(Integer)")).getType());
    assertTrue(rm.indexOfValue("Average(Integer)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_INTEGER, rm.getValueMeta(rm.indexOfValue("Average(Integer)")).getType());
    assertTrue(rm.indexOfValue("Average(Date)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Average(Date)")).getType());
    // Test Compatibility
    rm = getInputRowMeta();
    vars.setVariable(Const.KETTLE_COMPATIBILITY_MEMORY_GROUP_BY_SUM_AVERAGE_RETURN_NUMBER_TYPE, "Y");
    meta.getFields(rm, stepName, null, null, vars, null, null);
    assertNotNull(rm);
    assertEquals(26, rm.size());
    assertTrue(rm.indexOfValue("Average(Integer)") >= 0);
    assertEquals(ValueMetaInterface.TYPE_NUMBER, rm.getValueMeta(rm.indexOfValue("Average(Integer)")).getType());
}

Example 29 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class MergeJoinMetaTest method testGetFieldsEmptyInput.

@Test
public void testGetFieldsEmptyInput() throws Exception {
    RowMeta outputRowMeta = new RowMeta();
    MergeJoinMeta meta = new MergeJoinMeta();
    RowMeta inputRow1 = new RowMeta();
    ValueMetaInteger field1_row1 = new ValueMetaInteger("field1");
    field1_row1.setOrigin("inputStep1");
    inputRow1.addValueMeta(field1_row1);
    ValueMetaString field2_row1 = new ValueMetaString("field2");
    field2_row1.setOrigin("inputStep1");
    inputRow1.addValueMeta(field2_row1);
    RowMeta inputRow2 = new RowMeta();
    ValueMetaString field1_row2 = new ValueMetaString("field1");
    field1_row2.setOrigin("inputStep2");
    inputRow2.addValueMeta(field1_row2);
    ValueMetaString field3_row2 = new ValueMetaString("field3");
    field3_row2.setOrigin("inputStep2");
    inputRow2.addValueMeta(field3_row2);
    StepMeta stepMeta = new StepMeta("Merge", meta);
    meta.getFields(outputRowMeta, "Merge Join", new RowMetaInterface[] { inputRow1, inputRow2 }, stepMeta, new Variables(), null, null);
    assertNotNull(outputRowMeta);
    assertFalse(outputRowMeta.isEmpty());
    assertEquals(4, outputRowMeta.size());
    List<ValueMetaInterface> vmi = outputRowMeta.getValueMetaList();
    assertNotNull(vmi);
    // Proceed in order
    ValueMetaInterface field1 = outputRowMeta.getValueMeta(0);
    assertNotNull(field1);
    assertEquals("field1", field1.getName());
    assertTrue(field1 instanceof ValueMetaInteger);
    assertEquals("inputStep1", field1.getOrigin());
    ValueMetaInterface field2 = outputRowMeta.getValueMeta(1);
    assertNotNull(field2);
    assertEquals("field2", field2.getName());
    assertTrue(field2 instanceof ValueMetaString);
    assertEquals("inputStep1", field2.getOrigin());
    ValueMetaInterface field1_1 = outputRowMeta.getValueMeta(2);
    assertNotNull(field1_1);
    assertEquals("field1_1", field1_1.getName());
    assertTrue(field1_1 instanceof ValueMetaString);
    assertEquals("Merge Join", field1_1.getOrigin());
    ValueMetaInterface field3 = outputRowMeta.getValueMeta(3);
    assertNotNull(field3);
    assertEquals("field3", field3.getName());
    assertTrue(field3 instanceof ValueMetaString);
    assertEquals("inputStep2", field3.getOrigin());
}

Example 30 with Variables

use of org.pentaho.di.core.variables.Variables in project pentaho-kettle by pentaho.

the class MergeRowsMetaCheckTest method testCheckInputRowsBothEmpty.

@Test
public void testCheckInputRowsBothEmpty() throws KettleStepException {
    when(transMeta.getPrevStepFields(REFERENCE_STEP_NAME)).thenReturn(generateRowMetaEmpty());
    when(transMeta.getPrevStepFields(COMPARISON_STEP_NAME)).thenReturn(generateRowMetaEmpty());
    meta.check(remarks, transMeta, stepMeta, (RowMeta) null, new String[0], new String[0], (RowMeta) null, new Variables(), (Repository) null, (IMetaStore) null);
    assertNotNull(remarks);
    assertTrue(remarks.size() >= 2);
    assertEquals(remarks.get(1).getType(), CheckResultInterface.TYPE_RESULT_OK);
}