Examples with NormalizeProperties org.talend.components.processing.definition.normalize.NormalizeProperties used on opensource projects

Example 11 with NormalizeProperties

use of org.talend.components.processing.definition.normalize.NormalizeProperties in project components by Talend.

the class NormalizeDoFnTest method testNormalizeSimpleFields_a.

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize simple field: `a`
 *
 * Expected normalized results of the field `a`:
 *
 * {"a": "aaa", "b": {"x": "x1;x2", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k1;k2"}, "e": "e"}}, "c":
 * {"f": "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]}
 *
 * @throws Exception
 */
@Test
public void testNormalizeSimpleFields_a() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);
    // Normalize `a` simple field
    properties.columnToNormalize.setValue("a");
    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(1, outputs.size());
    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    Assert.assertEquals(inputParentRecord.toString(), outputRecord.toString());
    Assert.assertEquals(inputParentRecord.getSchema().toString(), outputRecord.getSchema().toString());
}

Example 12 with NormalizeProperties

use of org.talend.components.processing.definition.normalize.NormalizeProperties in project components by Talend.

the class NormalizeDoFnTest method testNormalizeSimpleFields_bydk.

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize simple field: `b.y.d.k`
 *
 * Expected normalized results of the field `b.y.d.k`:
 *
 * [{"a": "aaa", "b": {"x": "x1;x2", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k1"}, "e": "e"}}, "c": {"f":
 * "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]},
 *
 * {"a": "aaa", "b": {"x": "x1;x2", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k2"}, "e": "e"}}, "c": {"f":
 * "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]}]
 *
 * @throws Exception
 */
@Test
public void testNormalizeSimpleFields_bydk() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);
    // Normalize `b.y.d.k` simple field
    properties.columnToNormalize.setValue("b.y.d.k");
    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(2, outputs.size());
    GenericRecord expectedRecordJK1 = // 
    new GenericRecordBuilder(inputSchemaJK).set("j", // 
    listInputRecordL).set("k", // 
    "k1").build();
    GenericRecord expectedRecordDE1 = // 
    new GenericRecordBuilder(inputSchemaDE).set("d", // 
    expectedRecordJK1).set("e", // 
    "e").build();
    GenericRecord expectedRecordXY1 = // 
    new GenericRecordBuilder(inputSchemaXY).set("x", // 
    "x1;x2").set("y", // 
    expectedRecordDE1).build();
    GenericRecord expectedParentRecordK1 = // 
    new GenericRecordBuilder(inputParentSchema).set("a", // 
    "aaa").set("b", // 
    expectedRecordXY1).set("c", // 
    inputRecordFG).set("m", // 
    listInputRecordM).build();
    GenericRecord expectedRecordJK2 = // 
    new GenericRecordBuilder(inputSchemaJK).set("j", // 
    listInputRecordL).set("k", // 
    "k2").build();
    GenericRecord expectedRecordDE2 = // 
    new GenericRecordBuilder(inputSchemaDE).set("d", // 
    expectedRecordJK2).set("e", // 
    "e").build();
    GenericRecord expectedRecordXY2 = // 
    new GenericRecordBuilder(inputSchemaXY).set("x", // 
    "x1;x2").set("y", // 
    expectedRecordDE2).build();
    GenericRecord expectedParentRecordK2 = // 
    new GenericRecordBuilder(inputParentSchema).set("a", // 
    "aaa").set("b", // 
    expectedRecordXY2).set("c", // 
    inputRecordFG).set("m", // 
    listInputRecordM).build();
    GenericRecord outputRecord1 = (GenericRecord) outputs.get(0);
    GenericRecord outputRecord2 = (GenericRecord) outputs.get(1);
    Assert.assertEquals(expectedParentRecordK1.toString(), outputRecord1.toString());
    Assert.assertEquals(expectedParentRecordK1.getSchema().toString(), outputRecord1.getSchema().toString());
    Assert.assertEquals(expectedParentRecordK2.toString(), outputRecord2.toString());
    Assert.assertEquals(expectedParentRecordK2.getSchema().toString(), outputRecord2.getSchema().toString());
}

Example 13 with NormalizeProperties

use of org.talend.components.processing.definition.normalize.NormalizeProperties in project components by Talend.

the class NormalizeDoFnTest method testNormalizeSimpleFields_bx_withDot.

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize simple field: `.b.x`
 *
 * Expected normalized results of the field `b.x`:
 *
 * [{"a": "aaa", "b": {"x": "x1", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k1;k2"}, "e": "e"}}, "c": {"f":
 * "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]},
 *
 * {"a": "aaa", "b": {"x": "x2", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k1;k2"}, "e": "e"}}, "c": {"f":
 * "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]}]
 *
 * @throws Exception
 */
@Test
public void testNormalizeSimpleFields_bx_withDot() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);
    // Normalize `b.x` simple field
    properties.columnToNormalize.setValue(".b.x");
    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(2, outputs.size());
    GenericRecord expectedRecordX1Y = // 
    new GenericRecordBuilder(inputSchemaXY).set("x", // 
    "x1").set("y", // 
    inputRecordDE).build();
    GenericRecord expectedRecordX2Y = // 
    new GenericRecordBuilder(inputSchemaXY).set("x", // 
    "x2").set("y", // 
    inputRecordDE).build();
    GenericRecord expectedParentRecordX1 = // 
    new GenericRecordBuilder(inputParentSchema).set("a", // 
    "aaa").set("b", // 
    expectedRecordX1Y).set("c", // 
    inputRecordFG).set("m", // 
    listInputRecordM).build();
    GenericRecord expectedParentRecordX2 = // 
    new GenericRecordBuilder(inputParentSchema).set("a", // 
    "aaa").set("b", // 
    expectedRecordX2Y).set("c", // 
    inputRecordFG).set("m", // 
    listInputRecordM).build();
    GenericRecord outputRecord1 = (GenericRecord) outputs.get(0);
    GenericRecord outputRecord2 = (GenericRecord) outputs.get(1);
    Assert.assertEquals(expectedParentRecordX1.toString(), outputRecord1.toString());
    Assert.assertEquals(expectedParentRecordX1.getSchema().toString(), outputRecord1.getSchema().toString());
    Assert.assertEquals(expectedParentRecordX2.toString(), outputRecord2.toString());
    Assert.assertEquals(expectedParentRecordX2.getSchema().toString(), outputRecord2.getSchema().toString());
}

Example 14 with NormalizeProperties

use of org.talend.components.processing.definition.normalize.NormalizeProperties in project components by Talend.

the class NormalizeDoFnTest method testNormalizeSimpleFields_bydkt.

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize simple field: `b.y.d.k.t`
 *
 * Throw an exception: the element t does not exist
 *
 * @throws Exception
 */
@Test(expected = TalendRuntimeException.class)
public void testNormalizeSimpleFields_bydkt() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    // Normalize `b.y.d.k.t` simple field
    properties.columnToNormalize.setValue("b.y.d.k.t");
    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    fnTester.processBundle(inputParentRecord);
}

Example 15 with NormalizeProperties

use of org.talend.components.processing.definition.normalize.NormalizeProperties in project components by Talend.

the class NormalizeDoFnTest method testNormalize_nothing.

/**
 * Invalid path to normalize => throw error
 */
@Test
public void testNormalize_nothing() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);
    // Normalize `a` simple field
    properties.columnToNormalize.setValue(null);
    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(0, outputs.size());
}