Examples with IndexedRecord org.apache.avro.generic.IndexedRecord used on opensource projects

Example 6 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class DiIncomingSchemaEnforcerTest method testDynamicColumnALLSupportedType.

@Test
public void testDynamicColumnALLSupportedType() {
    Schema talend6Schema = //
    SchemaBuilder.builder().record("Record").fields().name("valid").type().booleanType().noDefault().endRecord();
    talend6Schema = AvroUtils.setIncludeAllFields(talend6Schema, true);
    talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "0");
    talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'000Z'");
    // 1) Test all type which supported in dynamic
    DiIncomingSchemaEnforcer enforcer = new DiIncomingSchemaEnforcer(talend6Schema);
    // The enforcer isn't usable yet.
    assertThat(enforcer.getDesignSchema(), is(talend6Schema));
    assertThat(enforcer.needsInitDynamicColumns(), is(true));
    assertThat(enforcer.getRuntimeSchema(), nullValue());
    enforcer.initDynamicColumn("Test_String", null, "id_String", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Boolean", null, "id_Boolean", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Integer", null, "id_Integer", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Long", null, "id_Long", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Double", null, "id_Double", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Float", null, "id_Float", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_BigDecimal", null, "id_BigDecimal", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Date", null, "id_Date", null, 0, 0, 0, "yyyy-MM-dd'T'HH:mm:ss'000Z'", null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Byte", null, "id_Byte", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Short", null, "id_Short", null, 0, 0, 0, null, null, false, false, null, null);
    enforcer.initDynamicColumn("Test_Character", null, "id_Character", null, 0, 0, 0, null, null, false, true, null, null);
    try {
        // Throw an exception rather than put schema null to avoid NPE
        enforcer.initDynamicColumn("Test_Unsupported", null, "id_Unsupported", null, 0, 0, 0, null, null, false, false, null, null);
        fail("Expect to get unsupported type exception!");
    } catch (UnsupportedOperationException e) {
    }
    assertThat(enforcer.needsInitDynamicColumns(), is(true));
    enforcer.initDynamicColumnsFinished();
    assertThat(enforcer.needsInitDynamicColumns(), is(false));
    // Check the run-time schema was created.
    assertThat(enforcer.getDesignSchema(), is(talend6Schema));
    assertThat(enforcer.getRuntimeSchema(), not(nullValue()));
    // Put values into the enforcer and get them as an IndexedRecord.
    enforcer.put(0, "string value");
    enforcer.put(1, true);
    enforcer.put(2, 100);
    enforcer.put(3, 1234567891011L);
    enforcer.put(4, 2.15);
    enforcer.put(5, 3.6f);
    enforcer.put(6, new BigDecimal("630.1020"));
    enforcer.put(7, new Date(1234567891011L));
    enforcer.put(8, Byte.parseByte("20"));
    enforcer.put(9, Short.parseShort("2016"));
    enforcer.put(10, 'A');
    assertThat(enforcer.getRuntimeSchema().getFields().get(7).getProp(SchemaConstants.TALEND_COLUMN_PATTERN), is((Object) "yyyy-MM-dd'T'HH:mm:ss'000Z'"));
    IndexedRecord adapted = enforcer.createIndexedRecord();
    assertThat(adapted.get(0), is((Object) "string value"));
    assertThat(adapted.get(1), is((Object) true));
    assertThat(adapted.get(2), is((Object) 100));
    assertThat(adapted.get(3), is((Object) 1234567891011L));
    assertThat(adapted.get(4), is((Object) 2.15));
    assertThat(adapted.get(5), is((Object) 3.6f));
    assertThat(adapted.get(6), is((Object) new BigDecimal("630.1020")));
    assertThat(adapted.get(7), is((Object) new Date(1234567891011L)));
    assertThat(adapted.get(8), is((Object) 20));
    assertThat(adapted.get(9), is((Object) 2016));
    assertThat(adapted.get(10), is((Object) "A"));
    // To date with specified pattern "yyyy-MM-dd'T'HH:mm:ss'000Z'"
    enforcer.put(7, "2009-02-13T23:31:31.000Z");
    adapted = enforcer.createIndexedRecord();
    assertThat(adapted.get(7), is((Object) new Date(1234567891000L)));
    // 2) Test BigDecimal and Date when nullable is true
    enforcer = new DiIncomingSchemaEnforcer(talend6Schema);
    assertThat(enforcer.getDesignSchema(), is(talend6Schema));
    assertThat(enforcer.needsInitDynamicColumns(), is(true));
    assertThat(enforcer.getRuntimeSchema(), nullValue());
    enforcer.initDynamicColumn("Test_BigDecimal", null, "id_BigDecimal", null, 0, 0, 0, null, null, false, true, null, null);
    enforcer.initDynamicColumn("Test_Date", null, "id_Date", null, 0, 0, 0, "yyyy-MM-dd'T'HH:mm:ss'000Z'", null, false, true, null, null);
    assertThat(enforcer.needsInitDynamicColumns(), is(true));
    enforcer.initDynamicColumnsFinished();
    assertThat(enforcer.needsInitDynamicColumns(), is(false));
    // Check the run-time schema was created.
    assertThat(enforcer.getDesignSchema(), is(talend6Schema));
    assertThat(enforcer.getRuntimeSchema(), not(nullValue()));
    enforcer.put(0, new BigDecimal("630.1020"));
    enforcer.put(1, new Date(1234567891011L));
    assertThat(enforcer.getRuntimeSchema().getFields().get(1).getProp(SchemaConstants.TALEND_COLUMN_PATTERN), is((Object) "yyyy-MM-dd'T'HH:mm:ss'000Z'"));
    adapted = enforcer.createIndexedRecord();
    assertThat(adapted.get(0), is((Object) new BigDecimal("630.1020")));
    assertThat(adapted.get(1), is((Object) new Date(1234567891011L)));
}

Example 7 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesProcessor method processRootRecord.

/**
     * Retrieves Main and Flow variables data from Root record.
     * Stores flow variables in Runtime container
     * 
     * @param Root record
     * @return Main data retrieved from root record
     */
private Object processRootRecord(IndexedRecord rootRecord) {
    Object mainData = rootRecord.get(0);
    IndexedRecord flowVariablesRecord = (IndexedRecord) rootRecord.get(1);
    String componentId = runtimeContainer.getCurrentComponentId();
    for (Field field : flowVariablesSchema.getFields()) {
        String flowVariableKey = field.name();
        int fieldPos = field.pos();
        Object flowVariableValue = flowVariablesRecord.get(fieldPos);
        runtimeContainer.setComponentData(componentId, flowVariableKey, flowVariableValue);
    }
    return mainData;
}

Example 8 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class DiIncomingSchemaEnforcer method createIndexedRecord.

/**
     * @return An IndexedRecord created from the values stored in this enforcer and clears out any existing values.
     */
public IndexedRecord createIndexedRecord() {
    // Send the data to a new instance of IndexedRecord and clear out the existing values.
    IndexedRecord copy = wrapped;
    wrapped = null;
    return copy;
}

Example 9 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesWriterTest method testGetSuccessfulWritesSeveralData.

/**
     * Checks {@link FlowVariablesWriter#getSuccessfulWrites()} returns instance of {@link Iterable} with the same
     * number
     * of data as input {@link Iterable} has. All elements in output {@link Iterable} should be instances of Main data.
     * {@link RuntimeContainer} contains value of first flow variable value after this method call
     */
@Test
public void testGetSuccessfulWritesSeveralData() throws IOException {
    Schema mainSchema = //
    SchemaBuilder.record("Main").fields().name("name").type().stringType().noDefault().name("age").type().intType().noDefault().endRecord();
    Schema outOfBandSchema = //
    SchemaBuilder.record("OutOfBand").fields().name("id").type().intType().noDefault().endRecord();
    Schema rootSchema = RootSchemaUtils.createRootSchema(mainSchema, outOfBandSchema);
    IndexedRecord mainRecord1 = new GenericData.Record(mainSchema);
    mainRecord1.put(0, "Abraham Lincoln");
    mainRecord1.put(1, 208);
    IndexedRecord outOfBandRecord1 = new GenericData.Record(outOfBandSchema);
    outOfBandRecord1.put(0, 123);
    IndexedRecord mainRecord2 = new GenericData.Record(mainSchema);
    mainRecord2.put(0, "George Washington");
    mainRecord2.put(1, 284);
    IndexedRecord outOfBandRecord2 = new GenericData.Record(outOfBandSchema);
    outOfBandRecord2.put(0, 321);
    IndexedRecord rootRecord1 = new GenericData.Record(rootSchema);
    rootRecord1.put(0, mainRecord1);
    rootRecord1.put(1, outOfBandRecord1);
    IndexedRecord rootRecord2 = new GenericData.Record(rootSchema);
    rootRecord2.put(0, mainRecord2);
    rootRecord2.put(1, outOfBandRecord2);
    ArrayList<Object> records = new ArrayList<>();
    records.add(rootRecord1);
    records.add(rootRecord2);
    when(wrappedWriter.getSuccessfulWrites()).thenReturn(records);
    FlowVariablesWriter<Object> writer = new FlowVariablesWriter<>(wrappedWriter, runtimeContainer);
    Iterable<Object> actualDataIterable = writer.getSuccessfulWrites();
    Iterator<Object> actualDataIterator = actualDataIterable.iterator();
    assertTrue(actualDataIterator.hasNext());
    Object actualData1 = actualDataIterator.next();
    assertEquals(mainRecord1, actualData1);
    assertTrue(actualDataIterator.hasNext());
    Object actualData2 = actualDataIterator.next();
    assertEquals(mainRecord2, actualData2);
    assertFalse(actualDataIterator.hasNext());
    Object flowVariable = runtimeContainer.getComponentData("tComponent_1", "id");
    assertNotNull(flowVariable);
    assertThat(flowVariable, instanceOf(Integer.class));
    assertEquals(123, flowVariable);
    writer.close();
}

Example 10 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesProcessor method processDataIterable.

/**
     * Processes incoming <code>dataIterable</code> as {@link Iterable} with root records:
     * Retrieves and stores Flow variables data only from first element. In most cases {@link Iterable} contains
     * only one element.
     * Retrieves main data from each root record and copies it to another {@link Iterable},
     * which will be returned
     *
     * @param dataIterable data {@link Iterable} retrieved from component runtime class
     * @return new {@link Iterable} with only main data
     */
@Override
Iterable<Object> processDataIterable(Iterable<Object> dataIterable) {
    LinkedList<Object> mainDataList = new LinkedList<>();
    Iterator<Object> dataIterator = dataIterable.iterator();
    // process first data
    if (dataIterator.hasNext()) {
        IndexedRecord rootRecord = (IndexedRecord) dataIterator.next();
        Object mainData = processRootRecord(rootRecord);
        mainDataList.add(mainData);
    }
    // process rest data
    while (dataIterator.hasNext()) {
        IndexedRecord rootRecord = (IndexedRecord) dataIterator.next();
        Object mainData = rootRecord.get(0);
        mainDataList.add(mainData);
    }
    return mainDataList;
}