Examples with IndexedRecord org.apache.avro.generic.IndexedRecord used on opensource projects

Example 11 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesWriter method getSuccessfulWrites.

/**
     * Retrieves successful writes from wrapped {@link WriterWithFeedback}. Cleans writes from flow variables (aka Out of band)
     * data,
     * stores flow variables in {@link RuntimeContainer} in case they are present
     * 
     * <p>
     * It checks whether {@link Iterable} content is Root records or not and chooses appropriate processing strategy
     * during first call to this method.
     * 
     * <p>
     * Flow variables data is retrieved and stored only from first element of incoming {@link Iterable}
     * It is assumed that {@link Iterable} contains only 1 element in most cases. More elements have no sense
     * for flow variables mechanism, so they are ignored.
     * 
     * <p>
     * Also it is assumed that every element in {@link Iterable} is instance of the same class (has same schema if it is
     * {@link IndexedRecord}). Otherwise errors could appeared during Runtime
     * 
     * @return {@link Iterable} with main data which is released from flow variables data
     */
@Override
public Iterable<Object> getSuccessfulWrites() {
    Iterable successfulWrites = wrappedWriter.getSuccessfulWrites();
    if (successfulWrites == null) {
        throw new NullPointerException("Null successful writes is not allowed");
    }
    if (firstSuccessData) {
        Iterator<?> writesIterator = successfulWrites.iterator();
        if (writesIterator.hasNext()) {
            Object firstData = writesIterator.next();
            if (RootRecordUtils.isRootRecord(firstData)) {
                IndexedRecord rootRecord = (IndexedRecord) firstData;
                successDataProcessor = new FlowVariablesProcessor(runtimeContainer);
                ((FlowVariablesProcessor) successDataProcessor).initSchema(rootRecord);
            } else {
                successDataProcessor = new MainDataProcessor();
            }
            firstSuccessData = false;
        } else {
            return successfulWrites;
        }
    }
    return successDataProcessor.processDataIterable(successfulWrites);
}

Example 12 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class DiIncomingSchemaEnforcerTest method checkEnforcerWithComponentRecordData.

private void checkEnforcerWithComponentRecordData(DiIncomingSchemaEnforcer enforcer) {
    // The enforcer must be ready to receive values.
    assertThat(enforcer.needsInitDynamicColumns(), is(false));
    // Put values into the enforcer and get them as an IndexedRecord.
    enforcer.put(0, 1);
    enforcer.put(1, "User");
    enforcer.put(2, 100);
    enforcer.put(3, true);
    enforcer.put(4, "Main Street");
    enforcer.put(5, "This is a record with six columns.");
    IndexedRecord adapted = enforcer.createIndexedRecord();
    // Ensure that the result is the same as the expected component record.
    assertThat(adapted, is(componentRecord));
    // Ensure that we create a new instance when we give it another value.
    enforcer.put("id", 2);
    enforcer.put("name", "User2");
    enforcer.put("age", 200);
    enforcer.put("valid", false);
    enforcer.put("address", "2 Main Street");
    enforcer.put("comment", "2 This is a record with six columns.");
    IndexedRecord adapted2 = enforcer.createIndexedRecord();
    // It should have the same schema, but not be the same instance.
    assertThat(adapted2.getSchema(), sameInstance(adapted.getSchema()));
    assertThat(adapted2, not(sameInstance(adapted)));
    assertThat(adapted2.get(0), is((Object) 2));
    assertThat(adapted2.get(1), is((Object) "User2"));
    assertThat(adapted2.get(2), is((Object) 200));
    assertThat(adapted2.get(3), is((Object) false));
    assertThat(adapted2.get(4), is((Object) "2 Main Street"));
    assertThat(adapted2.get(5), is((Object) "2 This is a record with six columns."));
}

Example 13 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class DiIncomingSchemaEnforcerTest method testTypeConversion_toLogicalDate.

@Test
public void testTypeConversion_toLogicalDate() {
    // The expected schema after enforcement.
    Schema talend6Schema = SchemaBuilder.builder().record("Record").fields().name("field").type(AvroUtils._logicalDate()).noDefault().endRecord();
    talend6Schema = AvroUtils.setProperty(talend6Schema, DiSchemaConstants.TALEND6_DYNAMIC_COLUMN_POSITION, "3");
    DiIncomingSchemaEnforcer enforcer = new DiIncomingSchemaEnforcer(talend6Schema);
    // No dynamic columns, the schema is available.
    assertThat(enforcer.getDesignSchema(), is(talend6Schema));
    assertThat(enforcer.needsInitDynamicColumns(), is(false));
    assertThat(enforcer.getRuntimeSchema(), is(talend6Schema));
    // Put values into the enforcer and get them as an IndexedRecord.
    enforcer.put(0, new Date(1234567891011L));
    assertThat(enforcer.createIndexedRecord().get(0), is((Object) new Date(1234567891011L)));
    // 2016-05-02T17:30:38.000Z
    enforcer.put(0, "2009-02-13T23:31:31.000Z");
    // "yyyy-MM-dd'T'HH:mm:ss'000Z'"
    IndexedRecord adapted = enforcer.createIndexedRecord();
    assertThat(adapted.getSchema(), sameInstance(enforcer.getRuntimeSchema()));
    assertThat(adapted.get(0), is((Object) new Date(1234567891000L)));
}

Example 14 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesWriterTest method testGetRejectedWrites.

/**
     * Checks {@link FlowVariablesWriter#getRejectedWrites()} returns instance of {@link Iterable} with the only one
     * Main {@link IndexedRecord} unwrapped from Root record
     * and {@link RuntimeContainer} contains flow variable value after this method call
     */
@Test
public void testGetRejectedWrites() throws IOException {
    Schema mainSchema = //
    SchemaBuilder.record("Main").fields().name("name").type().stringType().noDefault().name("age").type().intType().noDefault().endRecord();
    Schema outOfBandSchema = //
    SchemaBuilder.record("OutOfBand").fields().name("id").type().intType().noDefault().endRecord();
    Schema rootSchema = RootSchemaUtils.createRootSchema(mainSchema, outOfBandSchema);
    IndexedRecord mainRecord = new GenericData.Record(mainSchema);
    mainRecord.put(0, "Abraham Lincoln");
    mainRecord.put(1, 208);
    IndexedRecord outOfBandRecord = new GenericData.Record(outOfBandSchema);
    outOfBandRecord.put(0, 123);
    IndexedRecord rootRecord = new GenericData.Record(rootSchema);
    rootRecord.put(0, mainRecord);
    rootRecord.put(1, outOfBandRecord);
    ArrayList<Object> records = new ArrayList<>();
    records.add(rootRecord);
    when(wrappedWriter.getRejectedWrites()).thenReturn(records);
    FlowVariablesWriter<Object> writer = new FlowVariablesWriter<>(wrappedWriter, runtimeContainer);
    Iterable<Object> actualDataIterable = writer.getRejectedWrites();
    Iterator<Object> actualDataIterator = actualDataIterable.iterator();
    assertTrue(actualDataIterator.hasNext());
    Object actualData = actualDataIterator.next();
    assertFalse(actualDataIterator.hasNext());
    assertThat(actualData, instanceOf(IndexedRecord.class));
    IndexedRecord actualMainRecord = (IndexedRecord) actualData;
    assertEquals(mainRecord, actualMainRecord);
    Object flowVariable = runtimeContainer.getComponentData("tComponent_1", "id");
    assertNotNull(flowVariable);
    assertThat(flowVariable, instanceOf(Integer.class));
    assertEquals(123, flowVariable);
    writer.close();
}

Example 15 with IndexedRecord

use of org.apache.avro.generic.IndexedRecord in project tdi-studio-se by Talend.

the class FlowVariablesWriterTest method testGetRejectedWritesSeveralData.

/**
     * Checks {@link FlowVariablesWriter#getRejectedWrites()} returns instance of {@link Iterable} with the same
     * number
     * of data as input {@link Iterable} has. All elements in output {@link Iterable} should be instances of Main data.
     * {@link RuntimeContainer} contains value of first flow variable value after this method call
     */
@Test
public void testGetRejectedWritesSeveralData() throws IOException {
    Schema mainSchema = //
    SchemaBuilder.record("Main").fields().name("name").type().stringType().noDefault().name("age").type().intType().noDefault().endRecord();
    Schema outOfBandSchema = //
    SchemaBuilder.record("OutOfBand").fields().name("id").type().intType().noDefault().endRecord();
    Schema rootSchema = RootSchemaUtils.createRootSchema(mainSchema, outOfBandSchema);
    IndexedRecord mainRecord1 = new GenericData.Record(mainSchema);
    mainRecord1.put(0, "Abraham Lincoln");
    mainRecord1.put(1, 208);
    IndexedRecord outOfBandRecord1 = new GenericData.Record(outOfBandSchema);
    outOfBandRecord1.put(0, 123);
    IndexedRecord mainRecord2 = new GenericData.Record(mainSchema);
    mainRecord2.put(0, "George Washington");
    mainRecord2.put(1, 284);
    IndexedRecord outOfBandRecord2 = new GenericData.Record(outOfBandSchema);
    outOfBandRecord2.put(0, 321);
    IndexedRecord rootRecord1 = new GenericData.Record(rootSchema);
    rootRecord1.put(0, mainRecord1);
    rootRecord1.put(1, outOfBandRecord1);
    IndexedRecord rootRecord2 = new GenericData.Record(rootSchema);
    rootRecord2.put(0, mainRecord2);
    rootRecord2.put(1, outOfBandRecord2);
    ArrayList<Object> records = new ArrayList<>();
    records.add(rootRecord1);
    records.add(rootRecord2);
    when(wrappedWriter.getRejectedWrites()).thenReturn(records);
    FlowVariablesWriter<Object> writer = new FlowVariablesWriter<>(wrappedWriter, runtimeContainer);
    Iterable<Object> actualDataIterable = writer.getRejectedWrites();
    Iterator<Object> actualDataIterator = actualDataIterable.iterator();
    assertTrue(actualDataIterator.hasNext());
    Object actualData1 = actualDataIterator.next();
    assertEquals(mainRecord1, actualData1);
    assertTrue(actualDataIterator.hasNext());
    Object actualData2 = actualDataIterator.next();
    assertEquals(mainRecord2, actualData2);
    assertFalse(actualDataIterator.hasNext());
    Object flowVariable = runtimeContainer.getComponentData("tComponent_1", "id");
    assertNotNull(flowVariable);
    assertThat(flowVariable, instanceOf(Integer.class));
    assertEquals(123, flowVariable);
    writer.close();
}