Examples with BoundedReader org.talend.components.api.component.runtime.BoundedReader used on opensource projects

Example 26 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testNameMappings.

@Test
@SuppressWarnings("rawtypes")
public void testNameMappings() throws Throwable {
    currentTable = tbl_test + "InsertWithNameMappings";
    properties.nameMapping.schemaColumnName.setValue(schemaMappings);
    properties.nameMapping.entityPropertyName.setValue(propertyMappings);
    insertTestValues(currentTable);
    properties.nameMapping.schemaColumnName.setValue(new ArrayList<String>());
    properties.nameMapping.entityPropertyName.setValue(new ArrayList<String>());
    // check results...
    BoundedReader reader = createReader(currentTable, filter, true);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertEquals(current.get(current.getSchema().getField("daty").pos()), testTimestamp);
        assertEquals(current.get(current.getSchema().getField("inty").pos()), 1000);
        assertEquals(current.get(current.getSchema().getField("stringy").pos()), testString);
        assertEquals(current.get(current.getSchema().getField("longy").pos()), 1000000L);
        assertEquals(current.get(current.getSchema().getField("doubly").pos()), 100.5562);
        assertEquals(new String((byte[]) current.get(current.getSchema().getField("bytys").pos())), "ABCDEFGH");
    } while (reader.advance());
    reader.close();
    // we should have read 9 rows...
    assertEquals(9, counted);
}

Example 27 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testInsertOrReplaceNullValue.

@SuppressWarnings("rawtypes")
@Test
public void testInsertOrReplaceNullValue() throws Throwable {
    currentTable = tbl_test + "InsertOrReplaceNullValue";
    insertTestValues(currentTable);
    // 
    properties.schema.schema.setValue(getSimpleTestSchema());
    properties.actionOnData.setValue(ActionOnData.Insert_Or_Replace);
    properties.schemaListener.afterSchema();
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    IndexedRecord entity;
    for (String p : partitions) {
        for (String r : rows) {
            entity = new GenericData.Record(getSimpleTestSchema());
            assertEquals(3, entity.getSchema().getFields().size());
            entity.put(0, p);
            entity.put(1, r);
            entity.put(2, null);
            writer.write(entity);
        }
    }
    writer.close();
    // check results
    BoundedReader reader = createReader(currentTable, filter, false);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertNull(current.getSchema().getField("StringValue"));
        // Column with null values are not writed to azure
        assertEquals(3, current.getSchema().getFields().size());
    } while (reader.advance());
    reader.close();
    assertEquals(9, counted);
}

Example 28 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testInsertOrReplace.

@SuppressWarnings("rawtypes")
@Test
public void testInsertOrReplace() throws Throwable {
    currentTable = tbl_test + "InsertOrReplace";
    insertTestValues(currentTable);
    // 
    properties.schema.schema.setValue(getSimpleTestSchema());
    properties.actionOnData.setValue(ActionOnData.Insert_Or_Replace);
    properties.schemaListener.afterSchema();
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    IndexedRecord entity;
    for (String p : partitions) {
        for (String r : rows) {
            entity = new GenericData.Record(getSimpleTestSchema());
            assertEquals(3, entity.getSchema().getFields().size());
            entity.put(0, p);
            entity.put(1, r);
            entity.put(2, "NewValue");
            writer.write(entity);
        }
    }
    // Adding extra entities with rowKey4
    entity = new GenericData.Record(getSimpleTestSchema());
    entity.put(0, pk_test1);
    entity.put(1, "rowKey4");
    entity.put(2, "NewValue");
    writer.write(entity);
    // Adding extra entities with rowKey5
    entity = new GenericData.Record(getWriteSchema());
    entity.put(0, pk_test1);
    entity.put(1, "rowKey5");
    entity.put(2, "NewValue");
    writer.write(entity);
    writer.close();
    // check results
    BoundedReader reader = createReader(currentTable, filter, false);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertEquals(current.get(current.getSchema().getField("StringValue").pos()), "NewValue");
        assertEquals(4, current.getSchema().getFields().size());
    } while (reader.advance());
    reader.close();
    // we should have read 9 +2 rows...
    assertEquals(11, counted);
}

Example 29 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testBatchInsert.

@SuppressWarnings("rawtypes")
@Test
public void testBatchInsert() throws Throwable {
    currentTable = tbl_test + "BatchInsert";
    // 
    properties.tableName.setValue(currentTable);
    properties.schema.schema.setValue(getDynamicSchema());
    properties.actionOnData.setValue(ActionOnData.Insert);
    properties.processOperationInBatch.setValue(true);
    properties.schemaListener.afterSchema();
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    for (String r : rows) {
        IndexedRecord entity = new GenericData.Record(getWriteSchema());
        entity.put(0, pk_test1);
        entity.put(1, r);
        entity.put(2, testTimestamp);
        entity.put(3, true);
        entity.put(4, 1000);
        entity.put(5, testString);
        entity.put(6, 1000000L);
        entity.put(7, 100.5562);
        entity.put(8, "ABCDEFGH".getBytes(Charset.defaultCharset()));
        writer.write(entity);
    }
    // no record should exist we have less than 100 operations on the same PK...
    BoundedReader reader = createReader(currentTable, filter, false);
    assertFalse(reader.start());
    reader.close();
    // close should trigger the batch
    writer.close();
    reader = createReader(currentTable, filter, false);
    assertTrue(reader.start());
    reader.close();
    properties.processOperationInBatch.setValue(false);
}

Example 30 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class AzureStorageQueueCreateReaderTestIT method testCreateNewQueue.

@SuppressWarnings("rawtypes")
@Test
public void testCreateNewQueue() throws Throwable {
    TAzureStorageQueueCreateProperties properties = new TAzureStorageQueueCreateProperties("tests");
    properties.setupProperties();
    properties = (TAzureStorageQueueCreateProperties) setupConnectionProperties((AzureStorageProvideConnectionProperties) properties);
    String fqueue = TEST_QUEUE_NAME_CREATE + getRandomTestUID() + "a";
    properties.queueName.setValue(fqueue);
    BoundedReader reader = createBoundedReader(properties);
    assertTrue(reader.start());
    assertNotNull(reader.getCurrent());
    assertTrue(((IndexedRecord) reader.getCurrent()).get(0) instanceof String);
    assertEquals(((IndexedRecord) reader.getCurrent()).get(0), fqueue);
    assertTrue((int) reader.getReturnValues().get(AzureStorageDefinition.RETURN_TOTAL_RECORD_COUNT) > 0);
    reader.close();
}