Examples with BoundedReader org.talend.components.api.component.runtime.BoundedReader used on opensource projects

Example 6 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class AzureStorageBaseBlobTestIT method cleanupResidualContainersAndFolders.

@SuppressWarnings("rawtypes")
@Before
public void cleanupResidualContainersAndFolders() throws Exception {
    // 
    // cleaning remote containers
    BoundedReader reader = this.createContainerListReader();
    Object container;
    Boolean rows = reader.start();
    while (rows) {
        container = ((IndexedRecord) reader.getCurrent()).get(0);
        assertTrue(container instanceof String);
        if (container.toString().startsWith(TEST_CONTAINER_PREFIX)) {
            doContainerDelete(container.toString());
        }
        rows = reader.advance();
    }
    reader.close();
    // 
    // cleaning local test folder
    cleanupGetFolder();
}

Example 7 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class AzureStorageBaseBlobTestIT method listAllBlobs.

@SuppressWarnings("rawtypes")
public List<String> listAllBlobs(String container) throws Exception {
    List<String> blobs = new ArrayList<>();
    TAzureStorageListProperties props = new TAzureStorageListProperties("tests");
    props.container.setValue(container);
    setupConnectionProperties(props);
    RemoteBlobsTable rmt = new RemoteBlobsTable("tests");
    List<String> pfx = new ArrayList<>();
    List<Boolean> inc = new ArrayList<>();
    pfx.add("");
    inc.add(true);
    rmt.prefix.setValue(pfx);
    rmt.include.setValue(inc);
    props.remoteBlobs = rmt;
    props.schema.schema.setValue(schemaForBlobList);
    BoundedReader reader = createBoundedReader(props);
    Boolean rows = reader.start();
    Object row;
    while (rows) {
        row = ((IndexedRecord) reader.getCurrent()).get(0);
        assertNotNull(row);
        assertTrue(row instanceof String);
        blobs.add(row.toString());
        rows = reader.advance();
    }
    reader.close();
    return blobs;
}

Example 8 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testInsertOrMerge.

@SuppressWarnings("rawtypes")
@Test
public void testInsertOrMerge() throws Throwable {
    currentTable = tbl_test + "InsertOrMerge";
    insertTestValues(currentTable);
    // 
    properties.schema.schema.setValue(getWriteSchema());
    properties.actionOnData.setValue(ActionOnData.Insert_Or_Merge);
    properties.schemaListener.afterSchema();
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    IndexedRecord entity;
    for (String p : partitions) {
        for (String r : rows) {
            entity = new GenericData.Record(getWriteSchema());
            assertEquals(9, entity.getSchema().getFields().size());
            entity.put(0, p);
            entity.put(1, r);
            entity.put(entity.getSchema().getField("longy").pos(), 1000000L * 2);
            writer.write(entity);
        }
    }
    entity = new GenericData.Record(getWriteSchema());
    entity.put(0, pk_test1);
    entity.put(1, "rowKey4");
    entity.put(2, testTimestamp);
    entity.put(3, true);
    entity.put(4, 1000);
    entity.put(5, testString);
    entity.put(6, 1000000L * 2);
    entity.put(7, 100.5562);
    entity.put(8, "ABCDEFGH".getBytes(Charset.defaultCharset()));
    writer.write(entity);
    entity = new GenericData.Record(getWriteSchema());
    entity.put(0, pk_test1);
    entity.put(1, "rowKey5");
    entity.put(2, testTimestamp);
    entity.put(3, true);
    entity.put(4, 1000);
    entity.put(5, testString);
    entity.put(6, 1000000L * 2);
    entity.put(7, 100.5562);
    entity.put(8, "ABCDEFGH".getBytes(Charset.defaultCharset()));
    writer.write(entity);
    writer.close();
    // check results
    BoundedReader reader = createReader(currentTable, filter, false);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertEquals(1000000L * 2, current.get(current.getSchema().getField("longy").pos()));
        // checks that other fields remained the sames...
        assertEquals(current.get(current.getSchema().getField("daty").pos()), testTimestamp);
        assertEquals(current.get(current.getSchema().getField("inty").pos()), 1000);
        assertEquals(current.get(current.getSchema().getField("stringy").pos()), testString);
        assertEquals(current.get(current.getSchema().getField("doubly").pos()), 100.5562);
        assertEquals(new String((byte[]) current.get(current.getSchema().getField("bytys").pos())), "ABCDEFGH");
        assertEquals(10, current.getSchema().getFields().size());
    } while (reader.advance());
    reader.close();
    // we should have read 9 + 2 rows...
    assertEquals(11, counted);
}

Example 9 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class TAzureStorageOuputTableTestIT method testInsert.

@Test
@SuppressWarnings("rawtypes")
public void testInsert() throws Throwable {
    currentTable = tbl_test + "Insert";
    insertTestValues(currentTable);
    // check results...
    BoundedReader reader = createReader(currentTable, filter, false);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertEquals(current.get(current.getSchema().getField("daty").pos()), testTimestamp);
        assertEquals(current.get(current.getSchema().getField("inty").pos()), 1000);
        assertEquals(current.get(current.getSchema().getField("stringy").pos()), testString);
        assertEquals(current.get(current.getSchema().getField("longy").pos()), 1000000L);
        assertEquals(current.get(current.getSchema().getField("doubly").pos()), 100.5562);
        assertEquals(new String((byte[]) current.get(current.getSchema().getField("bytys").pos())), "ABCDEFGH");
    } while (reader.advance());
    reader.close();
    // we should have read 9 rows...
    assertEquals(9, counted);
}

Example 10 with BoundedReader

use of org.talend.components.api.component.runtime.BoundedReader in project components by Talend.

the class AzureStorageQueueCreateReaderTestIT method testCreateExistingQueue.

@SuppressWarnings("rawtypes")
@Test
public void testCreateExistingQueue() throws Throwable {
    TAzureStorageQueueCreateProperties properties = new TAzureStorageQueueCreateProperties("tests");
    properties.setupProperties();
    properties = (TAzureStorageQueueCreateProperties) setupConnectionProperties((AzureStorageProvideConnectionProperties) properties);
    String fqueue = TEST_QUEUE_NAME_CREATE + getRandomTestUID() + "b";
    properties.queueName.setValue(fqueue);
    BoundedReader reader = createBoundedReader(properties);
    assertTrue(reader.start());
    assertTrue((int) reader.getReturnValues().get(AzureStorageDefinition.RETURN_TOTAL_RECORD_COUNT) > 0);
    reader.close();
    // 
    assertFalse(reader.start());
    assertTrue((int) reader.getReturnValues().get(AzureStorageDefinition.RETURN_TOTAL_RECORD_COUNT) > 0);
    reader.close();
}