Examples with RemoteBlobsTable org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable used on opensource projects

Example 1 with RemoteBlobsTable

use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.

the class AzureStorageListReaderTest method testStartAsStartabke.

@Test
public void testStartAsStartabke() {
    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob2.txt")));
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob3.txt")));
        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });
        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));
        boolean startable = reader.start();
        assertTrue(startable);
        assertNotNull(reader.getCurrent());
        while (reader.advance()) {
            assertNotNull(reader.getCurrent());
        }
        assertNotNull(reader.getReturnValues());
        assertEquals(3, reader.getReturnValues().get(ComponentDefinition.RETURN_TOTAL_RECORD_COUNT));
    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}

Example 2 with RemoteBlobsTable

use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.

the class AzureStorageListReaderTest method testStartAsNonStartable.

@Test
public void testStartAsNonStartable() {
    try {
        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(new ArrayList<CloudBlockBlob>());
            }
        });
        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("dummyFilter"));
        boolean startable = reader.start();
        assertFalse(startable);
        assertFalse(reader.advance());
    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}

Example 3 with RemoteBlobsTable

use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.

the class AzureStorageBaseBlobTestIT method listAllBlobs.

@SuppressWarnings("rawtypes")
public List<String> listAllBlobs(String container) throws Exception {
    List<String> blobs = new ArrayList<>();
    TAzureStorageListProperties props = new TAzureStorageListProperties("tests");
    props.container.setValue(container);
    setupConnectionProperties(props);
    RemoteBlobsTable rmt = new RemoteBlobsTable("tests");
    List<String> pfx = new ArrayList<>();
    List<Boolean> inc = new ArrayList<>();
    pfx.add("");
    inc.add(true);
    rmt.prefix.setValue(pfx);
    rmt.include.setValue(inc);
    props.remoteBlobs = rmt;
    props.schema.schema.setValue(schemaForBlobList);
    BoundedReader reader = createBoundedReader(props);
    Boolean rows = reader.start();
    Object row;
    while (rows) {
        row = ((IndexedRecord) reader.getCurrent()).get(0);
        assertNotNull(row);
        assertTrue(row instanceof String);
        blobs.add(row.toString());
        rows = reader.advance();
    }
    reader.close();
    return blobs;
}

Example 4 with RemoteBlobsTable

use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.

the class AzureStorageListReaderTest method getCurrentOnNonAdvancableReader.

@Test(expected = NoSuchElementException.class)
public void getCurrentOnNonAdvancableReader() {
    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));
        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });
        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));
        boolean startable = reader.start();
        assertTrue(startable);
        assertNotNull(reader.getCurrent());
        assertFalse(reader.advance());
        reader.getCurrent();
        fail("should throw NoSuchElementException");
    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}

Example 5 with RemoteBlobsTable

use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.

the class AzureStorageDeleteReaderTestIT method testBlobDelete.

@SuppressWarnings("rawtypes")
@Test
public void testBlobDelete() throws Exception {
    TAzureStorageDeleteProperties props = new TAzureStorageDeleteProperties("tests");
    props.container.setValue(CONTAINER);
    setupConnectionProperties(props);
    RemoteBlobsTable rmt = new RemoteBlobsTable("tests");
    List<String> pfx = new ArrayList<>();
    List<Boolean> inc = new ArrayList<>();
    pfx.add("");
    inc.add(false);
    rmt.prefix.setValue(pfx);
    rmt.include.setValue(inc);
    props.remoteBlobs = rmt;
    BoundedReader reader = createBoundedReader(props);
    assertTrue(reader.start());
    List<String> blobs = listAllBlobs(CONTAINER);
    // blob1.txt, blob2.txt & blob3.txt should be deleted
    for (String b : TEST_ROOT_BLOBS) assertFalse(isInBlobList(b, blobs));
    // the others should exist
    for (String b : TEST_SUB_BLOBS) assertTrue(isInBlobList(b, blobs));
    // 
    // delete sub1 and sub3
    // 
    pfx.clear();
    pfx.add("sub1/");
    pfx.add("sub3/");
    inc.clear();
    inc.add(true);
    inc.add(true);
    rmt.prefix.setValue(pfx);
    rmt.include.setValue(inc);
    props.remoteBlobs = rmt;
    reader = createBoundedReader(props);
    assertTrue(reader.start());
    blobs = listAllBlobs(CONTAINER);
    for (String b : TEST_ROOT_BLOBS) assertFalse(isInBlobList(b, blobs));
    for (String b : TEST_SUB1_BLOBS) assertFalse(isInBlobList(b, blobs));
    for (String b : TEST_SUB3_BLOBS) assertFalse(isInBlobList(b, blobs));
    // the others should exist
    for (String b : TEST_SUB2_BLOBS) assertTrue(isInBlobList(b, blobs));
    // 
    // finally delete everything
    // 
    pfx.clear();
    pfx.add("");
    inc.clear();
    inc.add(true);
    rmt.prefix.setValue(pfx);
    rmt.include.setValue(inc);
    props.remoteBlobs = rmt;
    reader = createBoundedReader(props);
    assertTrue(reader.start());
    blobs = listAllBlobs(CONTAINER);
    assertTrue(blobs.size() == 0);
}