use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.
the class AzureStorageListReaderTest method testStartAsStartabke.
@Test
public void testStartAsStartabke() {
try {
final List<CloudBlockBlob> list = new ArrayList<>();
list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));
list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob2.txt")));
list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob3.txt")));
when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new DummyListBlobItemIterator(list);
}
});
properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
properties.remoteBlobs.include.setValue(Arrays.asList(true));
properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));
boolean startable = reader.start();
assertTrue(startable);
assertNotNull(reader.getCurrent());
while (reader.advance()) {
assertNotNull(reader.getCurrent());
}
assertNotNull(reader.getReturnValues());
assertEquals(3, reader.getReturnValues().get(ComponentDefinition.RETURN_TOTAL_RECORD_COUNT));
} catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
fail("should not throw " + e.getMessage());
}
}
use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.
the class AzureStorageListReaderTest method testStartAsNonStartable.
@Test
public void testStartAsNonStartable() {
try {
when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new DummyListBlobItemIterator(new ArrayList<CloudBlockBlob>());
}
});
properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
properties.remoteBlobs.include.setValue(Arrays.asList(true));
properties.remoteBlobs.prefix.setValue(Arrays.asList("dummyFilter"));
boolean startable = reader.start();
assertFalse(startable);
assertFalse(reader.advance());
} catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
fail("should not throw " + e.getMessage());
}
}
use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.
the class AzureStorageBaseBlobTestIT method listAllBlobs.
@SuppressWarnings("rawtypes")
public List<String> listAllBlobs(String container) throws Exception {
List<String> blobs = new ArrayList<>();
TAzureStorageListProperties props = new TAzureStorageListProperties("tests");
props.container.setValue(container);
setupConnectionProperties(props);
RemoteBlobsTable rmt = new RemoteBlobsTable("tests");
List<String> pfx = new ArrayList<>();
List<Boolean> inc = new ArrayList<>();
pfx.add("");
inc.add(true);
rmt.prefix.setValue(pfx);
rmt.include.setValue(inc);
props.remoteBlobs = rmt;
props.schema.schema.setValue(schemaForBlobList);
BoundedReader reader = createBoundedReader(props);
Boolean rows = reader.start();
Object row;
while (rows) {
row = ((IndexedRecord) reader.getCurrent()).get(0);
assertNotNull(row);
assertTrue(row instanceof String);
blobs.add(row.toString());
rows = reader.advance();
}
reader.close();
return blobs;
}
use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.
the class AzureStorageListReaderTest method getCurrentOnNonAdvancableReader.
@Test(expected = NoSuchElementException.class)
public void getCurrentOnNonAdvancableReader() {
try {
final List<CloudBlockBlob> list = new ArrayList<>();
list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));
when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new DummyListBlobItemIterator(list);
}
});
properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
properties.remoteBlobs.include.setValue(Arrays.asList(true));
properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));
boolean startable = reader.start();
assertTrue(startable);
assertNotNull(reader.getCurrent());
assertFalse(reader.advance());
reader.getCurrent();
fail("should throw NoSuchElementException");
} catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
fail("should not throw " + e.getMessage());
}
}
use of org.talend.components.azurestorage.blob.helpers.RemoteBlobsTable in project components by Talend.
the class AzureStorageDeleteReaderTestIT method testBlobDelete.
@SuppressWarnings("rawtypes")
@Test
public void testBlobDelete() throws Exception {
TAzureStorageDeleteProperties props = new TAzureStorageDeleteProperties("tests");
props.container.setValue(CONTAINER);
setupConnectionProperties(props);
RemoteBlobsTable rmt = new RemoteBlobsTable("tests");
List<String> pfx = new ArrayList<>();
List<Boolean> inc = new ArrayList<>();
pfx.add("");
inc.add(false);
rmt.prefix.setValue(pfx);
rmt.include.setValue(inc);
props.remoteBlobs = rmt;
BoundedReader reader = createBoundedReader(props);
assertTrue(reader.start());
List<String> blobs = listAllBlobs(CONTAINER);
// blob1.txt, blob2.txt & blob3.txt should be deleted
for (String b : TEST_ROOT_BLOBS) assertFalse(isInBlobList(b, blobs));
// the others should exist
for (String b : TEST_SUB_BLOBS) assertTrue(isInBlobList(b, blobs));
//
// delete sub1 and sub3
//
pfx.clear();
pfx.add("sub1/");
pfx.add("sub3/");
inc.clear();
inc.add(true);
inc.add(true);
rmt.prefix.setValue(pfx);
rmt.include.setValue(inc);
props.remoteBlobs = rmt;
reader = createBoundedReader(props);
assertTrue(reader.start());
blobs = listAllBlobs(CONTAINER);
for (String b : TEST_ROOT_BLOBS) assertFalse(isInBlobList(b, blobs));
for (String b : TEST_SUB1_BLOBS) assertFalse(isInBlobList(b, blobs));
for (String b : TEST_SUB3_BLOBS) assertFalse(isInBlobList(b, blobs));
// the others should exist
for (String b : TEST_SUB2_BLOBS) assertTrue(isInBlobList(b, blobs));
//
// finally delete everything
//
pfx.clear();
pfx.add("");
inc.clear();
inc.add(true);
rmt.prefix.setValue(pfx);
rmt.include.setValue(inc);
props.remoteBlobs = rmt;
reader = createBoundedReader(props);
assertTrue(reader.start());
blobs = listAllBlobs(CONTAINER);
assertTrue(blobs.size() == 0);
}
Aggregations