use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.
the class PollingListenerImpl method getStreamWriterForPipe.
/**
* create StreamWriter for pipe
*
* @param pipeConf the pipe configuration
* @return the pipe's streamWriter
* @throws java.io.IOException streamWriter creation failed
*/
private StreamWriter getStreamWriterForPipe(PipeConfiguration pipeConf) throws IOException {
StreamClient client = pipeConf.getSinkConfiguration().getStreamClient();
String streamName = pipeConf.getSinkConfiguration().getStreamName();
try {
client.create(streamName);
StreamWriter writer = client.createWriter(streamName);
return writer;
} catch (IOException e) {
throw new IOException(String.format("Cannot create/get client stream by name: %s: %s", streamName, e));
}
}
use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.
the class FileTailerSinkTest method getDummyConcurrentWriter.
private StreamWriter getDummyConcurrentWriter(final AtomicInteger count) {
StreamWriter writerMock = Mockito.mock(StreamWriter.class);
Mockito.doAnswer(new Answer<ListenableFuture<Void>>() {
@Override
public ListenableFuture<Void> answer(InvocationOnMock invocationOnMock) throws Throwable {
count.incrementAndGet();
return Futures.immediateFuture((Void) null);
}
}).when(writerMock).write("test", Charset.defaultCharset());
return writerMock;
}
use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.
the class FileTailerSinkTest method basicTestWithCustomPackSizeMultipleWriters.
@Test
public void basicTestWithCustomPackSizeMultipleWriters() throws Exception {
FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
final AtomicInteger count = new AtomicInteger(0);
StreamWriter writers = getDummyConcurrentWriter(count);
boolean success = false;
FileTailerSink sink = new FileTailerSink(queue, writers, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, CUSTOM_PACK_SIZE);
try {
sink.startAsync();
for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
}
int attempts = 10;
while (attempts > 0) {
attempts--;
if (count.get() == TEST_EVENTS_SIZE) {
success = true;
break;
}
Thread.sleep(1000);
}
} finally {
sink.stopAsync();
}
org.junit.Assert.assertTrue(success);
}
use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.
the class FileTailerSinkTest method basicTestWithDefaultPackSize.
@Test
public void basicTestWithDefaultPackSize() throws Exception {
FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
StreamWriter writerMock = getDummyStreamWriter();
FileTailerSink sink = new FileTailerSink(queue, writerMock, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null);
sink.startAsync();
for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
}
Mockito.verify(writerMock, Mockito.timeout(10000).times(TEST_EVENTS_SIZE)).write("test", Charset.defaultCharset());
sink.stopAsync();
}
use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.
the class RestStreamClientTest method testNotExistStreamCreateWriter.
@Test
public void testNotExistStreamCreateWriter() throws IOException {
try {
StreamWriter streamWriter = streamClient.createWriter(TestUtils.NOT_FOUND_STREAM_NAME);
assertNotNull(streamWriter);
assertEquals(RestStreamWriter.class, streamWriter.getClass());
RestStreamWriter restStreamWriter = (RestStreamWriter) streamWriter;
assertEquals(TestUtils.SUCCESS_STREAM_NAME, restStreamWriter.getStreamName());
Assert.fail("Expected HttpFailureException");
} catch (HttpFailureException e) {
Assert.assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode());
}
}
Aggregations