Search in sources :

Example 1 with StreamWriter

use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.

the class PollingListenerImpl method getStreamWriterForPipe.

/**
   * create StreamWriter for pipe
   *
   * @param pipeConf the pipe configuration
   * @return the pipe's streamWriter
   * @throws java.io.IOException streamWriter creation failed
   */
private StreamWriter getStreamWriterForPipe(PipeConfiguration pipeConf) throws IOException {
    StreamClient client = pipeConf.getSinkConfiguration().getStreamClient();
    String streamName = pipeConf.getSinkConfiguration().getStreamName();
    try {
        client.create(streamName);
        StreamWriter writer = client.createWriter(streamName);
        return writer;
    } catch (IOException e) {
        throw new IOException(String.format("Cannot create/get client stream by name: %s: %s", streamName, e));
    }
}
Also used : StreamClient(co.cask.cdap.client.StreamClient) StreamWriter(co.cask.cdap.client.StreamWriter) IOException(java.io.IOException)

Example 2 with StreamWriter

use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.

the class FileTailerSinkTest method getDummyConcurrentWriter.

private StreamWriter getDummyConcurrentWriter(final AtomicInteger count) {
    StreamWriter writerMock = Mockito.mock(StreamWriter.class);
    Mockito.doAnswer(new Answer<ListenableFuture<Void>>() {

        @Override
        public ListenableFuture<Void> answer(InvocationOnMock invocationOnMock) throws Throwable {
            count.incrementAndGet();
            return Futures.immediateFuture((Void) null);
        }
    }).when(writerMock).write("test", Charset.defaultCharset());
    return writerMock;
}
Also used : InvocationOnMock(org.mockito.invocation.InvocationOnMock) StreamWriter(co.cask.cdap.client.StreamWriter) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Example 3 with StreamWriter

use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.

the class FileTailerSinkTest method basicTestWithCustomPackSizeMultipleWriters.

@Test
public void basicTestWithCustomPackSizeMultipleWriters() throws Exception {
    FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
    FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
    FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
    final AtomicInteger count = new AtomicInteger(0);
    StreamWriter writers = getDummyConcurrentWriter(count);
    boolean success = false;
    FileTailerSink sink = new FileTailerSink(queue, writers, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null, CUSTOM_PACK_SIZE);
    try {
        sink.startAsync();
        for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
            queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
        }
        int attempts = 10;
        while (attempts > 0) {
            attempts--;
            if (count.get() == TEST_EVENTS_SIZE) {
                success = true;
                break;
            }
            Thread.sleep(1000);
        }
    } finally {
        sink.stopAsync();
    }
    org.junit.Assert.assertTrue(success);
}
Also used : FileTailerState(co.cask.cdap.filetailer.state.FileTailerState) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) StreamWriter(co.cask.cdap.client.StreamWriter) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Test(org.junit.Test)

Example 4 with StreamWriter

use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.

the class FileTailerSinkTest method basicTestWithDefaultPackSize.

@Test
public void basicTestWithDefaultPackSize() throws Exception {
    FileTailerStateProcessor stateProcessor = Mockito.mock(FileTailerStateProcessor.class);
    FileTailerMetricsProcessor metricsProcessor = Mockito.mock(FileTailerMetricsProcessor.class);
    FileTailerQueue queue = new FileTailerQueue(DEFAULT_QUEUE_SIZE);
    StreamWriter writerMock = getDummyStreamWriter();
    FileTailerSink sink = new FileTailerSink(queue, writerMock, SinkStrategy.LOADBALANCE, stateProcessor, metricsProcessor, null);
    sink.startAsync();
    for (int i = 0; i < TEST_EVENTS_SIZE; i++) {
        queue.put(new FileTailerEvent(new FileTailerState("file", 0L, 42, 0L), "test", Charset.defaultCharset()));
    }
    Mockito.verify(writerMock, Mockito.timeout(10000).times(TEST_EVENTS_SIZE)).write("test", Charset.defaultCharset());
    sink.stopAsync();
}
Also used : FileTailerState(co.cask.cdap.filetailer.state.FileTailerState) FileTailerQueue(co.cask.cdap.filetailer.queue.FileTailerQueue) FileTailerEvent(co.cask.cdap.filetailer.event.FileTailerEvent) StreamWriter(co.cask.cdap.client.StreamWriter) FileTailerStateProcessor(co.cask.cdap.filetailer.state.FileTailerStateProcessor) FileTailerMetricsProcessor(co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor) Test(org.junit.Test)

Example 5 with StreamWriter

use of co.cask.cdap.client.StreamWriter in project cdap-ingest by caskdata.

the class RestStreamClientTest method testNotExistStreamCreateWriter.

@Test
public void testNotExistStreamCreateWriter() throws IOException {
    try {
        StreamWriter streamWriter = streamClient.createWriter(TestUtils.NOT_FOUND_STREAM_NAME);
        assertNotNull(streamWriter);
        assertEquals(RestStreamWriter.class, streamWriter.getClass());
        RestStreamWriter restStreamWriter = (RestStreamWriter) streamWriter;
        assertEquals(TestUtils.SUCCESS_STREAM_NAME, restStreamWriter.getStreamName());
        Assert.fail("Expected HttpFailureException");
    } catch (HttpFailureException e) {
        Assert.assertEquals(HttpURLConnection.HTTP_NOT_FOUND, e.getStatusCode());
    }
}
Also used : HttpFailureException(co.cask.common.http.exception.HttpFailureException) StreamWriter(co.cask.cdap.client.StreamWriter) Test(org.junit.Test)

Aggregations

StreamWriter (co.cask.cdap.client.StreamWriter)11 FileTailerQueue (co.cask.cdap.filetailer.queue.FileTailerQueue)6 FileTailerStateProcessor (co.cask.cdap.filetailer.state.FileTailerStateProcessor)6 FileTailerMetricsProcessor (co.cask.cdap.filetailer.metrics.FileTailerMetricsProcessor)5 Test (org.junit.Test)5 StreamClient (co.cask.cdap.client.StreamClient)3 PipeConfiguration (co.cask.cdap.filetailer.config.PipeConfiguration)3 FileTailerEvent (co.cask.cdap.filetailer.event.FileTailerEvent)3 FileTailerSink (co.cask.cdap.filetailer.sink.FileTailerSink)3 FileTailerState (co.cask.cdap.filetailer.state.FileTailerState)3 FileTailerStateProcessorImpl (co.cask.cdap.filetailer.state.FileTailerStateProcessorImpl)3 LogTailer (co.cask.cdap.filetailer.tailer.LogTailer)3 ServiceManager (com.google.common.util.concurrent.ServiceManager)2 Pipe (co.cask.cdap.filetailer.Pipe)1 PipeListener (co.cask.cdap.filetailer.PipeListener)1 ConfigurationLoadingException (co.cask.cdap.filetailer.config.exception.ConfigurationLoadingException)1 HttpFailureException (co.cask.common.http.exception.HttpFailureException)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 IOException (java.io.IOException)1 Field (java.lang.reflect.Field)1