Examples with AzureBlobConfig org.apache.samza.system.azureblob.AzureBlobConfig used on opensource projects

Example 1 with AzureBlobConfig

use of org.apache.samza.system.azureblob.AzureBlobConfig in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThreadSendToSingleWriter.

@Test
public void testMutipleThreadSendToSingleWriter() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    String stream1 = "FAKE_STREAM_1";
    int sendsInFirstThread = 10;
    int sendsInSecondThread = 20;
    OutgoingMessageEnvelope ome1 = createOME(stream1);
    OutgoingMessageEnvelope ome2 = createAnotherOME(stream1);
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    setupWriterForProducer(systemProducer, mockAzureWriter1, stream1);
    systemProducer.register(source1);
    systemProducer.start();
    Thread t1 = new Thread() {

        @Override
        public void run() {
            for (int i = 0; i < sendsInFirstThread; i++) {
                systemProducer.send(source1, ome1);
            }
        }
    };
    Thread t2 = new Thread() {

        @Override
        public void run() {
            for (int i = 0; i < sendsInSecondThread; i++) {
                systemProducer.send(source1, ome2);
            }
        }
    };
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    verify(mockAzureWriter1, times(sendsInFirstThread)).write(ome1);
    verify(mockAzureWriter1, times(sendsInSecondThread)).write(ome2);
}

Example 2 with AzureBlobConfig

use of org.apache.samza.system.azureblob.AzureBlobConfig in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThreadFlushToSingleWriter.

@Test
public void testMutipleThreadFlushToSingleWriter() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    setupWriterForProducer(systemProducer, mockAzureWriter1, STREAM);
    systemProducer.register(source1);
    systemProducer.start();
    // to create writer
    systemProducer.send(source1, ome);
    Thread t1 = new Thread() {

        @Override
        public void run() {
            systemProducer.flush(source1);
        }
    };
    Thread t2 = new Thread() {

        @Override
        public void run() {
            systemProducer.flush(source1);
        }
    };
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    // systemProducer.flush called twice but first flush clears the writer map of the source.
    // hence, writer.flush and close called only once.
    verify(mockAzureWriter1).flush();
    verify(mockAzureWriter1).close();
}

Example 3 with AzureBlobConfig

use of org.apache.samza.system.azureblob.AzureBlobConfig in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThread.

@Test
public void testMutipleThread() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    String source2 = "FAKE_SOURCE_2";
    String stream1 = "FAKE_STREAM_1";
    String stream2 = "FAKE_STREAM_2";
    int sendsInFirstThread = 10;
    int sendsInSecondThread = 20;
    OutgoingMessageEnvelope ome1 = createOME(stream1);
    OutgoingMessageEnvelope ome2 = createAnotherOME(stream2);
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobWriter mockAzureWriter2 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter2).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    doReturn(mockAzureWriter1).when(systemProducer).getOrCreateWriter(source1, ome1);
    doReturn(mockAzureWriter2).when(systemProducer).getOrCreateWriter(source2, ome2);
    systemProducer.register(source1);
    systemProducer.register(source2);
    systemProducer.start();
    Thread t1 = sendFlushInThread(source1, ome1, systemProducer, sendsInFirstThread);
    Thread t2 = sendFlushInThread(source2, ome2, systemProducer, sendsInSecondThread);
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    verify(mockAzureWriter1, times(sendsInFirstThread)).write(ome1);
    verify(mockAzureWriter2, times(sendsInSecondThread)).write(ome2);
}

Example 4 with AzureBlobConfig

use of org.apache.samza.system.azureblob.AzureBlobConfig in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThreadOneWriterFails.

@Test
public void testMutipleThreadOneWriterFails() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    String source2 = "FAKE_SOURCE_2";
    String stream1 = "FAKE_STREAM_1";
    String stream2 = "FAKE_STREAM_2";
    int sendsInFirstThread = 10;
    int sendsInSecondThread = 20;
    OutgoingMessageEnvelope ome1 = createOME(stream1);
    OutgoingMessageEnvelope ome2 = createAnotherOME(stream2);
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doThrow(new SystemProducerException("failed")).when(mockAzureWriter1).write(ome1);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobWriter mockAzureWriter2 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter2).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    doReturn(mockAzureWriter1).when(systemProducer).getOrCreateWriter(source1, ome1);
    doReturn(mockAzureWriter2).when(systemProducer).getOrCreateWriter(source2, ome2);
    systemProducer.register(source1);
    systemProducer.register(source2);
    systemProducer.start();
    Thread t1 = sendFlushInThread(source1, ome1, systemProducer, sendsInFirstThread);
    Thread t2 = sendFlushInThread(source2, ome2, systemProducer, sendsInSecondThread);
    Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {

        public void uncaughtException(Thread th, Throwable ex) {
            if (ex instanceof SystemProducerException) {
                exceptionOccured = true;
            }
        }
    };
    t1.setUncaughtExceptionHandler(handler);
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    if (!exceptionOccured) {
        Assert.fail("Expected SystemProducerException but did not occur.");
    }
    verify(mockAzureWriter1).write(ome1);
    verify(mockAzureWriter2, times(sendsInSecondThread)).write(ome2);
}

Example 5 with AzureBlobConfig

use of org.apache.samza.system.azureblob.AzureBlobConfig in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThreadSendFlushToSingleWriter.

@Test
public void testMutipleThreadSendFlushToSingleWriter() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    String stream1 = "FAKE_STREAM_1";
    int sendsInFirstThread = 10;
    int sendsInSecondThread = 20;
    OutgoingMessageEnvelope ome1 = createOME(stream1);
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    systemProducer.register(source1);
    systemProducer.start();
    setupWriterForProducer(systemProducer, mockAzureWriter1, stream1);
    Thread t1 = sendFlushInThread(source1, ome1, systemProducer, sendsInFirstThread);
    Thread t2 = sendFlushInThread(source1, ome1, systemProducer, sendsInSecondThread);
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    verify(mockAzureWriter1, times(sendsInFirstThread + sendsInSecondThread)).write(ome1);
    verify(mockAzureWriter1, times(2)).flush();
    verify(mockAzureWriter1, times(2)).close();
}