Examples with SystemProducerException org.apache.samza.system.SystemProducerException used on opensource projects

Example 1 with SystemProducerException

use of org.apache.samza.system.SystemProducerException in project samza by apache.

the class TestAzureBlobSystemProducer method testMutipleThreadOneWriterFails.

@Test
public void testMutipleThreadOneWriterFails() throws Exception {
    String source1 = "FAKE_SOURCE_1";
    String source2 = "FAKE_SOURCE_2";
    String stream1 = "FAKE_STREAM_1";
    String stream2 = "FAKE_STREAM_2";
    int sendsInFirstThread = 10;
    int sendsInSecondThread = 20;
    OutgoingMessageEnvelope ome1 = createOME(stream1);
    OutgoingMessageEnvelope ome2 = createAnotherOME(stream2);
    AzureBlobWriter mockAzureWriter1 = mock(AzureBlobWriter.class);
    doThrow(new SystemProducerException("failed")).when(mockAzureWriter1).write(ome1);
    doNothing().when(mockAzureWriter1).close();
    AzureBlobWriter mockAzureWriter2 = mock(AzureBlobWriter.class);
    doNothing().when(mockAzureWriter2).close();
    AzureBlobConfig azureBlobConfig = new AzureBlobConfig(getBasicConfigs());
    AzureBlobSystemProducer systemProducer = spy(new AzureBlobSystemProducer(SYSTEM_NAME, azureBlobConfig, mockMetricsRegistry));
    // bypass Azure connection setup
    doNothing().when(systemProducer).setupAzureContainer();
    doReturn(mockAzureWriter1).when(systemProducer).getOrCreateWriter(source1, ome1);
    doReturn(mockAzureWriter2).when(systemProducer).getOrCreateWriter(source2, ome2);
    systemProducer.register(source1);
    systemProducer.register(source2);
    systemProducer.start();
    Thread t1 = sendFlushInThread(source1, ome1, systemProducer, sendsInFirstThread);
    Thread t2 = sendFlushInThread(source2, ome2, systemProducer, sendsInSecondThread);
    Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {

        public void uncaughtException(Thread th, Throwable ex) {
            if (ex instanceof SystemProducerException) {
                exceptionOccured = true;
            }
        }
    };
    t1.setUncaughtExceptionHandler(handler);
    t1.start();
    t2.start();
    t1.join(60000);
    t2.join(60000);
    systemProducer.stop();
    if (!exceptionOccured) {
        Assert.fail("Expected SystemProducerException but did not occur.");
    }
    verify(mockAzureWriter1).write(ome1);
    verify(mockAzureWriter2, times(sendsInSecondThread)).write(ome2);
}

Example 2 with SystemProducerException

use of org.apache.samza.system.SystemProducerException in project samza by apache.

the class TestAzureBlobSystemProducer method testFlushWhenWriterUploadFails.

@Test
public void testFlushWhenWriterUploadFails() throws Exception {
    doThrow(new SystemProducerException("failed")).when(mockAzureWriter).flush();
    systemProducer.register(SOURCE);
    systemProducer.start();
    systemProducer.send(SOURCE, ome);
    try {
        systemProducer.flush(SOURCE);
        Assert.fail("Expected exception not thrown.");
    } catch (SystemProducerException e) {
    }
    verify(mockErrorCounter).inc();
}

Example 3 with SystemProducerException

use of org.apache.samza.system.SystemProducerException in project samza by apache.

the class AzureBlobSystemProducer method flush.

/**
 * {@inheritDoc}
 * @throws SystemProducerException
 */
@Override
public void flush(String source) {
    if (!isStarted) {
        throw new SystemProducerException("Trying to flush before producer has started.");
    }
    if (isStopped) {
        throw new SystemProducerException("Flushing after producer has been stopped.");
    }
    ReadWriteLock lock = sourceSendFlushLockMap.get(source);
    if (lock == null) {
        throw new SystemProducerException("Attempting to flush source: " + source + " but it was not registered");
    }
    lock.writeLock().lock();
    Map<String, AzureBlobWriter> sourceWriterMap = writerMap.get(source);
    try {
        // first flush all the writers
        // then close and remove all the writers
        flushWriters(sourceWriterMap);
        closeWriters(source, sourceWriterMap);
    } catch (Exception e) {
        metrics.updateErrorMetrics(source);
        throw new SystemProducerException("Flush failed for system:" + systemName + " and source: " + source, e);
    } finally {
        sourceWriterMap.clear();
        lock.writeLock().unlock();
    }
}

Example 4 with SystemProducerException

use of org.apache.samza.system.SystemProducerException in project samza by apache.

the class AzureBlobSystemProducer method register.

/**
 * {@inheritDoc}
 * @throws SystemProducerException
 */
@Override
public synchronized void register(String source) {
    LOG.info("Registering source {}", source);
    if (isStarted) {
        throw new SystemProducerException("Cannot register once the producer is started.");
    }
    if (writerMap.containsKey(source)) {
        // source already registered => writerMap and metrics have entries for the source
        LOG.warn("Source: {} already registered", source);
        return;
    }
    writerMap.put(source, new ConcurrentHashMap<>());
    sourceWriterCreationLockMap.put(source, new Object());
    sourceSendFlushLockMap.put(source, new ReentrantReadWriteLock());
    metrics.register(source);
}

Example 5 with SystemProducerException

use of org.apache.samza.system.SystemProducerException in project samza by apache.

the class AzureBlobSystemProducer method setupAzureContainer.

@VisibleForTesting
void setupAzureContainer() {
    try {
        BlobServiceAsyncClient storageClient = new AzureBlobClientBuilder(systemName, AZURE_URL, config).getBlobServiceAsyncClient();
        validateFlushThresholdSizeSupported(storageClient);
        containerAsyncClient = storageClient.getBlobContainerAsyncClient(systemName);
        // Only way to check if container exists or not is by creating it and look for failure/success.
        createContainerIfNotExists(containerAsyncClient);
    } catch (Exception e) {
        metrics.updateAzureContainerMetrics();
        throw new SystemProducerException("Failed to set up Azure container for SystemName: " + systemName, e);
    }
}