Examples with ComponentLog org.apache.nifi.logging.ComponentLog used on opensource projects

Example 26 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class ExtractMediaMetadata method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = this.getLogger();
    final AtomicReference<Map<String, String>> value = new AtomicReference<>(null);
    final Integer maxAttribCount = context.getProperty(MAX_NUMBER_OF_ATTRIBUTES).asInteger();
    final Integer maxAttribLength = context.getProperty(MAX_ATTRIBUTE_LENGTH).asInteger();
    final String prefix = context.getProperty(METADATA_KEY_PREFIX).evaluateAttributeExpressions(flowFile).getValue();
    try {
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(InputStream in) throws IOException {
                try {
                    Map<String, String> results = tika_parse(in, prefix, maxAttribCount, maxAttribLength);
                    value.set(results);
                } catch (SAXException | TikaException e) {
                    throw new IOException(e);
                }
            }
        });
        // Write the results to attributes
        Map<String, String> results = value.get();
        if (results != null && !results.isEmpty()) {
            flowFile = session.putAllAttributes(flowFile, results);
        }
        session.transfer(flowFile, SUCCESS);
        session.getProvenanceReporter().modifyAttributes(flowFile, "media attributes extracted");
    } catch (ProcessException e) {
        logger.error("Failed to extract media metadata from {} due to {}", new Object[] { flowFile, e });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, FAILURE);
    }
}

Example 27 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class ITConsumeKafka method validateGetAllMessagesPattern.

@Test
public void validateGetAllMessagesPattern() throws Exception {
    String groupName = "validateGetAllMessagesPattern";
    when(mockConsumerPool.obtainConsumer(anyObject(), anyObject())).thenReturn(mockLease);
    when(mockLease.continuePolling()).thenReturn(Boolean.TRUE, Boolean.TRUE, Boolean.FALSE);
    when(mockLease.commit()).thenReturn(Boolean.TRUE);
    ConsumeKafka_1_0 proc = new ConsumeKafka_1_0() {

        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "0.0.0.0:1234");
    runner.setProperty(ConsumeKafka_1_0.TOPICS, "(fo.*)|(ba)");
    runner.setProperty(ConsumeKafka_1_0.TOPIC_TYPE, "pattern");
    runner.setProperty(ConsumeKafka_1_0.GROUP_ID, groupName);
    runner.setProperty(ConsumeKafka_1_0.AUTO_OFFSET_RESET, ConsumeKafka_1_0.OFFSET_EARLIEST);
    runner.run(1, false);
    verify(mockConsumerPool, times(1)).obtainConsumer(anyObject(), anyObject());
    verify(mockLease, times(3)).continuePolling();
    verify(mockLease, times(2)).poll();
    verify(mockLease, times(1)).commit();
    verify(mockLease, times(1)).close();
    verifyNoMoreInteractions(mockConsumerPool);
    verifyNoMoreInteractions(mockLease);
}

Example 28 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class ITConsumeKafka method validateGetErrorMessages.

@Test
public void validateGetErrorMessages() throws Exception {
    String groupName = "validateGetErrorMessages";
    when(mockConsumerPool.obtainConsumer(anyObject(), anyObject())).thenReturn(mockLease);
    when(mockLease.continuePolling()).thenReturn(true, false);
    when(mockLease.commit()).thenReturn(Boolean.FALSE);
    ConsumeKafka_1_0 proc = new ConsumeKafka_1_0() {

        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "0.0.0.0:1234");
    runner.setProperty(ConsumeKafka_1_0.TOPICS, "foo,bar");
    runner.setProperty(ConsumeKafka_1_0.GROUP_ID, groupName);
    runner.setProperty(ConsumeKafka_1_0.AUTO_OFFSET_RESET, ConsumeKafka_1_0.OFFSET_EARLIEST);
    runner.run(1, false);
    verify(mockConsumerPool, times(1)).obtainConsumer(anyObject(), anyObject());
    verify(mockLease, times(2)).continuePolling();
    verify(mockLease, times(1)).poll();
    verify(mockLease, times(1)).commit();
    verify(mockLease, times(1)).close();
    verifyNoMoreInteractions(mockConsumerPool);
    verifyNoMoreInteractions(mockLease);
}

Example 29 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class TestConsumeKafkaRecord_1_0 method setup.

@Before
public void setup() throws InitializationException {
    mockLease = mock(ConsumerLease.class);
    mockConsumerPool = mock(ConsumerPool.class);
    ConsumeKafkaRecord_1_0 proc = new ConsumeKafkaRecord_1_0() {

        @Override
        protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) {
            return mockConsumerPool;
        }
    };
    runner = TestRunners.newTestRunner(proc);
    runner.setProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS, "okeydokey:1234");
    final String readerId = "record-reader";
    final MockRecordParser readerService = new MockRecordParser();
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    runner.addControllerService(readerId, readerService);
    runner.enableControllerService(readerService);
    final String writerId = "record-writer";
    final RecordSetWriterFactory writerService = new MockRecordWriter("name, age");
    runner.addControllerService(writerId, writerService);
    runner.enableControllerService(writerService);
    runner.setProperty(ConsumeKafkaRecord_1_0.RECORD_READER, readerId);
    runner.setProperty(ConsumeKafkaRecord_1_0.RECORD_WRITER, writerId);
}

Example 30 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class ConsumeKafka method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) {
    final PropertyValue heartbeatIntervalMsConfig = context.getProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG);
    if (heartbeatIntervalMsConfig != null && heartbeatIntervalMsConfig.isSet()) {
        heartbeatCheckIntervalMillis = heartbeatIntervalMsConfig.asInteger();
    } else {
        // Derived from org.apache.kafka.clients.consumer.ConsumerConfig.
        heartbeatCheckIntervalMillis = 3_000;
    }
    // Without this, it remains -1 if downstream connections are full when this processor is scheduled at the 1st run after restart.
    lastTriggeredTimestamp = System.currentTimeMillis();
    // Stop previous connectionRetainer, if any, just in case, this shouldn't happen though
    final ComponentLog logger = getLogger();
    if (connectionRetainer != null) {
        logger.warn("Connection retainer {} is still running, indicating something had happened.", new Object[] { connectionRetainer });
        stopConnectionRetainer();
    }
    connectionRetainer = Executors.newSingleThreadScheduledExecutor();
    connectionRetainer.scheduleAtFixedRate(() -> {
        final long now = System.currentTimeMillis();
        if (lastTriggeredTimestamp < 0 || lastTriggeredTimestamp > now - heartbeatCheckIntervalMillis) {
            if (logger.isDebugEnabled()) {
                logger.debug("No need to retain connection. Triggered at {}, {} millis ago.", new Object[] { lastTriggeredTimestamp, now - lastTriggeredTimestamp });
            }
            return;
        }
        try {
            final ConsumerPool pool = getConsumerPool(context);
            if (logger.isDebugEnabled()) {
                final ConsumerPool.PoolStats stats = pool.getPoolStats();
                logger.debug("Trying to retain connection. Obtained pool={}," + " leaseObtainedCount={}, consumerCreatedCount={}, consumerClosedCount={}", new Object[] { pool, stats.leasesObtainedCount, stats.consumerCreatedCount, stats.consumerClosedCount });
            }
            pool.retainConsumers();
        } catch (final Exception e) {
            logger.warn("Failed to retain connection due to {}", new Object[] { e }, e);
        }
    }, heartbeatCheckIntervalMillis, heartbeatCheckIntervalMillis, TimeUnit.MILLISECONDS);
}