Examples with PropertyValue org.apache.nifi.components.PropertyValue used on opensource projects

Example 16 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class Notify method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    final PropertyValue signalIdProperty = context.getProperty(RELEASE_SIGNAL_IDENTIFIER);
    final PropertyValue counterNameProperty = context.getProperty(SIGNAL_COUNTER_NAME);
    final PropertyValue deltaProperty = context.getProperty(SIGNAL_COUNTER_DELTA);
    final String attributeCacheRegex = context.getProperty(ATTRIBUTE_CACHE_REGEX).getValue();
    final Integer bufferCount = context.getProperty(SIGNAL_BUFFER_COUNT).asInteger();
    // the cache client used to interact with the distributed cache.
    final AtomicDistributedMapCacheClient cache = context.getProperty(DISTRIBUTED_CACHE_SERVICE).asControllerService(AtomicDistributedMapCacheClient.class);
    final WaitNotifyProtocol protocol = new WaitNotifyProtocol(cache);
    final Map<String, SignalBuffer> signalBuffers = new HashMap<>();
    for (int i = 0; i < bufferCount; i++) {
        final FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }
        // Signal id is computed from attribute 'RELEASE_SIGNAL_IDENTIFIER' with expression language support
        final String signalId = signalIdProperty.evaluateAttributeExpressions(flowFile).getValue();
        // if the computed value is null, or empty, we transfer the flow file to failure relationship
        if (StringUtils.isBlank(signalId)) {
            logger.error("FlowFile {} has no attribute for given Release Signal Identifier", new Object[] { flowFile });
            // set 'notified' attribute
            session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(false)), REL_FAILURE);
            continue;
        }
        String counterName = counterNameProperty.evaluateAttributeExpressions(flowFile).getValue();
        if (StringUtils.isEmpty(counterName)) {
            counterName = WaitNotifyProtocol.DEFAULT_COUNT_NAME;
        }
        int delta = 1;
        if (deltaProperty.isSet()) {
            final String deltaStr = deltaProperty.evaluateAttributeExpressions(flowFile).getValue();
            try {
                delta = Integer.parseInt(deltaStr);
            } catch (final NumberFormatException e) {
                logger.error("Failed to calculate delta for FlowFile {} due to {}", new Object[] { flowFile, e }, e);
                session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(false)), REL_FAILURE);
                continue;
            }
        }
        if (!signalBuffers.containsKey(signalId)) {
            signalBuffers.put(signalId, new SignalBuffer());
        }
        final SignalBuffer signalBuffer = signalBuffers.get(signalId);
        if (StringUtils.isNotEmpty(attributeCacheRegex)) {
            flowFile.getAttributes().entrySet().stream().filter(e -> (!e.getKey().equals("uuid") && e.getKey().matches(attributeCacheRegex))).forEach(e -> signalBuffer.attributesToCache.put(e.getKey(), e.getValue()));
        }
        signalBuffer.incrementDelta(counterName, delta);
        signalBuffer.flowFiles.add(flowFile);
        if (logger.isDebugEnabled()) {
            logger.debug("Cached release signal identifier {} counterName {} from FlowFile {}", new Object[] { signalId, counterName, flowFile });
        }
    }
    signalBuffers.forEach((signalId, signalBuffer) -> {
        // retry after yielding for a while.
        try {
            protocol.notify(signalId, signalBuffer.deltas, signalBuffer.attributesToCache);
            signalBuffer.flowFiles.forEach(flowFile -> session.transfer(session.putAttribute(flowFile, NOTIFIED_ATTRIBUTE_NAME, String.valueOf(true)), REL_SUCCESS));
        } catch (IOException e) {
            throw new RuntimeException(String.format("Unable to communicate with cache when processing %s due to %s", signalId, e), e);
        }
    });
}

Example 17 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class TestSiteToSiteBulletinReportingTask method testUrls.

@Test
public void testUrls() throws IOException {
    final ValidationContext context = Mockito.mock(ValidationContext.class);
    Mockito.when(context.newPropertyValue(Mockito.anyString())).then(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(InvocationOnMock invocation) throws Throwable {
            String value = (String) invocation.getArguments()[0];
            return new StandardPropertyValue(value, null);
        }
    });
    assertTrue(new NiFiUrlValidator().validate("url", "http://localhost:8080/nifi", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "http://localhost:8080", context).isValid());
    assertFalse(new NiFiUrlValidator().validate("url", "", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "https://localhost:8080/nifi", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "https://localhost:8080/nifi,https://localhost:8080/nifi", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "https://localhost:8080/nifi, https://localhost:8080/nifi", context).isValid());
    assertFalse(new NiFiUrlValidator().validate("url", "http://localhost:8080/nifi, https://localhost:8080/nifi", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "http://localhost:8080/nifi,http://localhost:8080/nifi", context).isValid());
    assertTrue(new NiFiUrlValidator().validate("url", "http://localhost:8080/nifi,http://localhost:8080", context).isValid());
}

Example 18 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class TestSiteToSiteBulletinReportingTask method testWhenProvenanceMaxIdEqualToLastEventIdInStateManager.

@Test
public void testWhenProvenanceMaxIdEqualToLastEventIdInStateManager() throws IOException, InitializationException {
    // creating the list of bulletins
    final List<Bulletin> bulletins = new ArrayList<Bulletin>();
    bulletins.add(BulletinFactory.createBulletin("category", "severity", "message"));
    bulletins.add(BulletinFactory.createBulletin("category", "severity", "message"));
    bulletins.add(BulletinFactory.createBulletin("category", "severity", "message"));
    bulletins.add(BulletinFactory.createBulletin("category", "severity", "message"));
    // mock the access to the list of bulletins
    final ReportingContext context = Mockito.mock(ReportingContext.class);
    final BulletinRepository repository = Mockito.mock(BulletinRepository.class);
    Mockito.when(context.getBulletinRepository()).thenReturn(repository);
    Mockito.when(repository.findBulletins(Mockito.any(BulletinQuery.class))).thenReturn(bulletins);
    final long maxEventId = getMaxBulletinId(bulletins);
    ;
    // create the mock reporting task and mock state manager
    final MockSiteToSiteBulletinReportingTask task = new MockSiteToSiteBulletinReportingTask();
    final MockStateManager stateManager = new MockStateManager(task);
    // settings properties and mocking access to properties
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    for (final PropertyDescriptor descriptor : task.getSupportedPropertyDescriptors()) {
        properties.put(descriptor, descriptor.getDefaultValue());
    }
    properties.put(SiteToSiteBulletinReportingTask.BATCH_SIZE, "1000");
    properties.put(SiteToSiteBulletinReportingTask.PLATFORM, "nifi");
    properties.put(SiteToSiteBulletinReportingTask.TRANSPORT_PROTOCOL, SiteToSiteTransportProtocol.HTTP.name());
    properties.put(SiteToSiteBulletinReportingTask.HTTP_PROXY_HOSTNAME, "localhost");
    properties.put(SiteToSiteBulletinReportingTask.HTTP_PROXY_PORT, "80");
    properties.put(SiteToSiteBulletinReportingTask.HTTP_PROXY_USERNAME, "username");
    properties.put(SiteToSiteBulletinReportingTask.HTTP_PROXY_PASSWORD, "password");
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
    // create the state map and set the last id to the same value as maxEventId
    final Map<String, String> state = new HashMap<>();
    state.put(SiteToSiteProvenanceReportingTask.LAST_EVENT_ID_KEY, String.valueOf(maxEventId));
    stateManager.setState(state, Scope.LOCAL);
    // setup the mock reporting context to return the mock state manager
    Mockito.when(context.getStateManager()).thenReturn(stateManager);
    // setup the mock initialization context
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class);
    Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    Mockito.when(initContext.getLogger()).thenReturn(logger);
    task.initialize(initContext);
    // execute the reporting task and should not produce any data b/c max id same as previous id
    task.onTrigger(context);
    assertEquals(0, task.dataSent.size());
}

Example 19 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class UpdateRecord method process.

@Override
protected Record process(Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context) {
    final boolean evaluateValueAsRecordPath = context.getProperty(REPLACEMENT_VALUE_STRATEGY).getValue().equals(RECORD_PATH_VALUES.getValue());
    // Incorporate the RecordSchema that we will use for writing records into the Schema that we have
    // for the record, because it's possible that the updates to the record will not be valid otherwise.
    record.incorporateSchema(writeSchema);
    for (final String recordPathText : recordPaths) {
        final RecordPath recordPath = recordPathCache.getCompiled(recordPathText);
        final RecordPathResult result = recordPath.evaluate(record);
        if (evaluateValueAsRecordPath) {
            final String replacementValue = context.getProperty(recordPathText).evaluateAttributeExpressions(flowFile).getValue();
            final RecordPath replacementRecordPath = recordPathCache.getCompiled(replacementValue);
            // If the RecordPath is a Relative Path, then we have to evaluate it against each FieldValue.
            if (replacementRecordPath.isAbsolute()) {
                record = processAbsolutePath(replacementRecordPath, result.getSelectedFields(), record);
            } else {
                record = processRelativePath(replacementRecordPath, result.getSelectedFields(), record);
            }
        } else {
            final PropertyValue replacementValue = context.getProperty(recordPathText);
            if (replacementValue.isExpressionLanguagePresent()) {
                final Map<String, String> fieldVariables = new HashMap<>();
                result.getSelectedFields().forEach(fieldVal -> {
                    fieldVariables.clear();
                    fieldVariables.put(FIELD_NAME, fieldVal.getField().getFieldName());
                    fieldVariables.put(FIELD_VALUE, DataTypeUtils.toString(fieldVal.getValue(), (String) null));
                    fieldVariables.put(FIELD_TYPE, fieldVal.getField().getDataType().getFieldType().name());
                    final String evaluatedReplacementVal = replacementValue.evaluateAttributeExpressions(flowFile, fieldVariables).getValue();
                    fieldVal.updateValue(evaluatedReplacementVal);
                });
            } else {
                final String evaluatedReplacementVal = replacementValue.getValue();
                result.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(evaluatedReplacementVal));
            }
        }
    }
    return record;
}

Example 20 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class SFTPTransfer method getListing.

@Override
public List<FileInfo> getListing() throws IOException {
    final String path = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions().getValue();
    final int depth = 0;
    final int maxResults;
    final PropertyValue batchSizeValue = ctx.getProperty(FileTransfer.REMOTE_POLL_BATCH_SIZE);
    if (batchSizeValue == null) {
        maxResults = Integer.MAX_VALUE;
    } else {
        final Integer configuredValue = batchSizeValue.asInteger();
        maxResults = configuredValue == null ? Integer.MAX_VALUE : configuredValue;
    }
    final List<FileInfo> listing = new ArrayList<>(1000);
    getListing(path, depth, maxResults, listing);
    return listing;
}