Examples with PropertyValue org.apache.nifi.components.PropertyValue used on opensource projects

Example 56 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class LookupAttribute method onTrigger.

private void onTrigger(ComponentLog logger, LookupService lookupService, boolean includeEmptyValues, FlowFile flowFile, ProcessSession session) throws ProcessException, IOException {
    final Map<String, String> attributes = new HashMap<>(flowFile.getAttributes());
    boolean matched = false;
    try {
        final Set<String> requiredKeys = lookupService.getRequiredKeys();
        if (requiredKeys == null || requiredKeys.size() != 1) {
            throw new ProcessException("LookupAttribute requires a key-value lookup service supporting exactly one required key, was: " + (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size())));
        }
        final String coordinateKey = requiredKeys.iterator().next();
        for (final Map.Entry<PropertyDescriptor, PropertyValue> e : dynamicProperties.entrySet()) {
            final PropertyValue lookupKeyExpression = e.getValue();
            final String lookupKey = lookupKeyExpression.evaluateAttributeExpressions(flowFile).getValue();
            final String attributeName = e.getKey().getName();
            final Optional<String> attributeValue = lookupService.lookup(Collections.singletonMap(coordinateKey, lookupKey));
            matched = putAttribute(attributeName, attributeValue, attributes, includeEmptyValues, logger) || matched;
            if (!matched && logger.isDebugEnabled()) {
                logger.debug("No such value for key: {}", new Object[] { lookupKey });
            }
        }
        flowFile = session.putAllAttributes(flowFile, attributes);
        session.transfer(flowFile, matched ? REL_MATCHED : REL_UNMATCHED);
    } catch (final LookupFailureException e) {
        logger.error(e.getMessage(), e);
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 57 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class LookupAttribute method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) {
    // Load up all the dynamic properties once for use later in onTrigger
    final Map<PropertyDescriptor, PropertyValue> dynamicProperties = new HashMap<>();
    for (final Map.Entry<PropertyDescriptor, String> e : context.getProperties().entrySet()) {
        final PropertyDescriptor descriptor = e.getKey();
        if (descriptor.isDynamic()) {
            final PropertyValue value = context.getProperty(descriptor);
            dynamicProperties.put(descriptor, value);
        }
    }
    this.dynamicProperties = Collections.unmodifiableMap(dynamicProperties);
}

Example 58 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class GetHTTP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
    final ComponentLog logger = getLogger();
    final ProcessSession session = sessionFactory.createSession();
    final FlowFile incomingFlowFile = session.get();
    if (incomingFlowFile != null) {
        session.transfer(incomingFlowFile, REL_SUCCESS);
        logger.warn("found FlowFile {} in input queue; transferring to success", new Object[] { incomingFlowFile });
    }
    // get the URL
    final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue();
    final URI uri;
    String source = url;
    try {
        uri = new URI(url);
        source = uri.getHost();
    } catch (final URISyntaxException swallow) {
    // this won't happen as the url has already been validated
    }
    // get the ssl context service
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    // create the connection manager
    final HttpClientConnectionManager conMan;
    if (sslContextService == null) {
        conMan = new BasicHttpClientConnectionManager();
    } else {
        final SSLContext sslContext;
        try {
            sslContext = createSSLContext(sslContextService);
        } catch (final Exception e) {
            throw new ProcessException(e);
        }
        final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
        // Also include a plain socket factory for regular http connections (especially proxies)
        final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf).register("http", PlainConnectionSocketFactory.getSocketFactory()).build();
        conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry);
    }
    try {
        // build the request configuration
        final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
        requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean());
        switch(context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) {
            case STANDARD_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
                break;
            case STRICT_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT);
                break;
            case NETSCAPE_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE);
                break;
            case IGNORE_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
                break;
            case DEFAULT_COOKIE_POLICY_STR:
            default:
                requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
        }
        // build the http client
        final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setConnectionManager(conMan);
        // include the user agent
        final String userAgent = context.getProperty(USER_AGENT).getValue();
        if (userAgent != null) {
            clientBuilder.setUserAgent(userAgent);
        }
        // set the ssl context if necessary
        if (sslContextService != null) {
            clientBuilder.setSslcontext(sslContextService.createSSLContext(ClientAuth.REQUIRED));
        }
        final String username = context.getProperty(USERNAME).getValue();
        final String password = context.getProperty(PASSWORD).getValue();
        // set the credentials if appropriate
        if (username != null) {
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            if (password == null) {
                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username));
            } else {
                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
            }
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }
        // Set the proxy if specified
        if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
            final String host = context.getProperty(PROXY_HOST).getValue();
            final int port = context.getProperty(PROXY_PORT).asInteger();
            clientBuilder.setProxy(new HttpHost(host, port));
        }
        // create request
        final HttpGet get = new HttpGet(url);
        get.setConfig(requestConfigBuilder.build());
        final StateMap beforeStateMap;
        try {
            beforeStateMap = context.getStateManager().getState(Scope.LOCAL);
            final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url);
            if (lastModified != null) {
                get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue());
            }
            final String etag = beforeStateMap.get(ETAG + ":" + url);
            if (etag != null) {
                get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue());
            }
        } catch (final IOException ioe) {
            throw new ProcessException(ioe);
        }
        final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue();
        if (accept != null) {
            get.addHeader(HEADER_ACCEPT, accept);
        }
        // Add dynamic headers
        PropertyValue customHeaderValue;
        for (PropertyDescriptor customProperty : customHeaders) {
            customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions();
            if (StringUtils.isNotBlank(customHeaderValue.getValue())) {
                get.addHeader(customProperty.getName(), customHeaderValue.getValue());
            }
        }
        // create the http client
        try (final CloseableHttpClient client = clientBuilder.build()) {
            // NOTE: including this inner try in order to swallow exceptions on close
            try {
                final StopWatch stopWatch = new StopWatch(true);
                final HttpResponse response = client.execute(get);
                final int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode == NOT_MODIFIED) {
                    logger.info("content not retrieved because server returned HTTP Status Code {}: Not Modified", new Object[] { NOT_MODIFIED });
                    context.yield();
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }
                final String statusExplanation = response.getStatusLine().getReasonPhrase();
                if ((statusCode >= 300) || (statusCode == 204)) {
                    logger.error("received status code {}:{} from {}", new Object[] { statusCode, statusExplanation, url });
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }
                FlowFile flowFile = session.create();
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
                flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", source);
                flowFile = session.importFrom(response.getEntity().getContent(), flowFile);
                final Header contentTypeHeader = response.getFirstHeader("Content-Type");
                if (contentTypeHeader != null) {
                    final String contentType = contentTypeHeader.getValue();
                    if (!contentType.trim().isEmpty()) {
                        flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), contentType.trim());
                    }
                }
                final long flowFileSize = flowFile.getSize();
                stopWatch.stop();
                final String dataRate = stopWatch.calculateDataRate(flowFileSize);
                session.getProvenanceReporter().receive(flowFile, url, stopWatch.getDuration(TimeUnit.MILLISECONDS));
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully received {} from {} at a rate of {}; transferred to success", new Object[] { flowFile, url, dataRate });
                session.commit();
                updateStateMap(context, response, beforeStateMap, url);
            } catch (final IOException e) {
                context.yield();
                session.rollback();
                logger.error("Failed to retrieve file from {} due to {}; rolling back session", new Object[] { url, e.getMessage() }, e);
                throw new ProcessException(e);
            } catch (final Throwable t) {
                context.yield();
                session.rollback();
                logger.error("Failed to process due to {}; rolling back session", new Object[] { t.getMessage() }, t);
                throw t;
            }
        } catch (final IOException e) {
            logger.debug("Error closing client due to {}, continuing.", new Object[] { e.getMessage() });
        }
    } finally {
        conMan.shutdown();
    }
}

Example 59 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class TestSiteToSiteBulletinReportingTask method testSerializedForm.

@Test
public void testSerializedForm() throws IOException, InitializationException {
    // creating the list of bulletins
    final List<Bulletin> bulletins = new ArrayList<Bulletin>();
    bulletins.add(BulletinFactory.createBulletin("group-id", "group-name", "source-id", "source-name", "category", "severity", "message"));
    // mock the access to the list of bulletins
    final ReportingContext context = Mockito.mock(ReportingContext.class);
    final BulletinRepository repository = Mockito.mock(BulletinRepository.class);
    Mockito.when(context.getBulletinRepository()).thenReturn(repository);
    Mockito.when(repository.findBulletins(Mockito.any(BulletinQuery.class))).thenReturn(bulletins);
    // creating reporting task
    final MockSiteToSiteBulletinReportingTask task = new MockSiteToSiteBulletinReportingTask();
    Mockito.when(context.getStateManager()).thenReturn(new MockStateManager(task));
    // settings properties and mocking access to properties
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    for (final PropertyDescriptor descriptor : task.getSupportedPropertyDescriptors()) {
        properties.put(descriptor, descriptor.getDefaultValue());
    }
    properties.put(SiteToSiteBulletinReportingTask.BATCH_SIZE, "1000");
    properties.put(SiteToSiteBulletinReportingTask.PLATFORM, "nifi");
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
    // setup the mock initialization context
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class);
    Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    Mockito.when(initContext.getLogger()).thenReturn(logger);
    task.initialize(initContext);
    task.onTrigger(context);
    // test checking
    assertEquals(1, task.dataSent.size());
    final String msg = new String(task.dataSent.get(0), StandardCharsets.UTF_8);
    JsonReader jsonReader = Json.createReader(new ByteArrayInputStream(msg.getBytes()));
    JsonObject bulletinJson = jsonReader.readArray().getJsonObject(0);
    assertEquals("message", bulletinJson.getString("bulletinMessage"));
    assertEquals("group-name", bulletinJson.getString("bulletinGroupName"));
}

Example 60 with PropertyValue

use of org.apache.nifi.components.PropertyValue in project nifi by apache.

the class TestSiteToSiteProvenanceReportingTask method setup.

private MockSiteToSiteProvenanceReportingTask setup(ProvenanceEventRecord event, Map<PropertyDescriptor, String> properties, long maxEventId) throws IOException {
    final MockSiteToSiteProvenanceReportingTask task = new MockSiteToSiteProvenanceReportingTask();
    when(context.getStateManager()).thenReturn(new MockStateManager(task));
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(confContext).getProperty(Mockito.any(PropertyDescriptor.class));
    final AtomicInteger totalEvents = new AtomicInteger(0);
    final EventAccess eventAccess = Mockito.mock(EventAccess.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {

        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
            final long startId = invocation.getArgumentAt(0, long.class);
            final int maxRecords = invocation.getArgumentAt(1, int.class);
            final List<ProvenanceEventRecord> eventsToReturn = new ArrayList<>();
            for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && totalEvents.get() < maxEventId; i++) {
                if (event != null) {
                    eventsToReturn.add(event);
                }
                totalEvents.getAndIncrement();
            }
            return eventsToReturn;
        }
    }).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt());
    ProcessGroupStatus pgRoot = new ProcessGroupStatus();
    pgRoot.setId("root");
    when(eventAccess.getControllerStatus()).thenReturn(pgRoot);
    // Add child Process Groups.
    // Root -> (A, B -> (B2 -> (B3)))
    final ProcessGroupStatus pgA = new ProcessGroupStatus();
    pgA.setId("pgA");
    final ProcessGroupStatus pgB = new ProcessGroupStatus();
    pgB.setId("pgB");
    final ProcessGroupStatus pgB2 = new ProcessGroupStatus();
    pgB2.setId("pgB2");
    final ProcessGroupStatus pgB3 = new ProcessGroupStatus();
    pgB3.setId("pgB3");
    final Collection<ProcessGroupStatus> childPGs = pgRoot.getProcessGroupStatus();
    childPGs.add(pgA);
    childPGs.add(pgB);
    pgB.getProcessGroupStatus().add(pgB2);
    pgB2.getProcessGroupStatus().add(pgB3);
    // Add Processors.
    final ProcessorStatus prcRoot = new ProcessorStatus();
    prcRoot.setId("1234");
    pgRoot.getProcessorStatus().add(prcRoot);
    final ProcessorStatus prcA = new ProcessorStatus();
    prcA.setId("A001");
    prcA.setName("Processor in PGA");
    pgA.getProcessorStatus().add(prcA);
    final ProcessorStatus prcB = new ProcessorStatus();
    prcB.setId("B001");
    prcB.setName("Processor in PGB");
    pgB.getProcessorStatus().add(prcB);
    final ProcessorStatus prcB2 = new ProcessorStatus();
    prcB2.setId("B201");
    prcB2.setName("Processor in PGB2");
    pgB2.getProcessorStatus().add(prcB2);
    final ProcessorStatus prcB3 = new ProcessorStatus();
    prcB3.setId("B301");
    prcB3.setName("Processor in PGB3");
    pgB3.getProcessorStatus().add(prcB3);
    // Add connection status to test Remote Input/Output Ports
    final ConnectionStatus b2RemoteInputPort = new ConnectionStatus();
    b2RemoteInputPort.setGroupId("pgB2");
    b2RemoteInputPort.setSourceId("B201");
    b2RemoteInputPort.setDestinationId("riB2");
    b2RemoteInputPort.setDestinationName("Remote Input Port name");
    pgB2.getConnectionStatus().add(b2RemoteInputPort);
    final ConnectionStatus b3RemoteOutputPort = new ConnectionStatus();
    b3RemoteOutputPort.setGroupId("pgB3");
    b3RemoteOutputPort.setSourceId("roB3");
    b3RemoteOutputPort.setSourceName("Remote Output Port name");
    b3RemoteOutputPort.setDestinationId("B301");
    pgB3.getConnectionStatus().add(b3RemoteOutputPort);
    final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class);
    Mockito.doAnswer(new Answer<Long>() {

        @Override
        public Long answer(final InvocationOnMock invocation) throws Throwable {
            return maxEventId;
        }
    }).when(provenanceRepository).getMaxEventId();
    when(context.getEventAccess()).thenReturn(eventAccess);
    when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository);
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    when(initContext.getLogger()).thenReturn(logger);
    return task;
}