Examples with PropertyDescriptor org.apache.nifi.components.PropertyDescriptor used on opensource projects

Example 56 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class UpdateRecord method createRecordPaths.

@OnScheduled
public void createRecordPaths(final ProcessContext context) {
    recordPathCache = new RecordPathCache(context.getProperties().size() * 2);
    final List<String> recordPaths = new ArrayList<>(context.getProperties().size() - 2);
    for (final PropertyDescriptor property : context.getProperties().keySet()) {
        if (property.isDynamic()) {
            recordPaths.add(property.getName());
        }
    }
    this.recordPaths = recordPaths;
}

Example 57 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class DistributeLoad method createWeightedList.

@OnScheduled
public void createWeightedList(final ProcessContext context) {
    final Map<Integer, Integer> weightings = new LinkedHashMap<>();
    String distStrat = context.getProperty(DISTRIBUTION_STRATEGY).getValue();
    if (distStrat.equals(STRATEGY_LOAD_DISTRIBUTION_SERVICE)) {
        String hostNamesValue = context.getProperty(HOSTNAMES).getValue();
        String[] hostNames = hostNamesValue.split("(?:,+|;+|\\s+)");
        Set<String> hostNameSet = new HashSet<>();
        for (String hostName : hostNames) {
            if (StringUtils.isNotBlank(hostName)) {
                hostNameSet.add(hostName);
            }
        }
        LoadDistributionService svc = context.getProperty(LOAD_DISTRIBUTION_SERVICE_TEMPLATE).asControllerService(LoadDistributionService.class);
        myListener = new LoadDistributionListener() {

            @Override
            public void update(Map<String, Integer> loadInfo) {
                for (Relationship rel : relationshipsRef.get()) {
                    String hostname = rel.getDescription();
                    Integer weight = 1;
                    if (loadInfo.containsKey(hostname)) {
                        weight = loadInfo.get(hostname);
                    }
                    weightings.put(Integer.decode(rel.getName()), weight);
                }
                updateWeightedRelationships(weightings);
            }
        };
        Map<String, Integer> loadInfo = svc.getLoadDistribution(hostNameSet, myListener);
        for (Relationship rel : relationshipsRef.get()) {
            String hostname = rel.getDescription();
            Integer weight = 1;
            if (loadInfo.containsKey(hostname)) {
                weight = loadInfo.get(hostname);
            }
            weightings.put(Integer.decode(rel.getName()), weight);
        }
    } else {
        final int numRelationships = context.getProperty(NUM_RELATIONSHIPS).asInteger();
        for (int i = 1; i <= numRelationships; i++) {
            weightings.put(i, 1);
        }
        for (final PropertyDescriptor propDesc : context.getProperties().keySet()) {
            if (!this.properties.contains(propDesc)) {
                final int relationship = Integer.parseInt(propDesc.getName());
                final int weighting = context.getProperty(propDesc).asInteger();
                weightings.put(relationship, weighting);
            }
        }
    }
    updateWeightedRelationships(weightings);
}

Example 58 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class EvaluateJsonPath method customValidate.

@Override
protected Collection<ValidationResult> customValidate(final ValidationContext context) {
    final List<ValidationResult> results = new ArrayList<>(super.customValidate(context));
    final String destination = context.getProperty(DESTINATION).getValue();
    if (DESTINATION_CONTENT.equals(destination)) {
        int jsonPathCount = 0;
        for (final PropertyDescriptor desc : context.getProperties().keySet()) {
            if (desc.isDynamic()) {
                jsonPathCount++;
            }
        }
        if (jsonPathCount != 1) {
            results.add(new ValidationResult.Builder().subject("JsonPaths").valid(false).explanation("Exactly one JsonPath must be set if using destination of " + DESTINATION_CONTENT).build());
        }
    }
    return results;
}

Example 59 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class ExecuteProcess method launchProcess.

protected Future<?> launchProcess(final ProcessContext context, final List<String> commandStrings, final Long batchNanos, final ProxyOutputStream proxyOut) throws IOException {
    final Boolean redirectErrorStream = context.getProperty(REDIRECT_ERROR_STREAM).asBoolean();
    final ProcessBuilder builder = new ProcessBuilder(commandStrings);
    final String workingDirName = context.getProperty(WORKING_DIR).evaluateAttributeExpressions().getValue();
    if (workingDirName != null) {
        builder.directory(new File(workingDirName));
    }
    final Map<String, String> environment = new HashMap<>();
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        if (entry.getKey().isDynamic()) {
            environment.put(entry.getKey().getName(), entry.getValue());
        }
    }
    if (!environment.isEmpty()) {
        builder.environment().putAll(environment);
    }
    getLogger().info("Start creating new Process > {} ", new Object[] { commandStrings });
    this.externalProcess = builder.redirectErrorStream(redirectErrorStream).start();
    // Submit task to read error stream from process
    if (!redirectErrorStream) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try (final BufferedReader reader = new BufferedReader(new InputStreamReader(externalProcess.getErrorStream()))) {
                    reader.lines().filter(line -> line != null && line.length() > 0).forEach(getLogger()::warn);
                } catch (final IOException ioe) {
                }
            }
        });
    }
    // Submit task to read output of Process and write to FlowFile.
    failure = new AtomicBoolean(false);
    final Future<?> future = executor.submit(new Callable<Object>() {

        @Override
        public Object call() throws IOException {
            try {
                if (batchNanos == null) {
                    // process to the flowfile.
                    try (final BufferedInputStream bufferedIn = new BufferedInputStream(externalProcess.getInputStream())) {
                        final byte[] buffer = new byte[4096];
                        int len;
                        while ((len = bufferedIn.read(buffer)) > 0) {
                            // anymore anyway, we don't care?
                            if (!isScheduled()) {
                                return null;
                            }
                            proxyOut.write(buffer, 0, len);
                        }
                    }
                } else {
                    // to read lines of text and write them as lines of text.
                    try (final BufferedReader reader = new BufferedReader(new InputStreamReader(externalProcess.getInputStream()))) {
                        String line;
                        while ((line = reader.readLine()) != null) {
                            if (!isScheduled()) {
                                return null;
                            }
                            proxyOut.write((line + "\n").getBytes(StandardCharsets.UTF_8));
                        }
                    }
                }
            } catch (final IOException ioe) {
                failure.set(true);
                throw ioe;
            } finally {
                try {
                    // Since we are going to exit anyway, one sec gives it an extra chance to exit gracefully.
                    // In the future consider exposing it via configuration.
                    boolean terminated = externalProcess.waitFor(1000, TimeUnit.MILLISECONDS);
                    int exitCode = terminated ? externalProcess.exitValue() : -9999;
                    getLogger().info("Process finished with exit code {} ", new Object[] { exitCode });
                } catch (InterruptedException e1) {
                    Thread.currentThread().interrupt();
                }
            }
            return null;
        }
    });
    return future;
}

Example 60 with PropertyDescriptor

use of org.apache.nifi.components.PropertyDescriptor in project nifi by apache.

the class ExecuteStreamCommand method onTrigger.

@Override
public void onTrigger(ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile inputFlowFile = session.get();
    if (null == inputFlowFile) {
        return;
    }
    final ArrayList<String> args = new ArrayList<>();
    final boolean putToAttribute = context.getProperty(PUT_OUTPUT_IN_ATTRIBUTE).isSet();
    final Integer attributeSize = context.getProperty(PUT_ATTRIBUTE_MAX_LENGTH).asInteger();
    final String attributeName = context.getProperty(PUT_OUTPUT_IN_ATTRIBUTE).getValue();
    final String executeCommand = context.getProperty(EXECUTION_COMMAND).evaluateAttributeExpressions(inputFlowFile).getValue();
    args.add(executeCommand);
    final String commandArguments = context.getProperty(EXECUTION_ARGUMENTS).evaluateAttributeExpressions(inputFlowFile).getValue();
    final boolean ignoreStdin = Boolean.parseBoolean(context.getProperty(IGNORE_STDIN).getValue());
    if (!StringUtils.isBlank(commandArguments)) {
        for (String arg : ArgumentUtils.splitArgs(commandArguments, context.getProperty(ARG_DELIMITER).getValue().charAt(0))) {
            args.add(arg);
        }
    }
    final String workingDir = context.getProperty(WORKING_DIR).evaluateAttributeExpressions(inputFlowFile).getValue();
    final ProcessBuilder builder = new ProcessBuilder();
    logger.debug("Executing and waiting for command {} with arguments {}", new Object[] { executeCommand, commandArguments });
    File dir = null;
    if (!StringUtils.isBlank(workingDir)) {
        dir = new File(workingDir);
        if (!dir.exists() && !dir.mkdirs()) {
            logger.warn("Failed to create working directory {}, using current working directory {}", new Object[] { workingDir, System.getProperty("user.dir") });
        }
    }
    final Map<String, String> environment = new HashMap<>();
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        if (entry.getKey().isDynamic()) {
            environment.put(entry.getKey().getName(), entry.getValue());
        }
    }
    builder.environment().putAll(environment);
    builder.command(args);
    builder.directory(dir);
    builder.redirectInput(Redirect.PIPE);
    builder.redirectOutput(Redirect.PIPE);
    final Process process;
    try {
        process = builder.start();
    } catch (IOException e) {
        logger.error("Could not create external process to run command", e);
        throw new ProcessException(e);
    }
    try (final OutputStream pos = process.getOutputStream();
        final InputStream pis = process.getInputStream();
        final InputStream pes = process.getErrorStream();
        final BufferedInputStream bis = new BufferedInputStream(pis);
        final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(pes))) {
        int exitCode = -1;
        final BufferedOutputStream bos = new BufferedOutputStream(pos);
        FlowFile outputFlowFile = putToAttribute ? inputFlowFile : session.create(inputFlowFile);
        ProcessStreamWriterCallback callback = new ProcessStreamWriterCallback(ignoreStdin, bos, bis, logger, attributeName, session, outputFlowFile, process, putToAttribute, attributeSize);
        session.read(inputFlowFile, callback);
        outputFlowFile = callback.outputFlowFile;
        if (putToAttribute) {
            outputFlowFile = session.putAttribute(outputFlowFile, attributeName, new String(callback.outputBuffer, 0, callback.size));
        }
        exitCode = callback.exitCode;
        logger.debug("Execution complete for command: {}.  Exited with code: {}", new Object[] { executeCommand, exitCode });
        Map<String, String> attributes = new HashMap<>();
        final StringBuilder strBldr = new StringBuilder();
        try {
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                strBldr.append(line).append("\n");
            }
        } catch (IOException e) {
            strBldr.append("Unknown...could not read Process's Std Error");
        }
        int length = strBldr.length() > 4000 ? 4000 : strBldr.length();
        attributes.put("execution.error", strBldr.substring(0, length));
        final Relationship outputFlowFileRelationship = putToAttribute ? ORIGINAL_RELATIONSHIP : (exitCode != 0) ? NONZERO_STATUS_RELATIONSHIP : OUTPUT_STREAM_RELATIONSHIP;
        if (exitCode == 0) {
            logger.info("Transferring flow file {} to {}", new Object[] { outputFlowFile, outputFlowFileRelationship.getName() });
        } else {
            logger.error("Transferring flow file {} to {}. Executable command {} ended in an error: {}", new Object[] { outputFlowFile, outputFlowFileRelationship.getName(), executeCommand, strBldr.toString() });
        }
        attributes.put("execution.status", Integer.toString(exitCode));
        attributes.put("execution.command", executeCommand);
        attributes.put("execution.command.args", commandArguments);
        outputFlowFile = session.putAllAttributes(outputFlowFile, attributes);
        if (NONZERO_STATUS_RELATIONSHIP.equals(outputFlowFileRelationship)) {
            outputFlowFile = session.penalize(outputFlowFile);
        }
        // This will transfer the FlowFile that received the stream output to its destined relationship.
        // In the event the stream is put to the an attribute of the original, it will be transferred here.
        session.transfer(outputFlowFile, outputFlowFileRelationship);
        if (!putToAttribute) {
            logger.info("Transferring flow file {} to original", new Object[] { inputFlowFile });
            inputFlowFile = session.putAllAttributes(inputFlowFile, attributes);
            session.transfer(inputFlowFile, ORIGINAL_RELATIONSHIP);
        }
    } catch (final IOException ex) {
        // could not close Process related streams
        logger.warn("Problem terminating Process {}", new Object[] { process }, ex);
    } finally {
        // last ditch effort to clean up that process.
        process.destroy();
    }
}