Search in sources :

Example 1 with SparkApplicationCommandsBuilder

use of com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder in project kylo by Teradata.

the class ExecuteSparkApps method getMainArgs.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.nifi.v2.spark.ExecuteSparkJob#getMainArgs(org.apache.nifi.processor.ProcessContext, org.apache.nifi.flowfile.FlowFile)
     */
@Override
protected String[] getMainArgs(ProcessContext context, FlowFile flowFile) {
    SparkApplicationCommandsBuilder listBldr = new SparkApplicationCommandsBuilder();
    String namesCsv = context.getProperty(this.appNamesPropDescriptor).evaluateAttributeExpressions(flowFile).getValue();
    for (String appName : parseAppNames(namesCsv)) {
        AppCommand cmd = this.appCommands.get(appName);
        SparkCommandBuilder cmdBldr = listBldr.application(cmd.name);
        PropertyValue classProp = context.newPropertyValue(cmd.appClass);
        cmdBldr.className(classProp.evaluateAttributeExpressions(flowFile).getValue());
        for (Entry<NamedArgument, String> entry : cmd.namedArgs.entrySet()) {
            PropertyValue valueProp = context.newPropertyValue(entry.getValue());
            cmdBldr.addArgument(entry.getKey().argName, valueProp.evaluateAttributeExpressions(flowFile).getValue());
        }
        cmd.positionalArgs.entrySet().stream().sorted((e1, e2) -> Integer.compare(e1.getKey().position, e2.getKey().position)).map(entry -> context.newPropertyValue(entry.getValue())).forEach(prop -> cmdBldr.addArgument(prop.evaluateAttributeExpressions(flowFile).getValue()));
        cmdBldr.add();
    }
    List<SparkApplicationCommand> commands = listBldr.build();
    String[] args = MultiSparkExecArguments.createCommandLine(commands);
    getLog().info("Spark main args: {} {}", args);
    return args;
}
Also used : StandardValidators(org.apache.nifi.processor.util.StandardValidators) Arrays(java.util.Arrays) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) ValidationContext(org.apache.nifi.components.ValidationContext) EventDriven(org.apache.nifi.annotation.behavior.EventDriven) StringUtils(org.apache.commons.lang3.StringUtils) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) PropertyValue(org.apache.nifi.components.PropertyValue) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Matcher(java.util.regex.Matcher) Relationship(org.apache.nifi.processor.Relationship) Map(java.util.Map) SparkCommandBuilder(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder.SparkCommandBuilder) DynamicRelationship(org.apache.nifi.annotation.behavior.DynamicRelationship) ValidationResult(org.apache.nifi.components.ValidationResult) LinkedHashSet(java.util.LinkedHashSet) Validator(org.apache.nifi.components.Validator) FlowFile(org.apache.nifi.flowfile.FlowFile) SparkApplicationCommand(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommand) ProcessContext(org.apache.nifi.processor.ProcessContext) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MultiSparkExecArguments(com.thinkbiganalytics.spark.multiexec.MultiSparkExecArguments) Set(java.util.Set) Builder(org.apache.nifi.components.PropertyDescriptor.Builder) Collectors(java.util.stream.Collectors) Objects(jersey.repackaged.com.google.common.base.Objects) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) List(java.util.List) DynamicProperty(org.apache.nifi.annotation.behavior.DynamicProperty) Stream(java.util.stream.Stream) Entry(java.util.Map.Entry) Optional(java.util.Optional) Tags(org.apache.nifi.annotation.documentation.Tags) Pattern(java.util.regex.Pattern) SparkApplicationCommandsBuilder(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder) Comparator(java.util.Comparator) ProcessorInitializationContext(org.apache.nifi.processor.ProcessorInitializationContext) Collections(java.util.Collections) SparkApplicationCommandsBuilder(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder) SparkApplicationCommand(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommand) SparkCommandBuilder(com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder.SparkCommandBuilder) PropertyValue(org.apache.nifi.components.PropertyValue)

Aggregations

MultiSparkExecArguments (com.thinkbiganalytics.spark.multiexec.MultiSparkExecArguments)1 SparkApplicationCommand (com.thinkbiganalytics.spark.multiexec.SparkApplicationCommand)1 SparkApplicationCommandsBuilder (com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder)1 SparkCommandBuilder (com.thinkbiganalytics.spark.multiexec.SparkApplicationCommandsBuilder.SparkCommandBuilder)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 Comparator (java.util.Comparator)1 HashSet (java.util.HashSet)1 LinkedHashSet (java.util.LinkedHashSet)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Optional (java.util.Optional)1 Set (java.util.Set)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 Supplier (java.util.function.Supplier)1 Matcher (java.util.regex.Matcher)1 Pattern (java.util.regex.Pattern)1