Examples with InvalidMacroException io.cdap.cdap.api.macro.InvalidMacroException used on opensource projects

Example 1 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ProvisioningService method createProvisionTask.

private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner) {
    ProgramRunId programRunId = taskInfo.getProgramRunId();
    ProgramOptions programOptions = taskInfo.getProgramOptions();
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    ProvisionerContext context;
    try {
        SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), locationFactory.create(taskInfo.getSecureKeysDir()), createSSHKeyPair(taskInfo));
        context = createContext(cConf, programOptions, programRunId, taskInfo.getUser(), taskInfo.getProvisionerProperties(), sshContext);
    } catch (IOException e) {
        runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Failed to load ssh key. The run will be marked as failed.", e));
        programStateWriter.error(programRunId, new IllegalStateException("Failed to load ssh key.", e));
        provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
        return () -> {
        };
    } catch (InvalidMacroException e) {
        runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Could not evaluate macros while provisoning. " + "The run will be marked as failed.", e));
        programStateWriter.error(programRunId, new IllegalStateException("Could not evaluate macros while provisioning", e));
        provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
        return () -> {
        };
    }
    // TODO: (CDAP-13246) pick up timeout from profile instead of hardcoding
    ProvisioningTask task = new ProvisionTask(taskInfo, transactionRunner, provisioner, context, provisionerNotifier, programStateWriter, 300);
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.PROVISION);
    return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
        try {
            return task.executeOnce();
        } catch (InterruptedException e) {
            LOG.debug("Provision task for program run {} interrupted.", taskInfo.getProgramRunId());
            throw e;
        } catch (Exception e) {
            LOG.info("Provision task for program run {} failed.", taskInfo.getProgramRunId(), e);
            throw e;
        }
    }));
}

Example 2 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class MacroParser method findRightmostMacro.

/**
 * Find the rightmost macro in the specified string. If no macro is found, returns null.
 * The search will start at the specified position, then move left through the string.
 * Macros to the right of the specified position will be ignored.
 *
 * @param str the string to find a macro in
 * @param pos the position in the string to begin search for a macro.
 * @return the rightmost macro and its position in the string
 * @throws InvalidMacroException if invalid macro syntax was found.
 */
@Nullable
private MacroMetadata findRightmostMacro(String str, int pos) throws InvalidMacroException {
    // find opening "${" skipping escaped syntax "\${" and allowing doubly-escaping "\\${"
    int startIndex = getStartIndex(str, pos);
    if (startIndex < 0) {
        return null;
    }
    // found "${", now look for enclosing "}" and allow escaping "\}" and doubly-escaping "\\}"
    int endIndex = getFirstUnescapedTokenIndex('}', str, startIndex);
    if (endIndex < 0) {
        throw new InvalidMacroException(String.format("Could not find enclosing '}' for macro '%s'.", str.substring(startIndex)));
    } else if (endIndex == startIndex + 2) {
        throw new InvalidMacroException("Macro expression cannot be empty");
    }
    // macroStr = 'macroFunction(macroArguments)' or just 'property' for ${property}
    String macroStr = str.substring(startIndex + 2, endIndex).trim();
    // look for "(", which indicates there are arguments and allow escaping "\(" and doubly-escaping "\\("
    int argsStartIndex = getFirstUnescapedTokenIndex('(', macroStr, 0);
    // determine whether to use a macro function or a property lookup
    if (argsStartIndex >= 0) {
        return getMacroFunctionMetadata(startIndex, endIndex, macroStr, argsStartIndex, str);
    } else {
        macroStr = replaceEscapedSyntax(macroStr);
        if (lookupsEnabled) {
            try {
                return new MacroMetadata(macroEvaluator.lookup(macroStr), startIndex, endIndex, true);
            } catch (InvalidMacroException e) {
                if (!skipInvalid) {
                    throw e;
                }
            }
        }
        // and turn off recursive evaluation to prevent it from getting evaluated in an infinite loop
        return new MacroMetadata(String.format("${%s}", macroStr), startIndex, endIndex, false);
    }
}

Example 3 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class MacroParser method getMacroFunctionMetadata.

/**
 * Use macro function to evaluate the macro string
 *
 * @param startIndex the start index of the macro string "$"
 * @param endIndex the end index of the macro string
 * @param macroStr the macro string to evaluate, without bracelet
 * @param argsStartIndex the index of start parenthesis
 * @param originalString the original string
 */
private MacroMetadata getMacroFunctionMetadata(int startIndex, int endIndex, String macroStr, int argsStartIndex, String originalString) {
    // check for closing ")" and allow escaping "\)" and doubly-escaping "\\)"
    int closingParenIndex = getFirstUnescapedTokenIndex(')', macroStr, 0);
    // closing index will always be < 0.
    if ((!lookupsEnabled || skipInvalid) && closingParenIndex < 0) {
        // get ")" index using original string starting from end index
        int originalParenIndex = getFirstUnescapedTokenIndex(')', originalString, endIndex);
        // if found valid one, get the new macro string without "${" and set the correct closing ")" index
        if (originalParenIndex > endIndex) {
            // update end index to be next character after ")"
            endIndex = originalParenIndex + 1;
            // macro string should skip the first 2 characters "${"
            macroStr = originalString.substring(startIndex + 2, endIndex);
            closingParenIndex = getFirstUnescapedTokenIndex(')', macroStr, 0);
            // if this macro string contains unevaluated macros, there is no point to continue calling the macro function
            if (getStartIndex(macroStr, macroStr.length()) >= 0) {
                return new MacroMetadata(String.format("${%s}", macroStr), startIndex, endIndex, false);
            }
        }
    }
    if (closingParenIndex < 0 || !macroStr.endsWith(")")) {
        throw new InvalidMacroException(String.format("Could not find enclosing ')' for macro arguments in '%s'.", macroStr));
    } else if (closingParenIndex != macroStr.length() - 1) {
        throw new InvalidMacroException(String.format("Macro arguments in '%s' have extra invalid trailing ')'.", macroStr));
    }
    // arguments and macroFunction are expected to have escapes replaced when being evaluated
    String arguments = replaceEscapedSyntax(macroStr.substring(argsStartIndex + 1, macroStr.length() - 1));
    String macroFunction = replaceEscapedSyntax(macroStr.substring(0, argsStartIndex));
    String[] args = Iterables.toArray(Splitter.on(ARGUMENT_DELIMITER).split(arguments), String.class);
    // if the whitelist is empty, that means no whitelist was set, so every function should be evaluated.
    if (functionsEnabled && (functionWhitelist.isEmpty() || functionWhitelist.contains(macroFunction))) {
        try {
            switch(macroEvaluator.evaluateAs(macroFunction)) {
                case STRING:
                    return new MacroMetadata(macroEvaluator.evaluate(macroFunction, args), startIndex, endIndex, true);
                case MAP:
                    // evaluate the macro as map, and evaluate this map
                    Map<String, String> properties = macroEvaluator.evaluateMap(macroFunction, args);
                    Map<String, String> evaluated = new HashMap<>();
                    properties.forEach((key, val) -> {
                        evaluated.put(key, parse(val));
                    });
                    return new MacroMetadata(GSON.toJson(evaluated), startIndex, endIndex, true);
                default:
                    // should not happen
                    throw new IllegalStateException("Unsupported macro object type, the supported types are string and map.");
            }
        } catch (InvalidMacroException e) {
            if (!skipInvalid) {
                throw e;
            }
        }
    }
    // and turn off recursive evaluation to prevent it from getting evaluated in an infinite loop
    return new MacroMetadata(String.format("${%s}", macroStr), startIndex, endIndex, false);
}

Example 4 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ConnectionMacroEvaluator method evaluateMacroMap.

/**
 * Evaluates the connection macro function by calling the Connection service to retrieve the connection information.
 *
 * @param args should contains exactly one arguments. The argument should contain the connection name
 * @return the json representation of the properties of the connection
 */
@Override
Map<String, String> evaluateMacroMap(String macroFunction, String... args) throws InvalidMacroException, IOException, RetryableException {
    if (args.length != 1) {
        throw new InvalidMacroException("Macro '" + FUNCTION_NAME + "' should have exactly 1 arguments");
    }
    // only encode the connection name here since / will get encoded to %2f and some router cannot recognize it
    // we don't need to worry about space getting converted to plus here since connection lookup is based on id,
    // space and plus both get converted to _ in the id
    String connName = URLEncoder.encode(args[0], StandardCharsets.UTF_8.name());
    HttpURLConnection urlConn = serviceDiscoverer.openConnection(NamespaceId.SYSTEM.getNamespace(), Constants.PIPELINEID, Constants.STUDIO_SERVICE_NAME, String.format("v1/contexts/%s/connections/%s", namespace, connName));
    Connection connection = gson.fromJson(validateAndRetrieveContent(SERVICE_NAME, urlConn), Connection.class);
    return connection.getPlugin().getProperties();
}

Example 5 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ProvisioningService method getClusterStatus.

/**
 * Returns the {@link ClusterStatus} for the cluster being used to execute the given program run.
 *
 * @param programRunId the program run id for checking the cluster status
 * @param programOptions the program options for the given run
 * @param cluster the {@link Cluster} information for the given run
 * @param userId the user id to use for {@link SecureStore} operation.
 * @return the {@link ClusterStatus}
 * @throws Exception if non-retryable exception is encountered when querying cluster status
 */
public ClusterStatus getClusterStatus(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster, String userId) throws Exception {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // If there is no provisioner available, we can't do anything further, hence returning NOT_EXISTS
    if (provisioner == null) {
        return ClusterStatus.NOT_EXISTS;
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    // Create the ProvisionerContext and query the cluster status using the provisioner
    ProvisionerContext context;
    try {
        DefaultSSHContext defaultSSHContext = null;
        if (!getRuntimeJobManager(programRunId, programOptions).isPresent()) {
            defaultSSHContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, null);
        }
        context = createContext(cConf, programOptions, programRunId, userId, properties, defaultSSHContext);
    } catch (InvalidMacroException e) {
        // This shouldn't happen
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while checking cluster status.", e));
        return ClusterStatus.NOT_EXISTS;
    }
    return Retries.callWithRetries(() -> provisioner.getClusterStatus(context, cluster), RetryStrategies.exponentialDelay(1, 5, TimeUnit.SECONDS), RetryableProvisionException.class::isInstance);
}