Examples with InvalidMacroException io.cdap.cdap.api.macro.InvalidMacroException used on opensource projects

Example 6 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by cdapio.

the class ProvisioningService method createDeprovisionTask.

private Runnable createDeprovisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner, Consumer<ProgramRunId> taskCleanup) {
    Map<String, String> properties = taskInfo.getProvisionerProperties();
    ProvisionerContext context;
    SSHKeyPair sshKeyPair = null;
    try {
        sshKeyPair = createSSHKeyPair(taskInfo);
    } catch (IOException e) {
        LOG.warn("Failed to load ssh key. No SSH key will be available for the deprovision task", e);
    }
    ProgramRunId programRunId = taskInfo.getProgramRunId();
    Map<String, String> systemArgs = taskInfo.getProgramOptions().getArguments().asMap();
    try {
        SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, sshKeyPair);
        context = createContext(cConf, taskInfo.getProgramOptions(), programRunId, taskInfo.getUser(), properties, sshContext);
    } catch (InvalidMacroException e) {
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while deprovisoning. " + "The cluster will be marked as orphaned.", e));
        provisionerNotifier.orphaned(programRunId);
        return () -> {
        };
    }
    DeprovisionTask task = new DeprovisionTask(taskInfo, transactionRunner, 300, provisioner, context, provisionerNotifier, locationFactory);
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.DEPROVISION);
    return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
        try {
            long delay = task.executeOnce();
            if (delay < 0) {
                taskCleanup.accept(programRunId);
            }
            return delay;
        } catch (InterruptedException e) {
            // We can get interrupted if the task is cancelled or CDAP is stopped. In either case, just return.
            // If it was cancelled, state cleanup is left to the caller. If it was CDAP master stopping, the task
            // will be resumed on master startup
            LOG.debug("Deprovision task for program run {} interrupted.", programRunId);
            throw e;
        } catch (Exception e) {
            // Otherwise, if there was an error deprovisioning, run the cleanup
            LOG.info("Deprovision task for program run {} failed.", programRunId, e);
            taskCleanup.accept(programRunId);
            throw e;
        }
    }));
}

Example 7 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class MacroParser method getMacroFunctionMetadata.

/**
 * Use macro function to evaluate the macro string
 *
 * @param startIndex the start index of the macro string "$"
 * @param endIndex the end index of the macro string
 * @param macroStr the macro string to evaluate, without bracelet
 * @param argsStartIndex the index of start parenthesis
 * @param originalString the original string
 */
private MacroMetadata getMacroFunctionMetadata(int startIndex, int endIndex, String macroStr, int argsStartIndex, String originalString) {
    // check for closing ")" and allow escaping "\)" and doubly-escaping "\\)"
    int closingParenIndex = getFirstUnescapedTokenIndex(')', macroStr, 0);
    // closing index will always be < 0.
    if ((!lookupsEnabled || skipInvalid) && closingParenIndex < 0) {
        // get ")" index using original string starting from end index
        int originalParenIndex = getFirstUnescapedTokenIndex(')', originalString, endIndex);
        // if found valid one, get the new macro string without "${" and set the correct closing ")" index
        if (originalParenIndex > endIndex) {
            // update end index to be next character after ")"
            endIndex = originalParenIndex + 1;
            // macro string should skip the first 2 characters "${"
            macroStr = originalString.substring(startIndex + 2, endIndex);
            closingParenIndex = getFirstUnescapedTokenIndex(')', macroStr, 0);
            // if this macro string contains unevaluated macros, there is no point to continue calling the macro function
            if (getStartIndex(macroStr, macroStr.length()) >= 0) {
                return new MacroMetadata(String.format("${%s}", macroStr), startIndex, endIndex, false);
            }
        }
    }
    if (closingParenIndex < 0 || !macroStr.endsWith(")")) {
        throw new InvalidMacroException(String.format("Could not find enclosing ')' for macro arguments in '%s'.", macroStr));
    } else if (closingParenIndex != macroStr.length() - 1) {
        throw new InvalidMacroException(String.format("Macro arguments in '%s' have extra invalid trailing ')'.", macroStr));
    }
    // arguments and macroFunction are expected to have escapes replaced when being evaluated
    String arguments = replaceEscapedSyntax(macroStr.substring(argsStartIndex + 1, macroStr.length() - 1));
    String macroFunction = replaceEscapedSyntax(macroStr.substring(0, argsStartIndex));
    String[] args = Iterables.toArray(Splitter.on(ARGUMENT_DELIMITER).split(arguments), String.class);
    // if the whitelist is empty, that means no whitelist was set, so every function should be evaluated.
    if (functionsEnabled && (functionWhitelist.isEmpty() || functionWhitelist.contains(macroFunction))) {
        try {
            switch(macroEvaluator.evaluateAs(macroFunction)) {
                case STRING:
                    return new MacroMetadata(macroEvaluator.evaluate(macroFunction, args), startIndex, endIndex, true);
                case MAP:
                    // evaluate the macro as map, and evaluate this map
                    Map<String, String> properties = macroEvaluator.evaluateMap(macroFunction, args);
                    Map<String, String> evaluated = new HashMap<>();
                    properties.forEach((key, val) -> {
                        evaluated.put(key, parse(val));
                    });
                    return new MacroMetadata(GSON.toJson(evaluated), startIndex, endIndex, true);
                default:
                    // should not happen
                    throw new IllegalStateException("Unsupported macro object type, the supported types are string and map.");
            }
        } catch (InvalidMacroException e) {
            if (!skipInvalid) {
                throw e;
            }
        }
    }
    // and turn off recursive evaluation to prevent it from getting evaluated in an infinite loop
    return new MacroMetadata(String.format("${%s}", macroStr), startIndex, endIndex, false);
}

Example 8 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ConnectionMacroEvaluator method evaluateMacroMap.

/**
 * Evaluates the connection macro function by calling the Connection service to retrieve the connection information.
 *
 * @param args should contains exactly one arguments. The argument should contain the connection name
 * @return the json representation of the properties of the connection
 */
@Override
Map<String, String> evaluateMacroMap(String macroFunction, String... args) throws InvalidMacroException, IOException, RetryableException {
    if (args.length != 1) {
        throw new InvalidMacroException("Macro '" + FUNCTION_NAME + "' should have exactly 1 arguments");
    }
    // only encode the connection name here since / will get encoded to %2f and some router cannot recognize it
    // we don't need to worry about space getting converted to plus here since connection lookup is based on id,
    // space and plus both get converted to _ in the id
    String connName = URLEncoder.encode(args[0], StandardCharsets.UTF_8.name());
    HttpURLConnection urlConn = serviceDiscoverer.openConnection(NamespaceId.SYSTEM.getNamespace(), Constants.PIPELINEID, Constants.STUDIO_SERVICE_NAME, String.format("v1/contexts/%s/connections/%s", namespace, connName));
    Connection connection = gson.fromJson(validateAndRetrieveContent(SERVICE_NAME, urlConn), Connection.class);
    return connection.getPlugin().getProperties();
}

Example 9 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ProvisioningService method getClusterStatus.

/**
 * Returns the {@link ClusterStatus} for the cluster being used to execute the given program run.
 *
 * @param programRunId the program run id for checking the cluster status
 * @param programOptions the program options for the given run
 * @param cluster the {@link Cluster} information for the given run
 * @param userId the user id to use for {@link SecureStore} operation.
 * @return the {@link ClusterStatus}
 * @throws Exception if non-retryable exception is encountered when querying cluster status
 */
public ClusterStatus getClusterStatus(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster, String userId) throws Exception {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // If there is no provisioner available, we can't do anything further, hence returning NOT_EXISTS
    if (provisioner == null) {
        return ClusterStatus.NOT_EXISTS;
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    // Create the ProvisionerContext and query the cluster status using the provisioner
    ProvisionerContext context;
    try {
        DefaultSSHContext defaultSSHContext = null;
        if (!getRuntimeJobManager(programRunId, programOptions).isPresent()) {
            defaultSSHContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, null);
        }
        context = createContext(cConf, programOptions, programRunId, userId, properties, defaultSSHContext);
    } catch (InvalidMacroException e) {
        // This shouldn't happen
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while checking cluster status.", e));
        return ClusterStatus.NOT_EXISTS;
    }
    return Retries.callWithRetries(() -> provisioner.getClusterStatus(context, cluster), RetryStrategies.exponentialDelay(1, 5, TimeUnit.SECONDS), RetryableProvisionException.class::isInstance);
}

Example 10 with InvalidMacroException

use of io.cdap.cdap.api.macro.InvalidMacroException in project cdap by caskdata.

the class ProvisioningService method createDeprovisionTask.

private Runnable createDeprovisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner, Consumer<ProgramRunId> taskCleanup) {
    Map<String, String> properties = taskInfo.getProvisionerProperties();
    ProvisionerContext context;
    SSHKeyPair sshKeyPair = null;
    try {
        sshKeyPair = createSSHKeyPair(taskInfo);
    } catch (IOException e) {
        LOG.warn("Failed to load ssh key. No SSH key will be available for the deprovision task", e);
    }
    ProgramRunId programRunId = taskInfo.getProgramRunId();
    Map<String, String> systemArgs = taskInfo.getProgramOptions().getArguments().asMap();
    try {
        SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, sshKeyPair);
        context = createContext(cConf, taskInfo.getProgramOptions(), programRunId, taskInfo.getUser(), properties, sshContext);
    } catch (InvalidMacroException e) {
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while deprovisoning. " + "The cluster will be marked as orphaned.", e));
        provisionerNotifier.orphaned(programRunId);
        return () -> {
        };
    }
    DeprovisionTask task = new DeprovisionTask(taskInfo, transactionRunner, 300, provisioner, context, provisionerNotifier, locationFactory);
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.DEPROVISION);
    return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
        try {
            long delay = task.executeOnce();
            if (delay < 0) {
                taskCleanup.accept(programRunId);
            }
            return delay;
        } catch (InterruptedException e) {
            // We can get interrupted if the task is cancelled or CDAP is stopped. In either case, just return.
            // If it was cancelled, state cleanup is left to the caller. If it was CDAP master stopping, the task
            // will be resumed on master startup
            LOG.debug("Deprovision task for program run {} interrupted.", programRunId);
            throw e;
        } catch (Exception e) {
            // Otherwise, if there was an error deprovisioning, run the cleanup
            LOG.info("Deprovision task for program run {} failed.", programRunId, e);
            taskCleanup.accept(programRunId);
            throw e;
        }
    }));
}