Search in sources :

Example 1 with SecureStoreMacroEvaluator

use of io.cdap.cdap.etl.common.SecureStoreMacroEvaluator in project cdap by caskdata.

the class ValidationHandler method validateLocally.

private void validateLocally(HttpServiceRequest request, HttpServiceResponder responder, String namespace) throws IOException {
    StageValidationRequest validationRequest;
    try {
        validationRequest = GSON.fromJson(StandardCharsets.UTF_8.decode(request.getContent()).toString(), StageValidationRequest.class);
        validationRequest.validate();
    } catch (JsonSyntaxException e) {
        responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Unable to decode request body: " + e.getMessage());
        return;
    } catch (IllegalArgumentException e) {
        responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Invalid stage config: " + e.getMessage());
        return;
    }
    Map<String, String> arguments = Collections.emptyMap();
    // this option.
    if (validationRequest.getResolveMacrosFromPreferences()) {
        try {
            arguments = getContext().getPreferencesForNamespace(namespace, true);
        } catch (IllegalArgumentException iae) {
            // If this method returns IllegalArgumentException, it means the namespace doesn't exist.
            // If this is the case, we return a 404 error.
            responder.sendError(HttpURLConnection.HTTP_NOT_FOUND, String.format("Namespace '%s' does not exist", namespace));
            return;
        }
    }
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, getContext()), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(getContext()), ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, getContext()));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, DefaultMacroEvaluator.MAP_FUNCTIONS);
    MacroParserOptions macroParserOptions = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    Function<Map<String, String>, Map<String, String>> macroFn = macroProperties -> getContext().evaluateMacros(namespace, macroProperties, macroEvaluator, macroParserOptions);
    String validationResponse = GSON.toJson(ValidationUtils.validate(namespace, validationRequest, getContext().createServicePluginConfigurer(namespace), macroFn, getContext()));
    responder.sendString(validationResponse);
}
Also used : BatchSource(io.cdap.cdap.etl.api.batch.BatchSource) HttpURLConnection(java.net.HttpURLConnection) JsonObject(com.google.gson.JsonObject) RemoteExecutionException(io.cdap.cdap.api.service.worker.RemoteExecutionException) PathParam(javax.ws.rs.PathParam) TypeToken(com.google.gson.reflect.TypeToken) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) SchemaTypeAdapter(io.cdap.cdap.internal.io.SchemaTypeAdapter) GET(javax.ws.rs.GET) AccessException(io.cdap.cdap.api.security.AccessException) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) Bytes(io.cdap.cdap.api.common.Bytes) Function(java.util.function.Function) GsonBuilder(com.google.gson.GsonBuilder) RunnableTaskRequest(io.cdap.cdap.api.service.worker.RunnableTaskRequest) Gson(com.google.gson.Gson) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) PluginSpec(io.cdap.cdap.etl.proto.v2.spec.PluginSpec) Map(java.util.Map) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) RemoteTaskException(io.cdap.cdap.api.service.worker.RemoteTaskException) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) PipelineSpec(io.cdap.cdap.etl.proto.v2.spec.PipelineSpec) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) ImmutableMap(com.google.common.collect.ImmutableMap) JsonSyntaxException(com.google.gson.JsonSyntaxException) IOException(java.io.IOException) HttpServiceResponder(io.cdap.cdap.api.service.http.HttpServiceResponder) Schema(io.cdap.cdap.api.data.schema.Schema) StandardCharsets(java.nio.charset.StandardCharsets) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) AbstractSystemHttpServiceHandler(io.cdap.cdap.api.service.http.AbstractSystemHttpServiceHandler) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) Type(java.lang.reflect.Type) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) HttpServiceRequest(io.cdap.cdap.api.service.http.HttpServiceRequest) BatchPipelineSpec(io.cdap.cdap.etl.batch.BatchPipelineSpec) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) AppRequest(io.cdap.cdap.proto.artifact.AppRequest) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) JsonSyntaxException(com.google.gson.JsonSyntaxException) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 2 with SecureStoreMacroEvaluator

use of io.cdap.cdap.etl.common.SecureStoreMacroEvaluator in project cdap by caskdata.

the class RemoteValidationTask method run.

@Override
public void run(RunnableTaskContext context) throws Exception {
    SystemAppTaskContext systemAppContext = context.getRunnableTaskSystemAppContext();
    RemoteValidationRequest remoteValidationRequest = GSON.fromJson(context.getParam(), RemoteValidationRequest.class);
    String namespace = remoteValidationRequest.getNamespace();
    String originalRequest = remoteValidationRequest.getRequest();
    StageValidationRequest validationRequest;
    try {
        validationRequest = GSON.fromJson(originalRequest, StageValidationRequest.class);
        validationRequest.validate();
    } catch (JsonSyntaxException e) {
        throw new IllegalArgumentException(String.format("Unable to decode request body %s", originalRequest), e);
    } catch (IllegalArgumentException e) {
        throw new IllegalArgumentException("Invalid stage config", e);
    }
    Map<String, String> arguments = Collections.emptyMap();
    // this option.
    if (validationRequest.getResolveMacrosFromPreferences()) {
        try {
            arguments = systemAppContext.getPreferencesForNamespace(namespace, true);
        } catch (IllegalArgumentException iae) {
            // If this is the case, we return a 404 error.
            throw new IllegalArgumentException(String.format(NAMESPACE_DOES_NOT_EXIST, namespace), iae);
        }
    }
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, systemAppContext), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(systemAppContext), ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, systemAppContext));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, DefaultMacroEvaluator.MAP_FUNCTIONS);
    MacroParserOptions macroParserOptions = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    Function<Map<String, String>, Map<String, String>> macroFn = macroProperties -> systemAppContext.evaluateMacros(namespace, macroProperties, macroEvaluator, macroParserOptions);
    PluginConfigurer pluginConfigurer = systemAppContext.createPluginConfigurer(namespace);
    StageValidationResponse validationResponse = ValidationUtils.validate(namespace, validationRequest, pluginConfigurer, macroFn, systemAppContext);
    // If the validation success and if it only involves system artifacts, then we don't need to restart task runner
    if (validationResponse.getFailures().isEmpty()) {
        StageSpec spec = validationResponse.getSpec();
        if (spec != null) {
            context.setTerminateOnComplete(!ArtifactScope.SYSTEM.equals(spec.getPlugin().getArtifact().getScope()));
        }
    }
    context.writeResult(GSON.toJson(validationResponse).getBytes());
}
Also used : StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) SchemaTypeAdapter(io.cdap.cdap.internal.io.SchemaTypeAdapter) LoggerFactory(org.slf4j.LoggerFactory) Function(java.util.function.Function) GsonBuilder(com.google.gson.GsonBuilder) Gson(com.google.gson.Gson) Map(java.util.Map) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) RunnableTaskContext(io.cdap.cdap.api.service.worker.RunnableTaskContext) PluginConfigurer(io.cdap.cdap.api.plugin.PluginConfigurer) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) SystemAppTaskContext(io.cdap.cdap.api.service.worker.SystemAppTaskContext) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) JsonSyntaxException(com.google.gson.JsonSyntaxException) Schema(io.cdap.cdap.api.data.schema.Schema) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) RunnableTask(io.cdap.cdap.api.service.worker.RunnableTask) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) Collections(java.util.Collections) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) SystemAppTaskContext(io.cdap.cdap.api.service.worker.SystemAppTaskContext) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) PluginConfigurer(io.cdap.cdap.api.plugin.PluginConfigurer) JsonSyntaxException(com.google.gson.JsonSyntaxException) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse)

Example 3 with SecureStoreMacroEvaluator

use of io.cdap.cdap.etl.common.SecureStoreMacroEvaluator in project cdap by caskdata.

the class ConnectionHandler method getConnector.

private Connector getConnector(ServicePluginConfigurer configurer, PluginInfo pluginInfo, String namespace, TrackedPluginSelector pluginSelector) throws IOException {
    Map<String, String> arguments = getContext().getPreferencesForNamespace(namespace, true);
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, getContext()), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(getContext()));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, Collections.singleton(OAuthMacroEvaluator.FUNCTION_NAME));
    MacroParserOptions options = MacroParserOptions.builder().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    return ConnectionUtils.getConnector(configurer, pluginInfo, pluginSelector, macroEvaluator, options);
}
Also used : MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments)

Example 4 with SecureStoreMacroEvaluator

use of io.cdap.cdap.etl.common.SecureStoreMacroEvaluator in project cdap by caskdata.

the class RemoteConnectionTaskBase method getConnector.

/**
 * Returns {@link Connector} after evaluating macros
 *
 * @param systemAppContext {@link SystemAppTaskContext}
 * @param configurer       {@link ServicePluginConfigurer}
 * @param pluginInfo       {@link PluginInfo}
 * @param namespace        namespace string
 * @param pluginSelector   {@link TrackedPluginSelector}
 * @return {@link Connector}
 * @throws Exception
 */
protected Connector getConnector(SystemAppTaskContext systemAppContext, ServicePluginConfigurer configurer, PluginInfo pluginInfo, String namespace, TrackedPluginSelector pluginSelector) throws Exception {
    Map<String, String> arguments = systemAppContext.getPreferencesForNamespace(namespace, true);
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, systemAppContext), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(systemAppContext));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, Collections.singleton(OAuthMacroEvaluator.FUNCTION_NAME));
    MacroParserOptions options = MacroParserOptions.builder().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    return ConnectionUtils.getConnector(configurer, pluginInfo, pluginSelector, macroEvaluator, options);
}
Also used : MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments)

Aggregations

MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)4 MacroParserOptions (io.cdap.cdap.api.macro.MacroParserOptions)4 BasicArguments (io.cdap.cdap.etl.common.BasicArguments)4 DefaultMacroEvaluator (io.cdap.cdap.etl.common.DefaultMacroEvaluator)4 OAuthMacroEvaluator (io.cdap.cdap.etl.common.OAuthMacroEvaluator)4 SecureStoreMacroEvaluator (io.cdap.cdap.etl.common.SecureStoreMacroEvaluator)4 ImmutableMap (com.google.common.collect.ImmutableMap)2 Gson (com.google.gson.Gson)2 GsonBuilder (com.google.gson.GsonBuilder)2 JsonSyntaxException (com.google.gson.JsonSyntaxException)2 ArtifactScope (io.cdap.cdap.api.artifact.ArtifactScope)2 Schema (io.cdap.cdap.api.data.schema.Schema)2 ConnectionMacroEvaluator (io.cdap.cdap.etl.common.ConnectionMacroEvaluator)2 StageSpec (io.cdap.cdap.etl.proto.v2.spec.StageSpec)2 StageValidationRequest (io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest)2 SchemaTypeAdapter (io.cdap.cdap.internal.io.SchemaTypeAdapter)2 Collections (java.util.Collections)2 Map (java.util.Map)2 Function (java.util.function.Function)2 Logger (org.slf4j.Logger)2