Search in sources :

Example 1 with RunnableTaskContext

use of io.cdap.cdap.api.service.worker.RunnableTaskContext in project cdap by caskdata.

the class SystemAppTask method run.

@Override
public void run(RunnableTaskContext context) throws Exception {
    ArtifactId systemAppArtifactId = context.getArtifactId();
    if (systemAppArtifactId == null) {
        throw new IllegalArgumentException("Missing artifactId from the system app task request");
    }
    LOG.debug("Received system app task for artifact {}", systemAppArtifactId);
    Injector injector = createInjector(cConf);
    ArtifactRepository artifactRepository = injector.getInstance(ArtifactRepository.class);
    Impersonator impersonator = injector.getInstance(Impersonator.class);
    String systemAppNamespace = context.getNamespace();
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.from(systemAppNamespace), systemAppArtifactId.getName(), systemAppArtifactId.getVersion());
    ArtifactLocalizerClient localizerClient = injector.getInstance(ArtifactLocalizerClient.class);
    File artifactLocation = localizerClient.getUnpackedArtifactLocation(Artifacts.toProtoArtifactId(new NamespaceId(systemAppNamespace), systemAppArtifactId));
    EntityImpersonator classLoaderImpersonator = new EntityImpersonator(artifactId.toEntityId(), impersonator);
    try (CloseableClassLoader artifactClassLoader = artifactRepository.createArtifactClassLoader(new ArtifactDescriptor(artifactId.getNamespace().getId(), artifactId.toArtifactId(), Locations.toLocation(artifactLocation)), classLoaderImpersonator);
        SystemAppTaskContext systemAppTaskContext = buildTaskSystemAppContext(injector, systemAppNamespace, systemAppArtifactId, artifactClassLoader)) {
        RunnableTaskRequest taskRequest = context.getEmbeddedRequest();
        String taskClassName = taskRequest.getClassName();
        if (taskClassName == null) {
            LOG.debug("No system app task to execute");
            return;
        }
        LOG.debug("Requested to run system app task {}", taskClassName);
        Class<?> clazz = artifactClassLoader.loadClass(taskClassName);
        if (!(RunnableTask.class.isAssignableFrom(clazz))) {
            throw new ClassCastException(String.format("%s is not a RunnableTask", taskClassName));
        }
        LOG.debug("Launching system app task {}", taskClassName);
        RunnableTask runnableTask = (RunnableTask) injector.getInstance(clazz);
        RunnableTaskContext runnableTaskContext = new RunnableTaskContext(taskRequest.getParam().getSimpleParam(), null, null, null, systemAppTaskContext) {

            @Override
            public void writeResult(byte[] data) throws IOException {
                context.writeResult(data);
            }

            @Override
            public void setTerminateOnComplete(boolean terminate) {
                context.setTerminateOnComplete(terminate);
            }

            @Override
            public boolean isTerminateOnComplete() {
                return context.isTerminateOnComplete();
            }
        };
        runnableTask.run(runnableTaskContext);
        LOG.debug("System app task completed {}", taskClassName);
    }
}
Also used : ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) SystemAppTaskContext(io.cdap.cdap.api.service.worker.SystemAppTaskContext) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CloseableClassLoader(io.cdap.cdap.api.artifact.CloseableClassLoader) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) ArtifactDescriptor(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDescriptor) Injector(com.google.inject.Injector) RunnableTaskContext(io.cdap.cdap.api.service.worker.RunnableTaskContext) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) RunnableTask(io.cdap.cdap.api.service.worker.RunnableTask) ArtifactLocalizerClient(io.cdap.cdap.internal.app.worker.sidecar.ArtifactLocalizerClient) File(java.io.File) RunnableTaskRequest(io.cdap.cdap.api.service.worker.RunnableTaskRequest)

Example 2 with RunnableTaskContext

use of io.cdap.cdap.api.service.worker.RunnableTaskContext in project cdap by caskdata.

the class RemoteValidationTask method run.

@Override
public void run(RunnableTaskContext context) throws Exception {
    SystemAppTaskContext systemAppContext = context.getRunnableTaskSystemAppContext();
    RemoteValidationRequest remoteValidationRequest = GSON.fromJson(context.getParam(), RemoteValidationRequest.class);
    String namespace = remoteValidationRequest.getNamespace();
    String originalRequest = remoteValidationRequest.getRequest();
    StageValidationRequest validationRequest;
    try {
        validationRequest = GSON.fromJson(originalRequest, StageValidationRequest.class);
        validationRequest.validate();
    } catch (JsonSyntaxException e) {
        throw new IllegalArgumentException(String.format("Unable to decode request body %s", originalRequest), e);
    } catch (IllegalArgumentException e) {
        throw new IllegalArgumentException("Invalid stage config", e);
    }
    Map<String, String> arguments = Collections.emptyMap();
    // this option.
    if (validationRequest.getResolveMacrosFromPreferences()) {
        try {
            arguments = systemAppContext.getPreferencesForNamespace(namespace, true);
        } catch (IllegalArgumentException iae) {
            // If this is the case, we return a 404 error.
            throw new IllegalArgumentException(String.format(NAMESPACE_DOES_NOT_EXIST, namespace), iae);
        }
    }
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, systemAppContext), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(systemAppContext), ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, systemAppContext));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, DefaultMacroEvaluator.MAP_FUNCTIONS);
    MacroParserOptions macroParserOptions = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    Function<Map<String, String>, Map<String, String>> macroFn = macroProperties -> systemAppContext.evaluateMacros(namespace, macroProperties, macroEvaluator, macroParserOptions);
    PluginConfigurer pluginConfigurer = systemAppContext.createPluginConfigurer(namespace);
    StageValidationResponse validationResponse = ValidationUtils.validate(namespace, validationRequest, pluginConfigurer, macroFn, systemAppContext);
    // If the validation success and if it only involves system artifacts, then we don't need to restart task runner
    if (validationResponse.getFailures().isEmpty()) {
        StageSpec spec = validationResponse.getSpec();
        if (spec != null) {
            context.setTerminateOnComplete(!ArtifactScope.SYSTEM.equals(spec.getPlugin().getArtifact().getScope()));
        }
    }
    context.writeResult(GSON.toJson(validationResponse).getBytes());
}
Also used : StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) SchemaTypeAdapter(io.cdap.cdap.internal.io.SchemaTypeAdapter) LoggerFactory(org.slf4j.LoggerFactory) Function(java.util.function.Function) GsonBuilder(com.google.gson.GsonBuilder) Gson(com.google.gson.Gson) Map(java.util.Map) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) RunnableTaskContext(io.cdap.cdap.api.service.worker.RunnableTaskContext) PluginConfigurer(io.cdap.cdap.api.plugin.PluginConfigurer) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) SystemAppTaskContext(io.cdap.cdap.api.service.worker.SystemAppTaskContext) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) JsonSyntaxException(com.google.gson.JsonSyntaxException) Schema(io.cdap.cdap.api.data.schema.Schema) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) RunnableTask(io.cdap.cdap.api.service.worker.RunnableTask) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) Collections(java.util.Collections) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) SecureStoreMacroEvaluator(io.cdap.cdap.etl.common.SecureStoreMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) ConnectionMacroEvaluator(io.cdap.cdap.etl.common.ConnectionMacroEvaluator) SystemAppTaskContext(io.cdap.cdap.api.service.worker.SystemAppTaskContext) OAuthMacroEvaluator(io.cdap.cdap.etl.common.OAuthMacroEvaluator) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) PluginConfigurer(io.cdap.cdap.api.plugin.PluginConfigurer) JsonSyntaxException(com.google.gson.JsonSyntaxException) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) BasicArguments(io.cdap.cdap.etl.common.BasicArguments) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse)

Example 3 with RunnableTaskContext

use of io.cdap.cdap.api.service.worker.RunnableTaskContext in project cdap by caskdata.

the class TaskWorkerHttpHandlerInternal method run.

@POST
@Path("/run")
public void run(FullHttpRequest request, HttpResponder responder) {
    if (!hasInflightRequest.compareAndSet(false, true)) {
        responder.sendStatus(HttpResponseStatus.TOO_MANY_REQUESTS);
        return;
    }
    requestProcessedCount.incrementAndGet();
    long startTime = System.currentTimeMillis();
    String className = null;
    try {
        RunnableTaskRequest runnableTaskRequest = GSON.fromJson(request.content().toString(StandardCharsets.UTF_8), RunnableTaskRequest.class);
        className = getTaskClassName(runnableTaskRequest);
        RunnableTaskContext runnableTaskContext = runnableTaskLauncher.launchRunnableTask(runnableTaskRequest);
        responder.sendContent(HttpResponseStatus.OK, new RunnableTaskBodyProducer(runnableTaskContext, stopper, new TaskDetails(true, className, startTime)), new DefaultHttpHeaders().add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM));
    } catch (ClassNotFoundException | ClassCastException ex) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, exceptionToJson(ex), EmptyHttpHeaders.INSTANCE);
        // Since the user class is not even loaded, no user code ran, hence it's ok to not terminate the runner
        stopper.accept(false, new TaskDetails(false, className, startTime));
    } catch (Exception ex) {
        LOG.error("Failed to run task {}", request.content().toString(StandardCharsets.UTF_8), ex);
        responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, exceptionToJson(ex), EmptyHttpHeaders.INSTANCE);
        // Potentially ran user code, hence terminate the runner.
        stopper.accept(true, new TaskDetails(false, className, startTime));
    }
}
Also used : DefaultHttpHeaders(io.netty.handler.codec.http.DefaultHttpHeaders) RunnableTaskContext(io.cdap.cdap.api.service.worker.RunnableTaskContext) RunnableTaskRequest(io.cdap.cdap.api.service.worker.RunnableTaskRequest) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST)

Example 4 with RunnableTaskContext

use of io.cdap.cdap.api.service.worker.RunnableTaskContext in project cdap by caskdata.

the class RunnableTaskLauncher method launchRunnableTask.

public RunnableTaskContext launchRunnableTask(RunnableTaskRequest request) throws Exception {
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    if (classLoader == null) {
        classLoader = getClass().getClassLoader();
    }
    Class<?> clazz = classLoader.loadClass(request.getClassName());
    Injector injector = Guice.createInjector(new RunnableTaskModule(cConf));
    Object obj = injector.getInstance(clazz);
    if (!(obj instanceof RunnableTask)) {
        throw new ClassCastException(String.format("%s is not a RunnableTask", request.getClassName()));
    }
    RunnableTask runnableTask = (RunnableTask) obj;
    RunnableTaskContext runnableTaskContext = RunnableTaskContext.getBuilder().withParam(request.getParam()).withArtifactId(request.getArtifactId()).withNamespace(request.getNamespace()).build();
    runnableTask.run(runnableTaskContext);
    return runnableTaskContext;
}
Also used : Injector(com.google.inject.Injector) RunnableTaskContext(io.cdap.cdap.api.service.worker.RunnableTaskContext) RunnableTask(io.cdap.cdap.api.service.worker.RunnableTask)

Aggregations

RunnableTaskContext (io.cdap.cdap.api.service.worker.RunnableTaskContext)4 RunnableTask (io.cdap.cdap.api.service.worker.RunnableTask)3 Injector (com.google.inject.Injector)2 RunnableTaskRequest (io.cdap.cdap.api.service.worker.RunnableTaskRequest)2 SystemAppTaskContext (io.cdap.cdap.api.service.worker.SystemAppTaskContext)2 ImmutableMap (com.google.common.collect.ImmutableMap)1 Gson (com.google.gson.Gson)1 GsonBuilder (com.google.gson.GsonBuilder)1 JsonSyntaxException (com.google.gson.JsonSyntaxException)1 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)1 ArtifactScope (io.cdap.cdap.api.artifact.ArtifactScope)1 CloseableClassLoader (io.cdap.cdap.api.artifact.CloseableClassLoader)1 Schema (io.cdap.cdap.api.data.schema.Schema)1 MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)1 MacroParserOptions (io.cdap.cdap.api.macro.MacroParserOptions)1 PluginConfigurer (io.cdap.cdap.api.plugin.PluginConfigurer)1 Id (io.cdap.cdap.common.id.Id)1 BasicArguments (io.cdap.cdap.etl.common.BasicArguments)1 ConnectionMacroEvaluator (io.cdap.cdap.etl.common.ConnectionMacroEvaluator)1 DefaultMacroEvaluator (io.cdap.cdap.etl.common.DefaultMacroEvaluator)1