Search in sources :

Example 1 with PropertyFieldSetter

use of io.cdap.cdap.common.lang.PropertyFieldSetter in project cdap by caskdata.

the class WorkflowDriver method executeCondition.

@SuppressWarnings("unchecked")
private void executeCondition(ApplicationSpecification appSpec, final WorkflowConditionNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception {
    final BasicWorkflowContext context = new BasicWorkflowContext(workflowSpec, token, program, programOptions, cConf, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, nodeStates, pluginInstantiator, secureStore, secureStoreManager, messagingService, node.getConditionSpecification(), metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
    final Iterator<WorkflowNode> iterator;
    Class<?> clz = classLoader.loadClass(node.getPredicateClassName());
    Predicate<WorkflowContext> predicate = instantiator.get(TypeToken.of((Class<? extends Predicate<WorkflowContext>>) clz)).create();
    if (!(predicate instanceof Condition)) {
        iterator = predicate.apply(context) ? node.getIfBranch().iterator() : node.getElseBranch().iterator();
    } else {
        final Condition workflowCondition = (Condition) predicate;
        Reflections.visit(workflowCondition, workflowCondition.getClass(), new PropertyFieldSetter(node.getConditionSpecification().getProperties()), new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics()));
        TransactionControl defaultTxControl = workflowContext.getDefaultTxControl();
        try {
            // AbstractCondition implements final initialize(context) and requires subclass to
            // implement initialize(), whereas conditions that directly implement Condition can
            // override initialize(context)
            TransactionControl txControl = workflowCondition instanceof AbstractCondition ? Transactions.getTransactionControl(defaultTxControl, AbstractCondition.class, workflowCondition, "initialize") : Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "initialize", WorkflowContext.class);
            context.initializeProgram(workflowCondition, txControl, false);
            boolean result = context.execute(() -> workflowCondition.apply(context));
            iterator = result ? node.getIfBranch().iterator() : node.getElseBranch().iterator();
        } finally {
            TransactionControl txControl = Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "destroy");
            context.destroyProgram(workflowCondition, txControl, false);
        }
    }
    // If a workflow updates its token at a condition node, it will be persisted after the execution of the next node.
    // However, the call below ensures that even if the workflow fails/crashes after a condition node, updates from the
    // condition node are also persisted.
    workflowStateWriter.setWorkflowToken(workflowRunId, token);
    executeAll(iterator, appSpec, instantiator, classLoader, token);
}
Also used : AbstractCondition(io.cdap.cdap.api.workflow.AbstractCondition) Condition(io.cdap.cdap.api.workflow.Condition) WorkflowContext(io.cdap.cdap.api.workflow.WorkflowContext) WorkflowNode(io.cdap.cdap.api.workflow.WorkflowNode) DataSetFieldSetter(io.cdap.cdap.internal.app.runtime.DataSetFieldSetter) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) AbstractCondition(io.cdap.cdap.api.workflow.AbstractCondition)

Example 2 with PropertyFieldSetter

use of io.cdap.cdap.common.lang.PropertyFieldSetter in project cdap by caskdata.

the class MapperWrapper method run.

@SuppressWarnings("unchecked")
@Override
public void run(Context context) throws IOException, InterruptedException {
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(context.getConfiguration());
    ClassLoader weakReferenceClassLoader = new WeakReferenceDelegatorClassLoader(classLoader);
    BasicMapReduceTaskContext basicMapReduceContext = classLoader.getTaskContextProvider().get(context);
    String program = basicMapReduceContext.getProgramName();
    final MapTaskMetricsWriter mapTaskMetricsWriter = new MapTaskMetricsWriter(basicMapReduceContext.getProgramMetrics(), context);
    // this is a hook for periodic flushing of changes buffered by datasets (to avoid OOME)
    WrappedMapper.Context flushingContext = createAutoFlushingContext(context, basicMapReduceContext, mapTaskMetricsWriter);
    basicMapReduceContext.setHadoopContext(flushingContext);
    InputSplit inputSplit = context.getInputSplit();
    if (inputSplit instanceof MultiInputTaggedSplit) {
        basicMapReduceContext.setInputContext(InputContexts.create((MultiInputTaggedSplit) inputSplit));
    }
    ClassLoader programClassLoader = classLoader.getProgramClassLoader();
    Mapper delegate = createMapperInstance(programClassLoader, getWrappedMapper(context.getConfiguration()), context, program);
    // injecting runtime components, like datasets, etc.
    try {
        Reflections.visit(delegate, delegate.getClass(), new PropertyFieldSetter(basicMapReduceContext.getSpecification().getProperties()), new MetricsFieldSetter(basicMapReduceContext.getMetrics()), new DataSetFieldSetter(basicMapReduceContext));
    } catch (Throwable t) {
        Throwable rootCause = Throwables.getRootCause(t);
        USERLOG.error("Failed to initialize program '{}' with error: {}. Please check the system logs for more details.", program, rootCause.getMessage(), rootCause);
        throw new IOException(String.format("Failed to inject fields to %s", delegate.getClass()), t);
    }
    ClassLoader oldClassLoader;
    if (delegate instanceof ProgramLifecycle) {
        oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
        try {
            ((ProgramLifecycle) delegate).initialize(new MapReduceLifecycleContext(basicMapReduceContext));
        } catch (Exception e) {
            Throwable rootCause = Throwables.getRootCause(e);
            USERLOG.error("Failed to initialize program '{}' with error: {}. Please check the system logs for more " + "details.", program, rootCause.getMessage(), rootCause);
            throw new IOException(String.format("Failed to initialize mapper with %s", basicMapReduceContext), e);
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
    }
    oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
    try {
        delegate.run(flushingContext);
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
    // memory by tx agent)
    try {
        basicMapReduceContext.flushOperations();
    } catch (Exception e) {
        throw new IOException("Failed to flush operations at the end of mapper of " + basicMapReduceContext, e);
    }
    // Close all writers created by MultipleOutputs
    basicMapReduceContext.closeMultiOutputs();
    if (delegate instanceof ProgramLifecycle) {
        oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
        try {
            ((ProgramLifecycle<? extends RuntimeContext>) delegate).destroy();
        } catch (Exception e) {
            LOG.error("Error during destroy of mapper {}", basicMapReduceContext, e);
        // Do nothing, try to finish
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
    }
    // Emit metrics one final time
    mapTaskMetricsWriter.reportMetrics();
}
Also used : ProgramLifecycle(io.cdap.cdap.api.ProgramLifecycle) IOException(java.io.IOException) DataSetFieldSetter(io.cdap.cdap.internal.app.runtime.DataSetFieldSetter) IOException(java.io.IOException) Mapper(org.apache.hadoop.mapreduce.Mapper) WrappedMapper(org.apache.hadoop.mapreduce.lib.map.WrappedMapper) WeakReferenceDelegatorClassLoader(io.cdap.cdap.common.lang.WeakReferenceDelegatorClassLoader) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) WrappedMapper(org.apache.hadoop.mapreduce.lib.map.WrappedMapper) MultiInputTaggedSplit(io.cdap.cdap.internal.app.runtime.batch.dataset.input.MultiInputTaggedSplit) WeakReferenceDelegatorClassLoader(io.cdap.cdap.common.lang.WeakReferenceDelegatorClassLoader) RuntimeContext(io.cdap.cdap.api.RuntimeContext) InputSplit(org.apache.hadoop.mapreduce.InputSplit) TaggedInputSplit(io.cdap.cdap.internal.app.runtime.batch.dataset.input.TaggedInputSplit)

Example 3 with PropertyFieldSetter

use of io.cdap.cdap.common.lang.PropertyFieldSetter in project cdap by caskdata.

the class CustomActionExecutor method createCustomAction.

@SuppressWarnings("unchecked")
private CustomAction createCustomAction(BasicCustomActionContext context, InstantiatorFactory instantiator, ClassLoader classLoader) throws Exception {
    Class<?> clz = Class.forName(context.getSpecification().getClassName(), true, classLoader);
    Preconditions.checkArgument(CustomAction.class.isAssignableFrom(clz), "%s is not a CustomAction.", clz);
    CustomAction action = instantiator.get(TypeToken.of((Class<? extends CustomAction>) clz)).create();
    Reflections.visit(action, action.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics()));
    return action;
}
Also used : PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) AbstractCustomAction(io.cdap.cdap.api.customaction.AbstractCustomAction) CustomAction(io.cdap.cdap.api.customaction.CustomAction) DataSetFieldSetter(io.cdap.cdap.internal.app.runtime.DataSetFieldSetter)

Example 4 with PropertyFieldSetter

use of io.cdap.cdap.common.lang.PropertyFieldSetter in project cdap by caskdata.

the class WorkerDriver method startUp.

@Override
protected void startUp() throws Exception {
    LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
    // Instantiate worker instance
    Class<?> workerClass = program.getClassLoader().loadClass(spec.getClassName());
    @SuppressWarnings("unchecked") TypeToken<Worker> workerType = (TypeToken<Worker>) TypeToken.of(workerClass);
    worker = new InstantiatorFactory(false).get(workerType).create();
    // Fields injection
    Reflections.visit(worker, workerType.getType(), new MetricsFieldSetter(context.getMetrics()), new PropertyFieldSetter(spec.getProperties()));
    LOG.debug("Starting Worker Program {}", program.getId());
    // Initialize worker
    // Worker is always using Explicit transaction
    TransactionControl txControl = Transactions.getTransactionControl(TransactionControl.EXPLICIT, Worker.class, worker, "initialize", WorkerContext.class);
    context.initializeProgram(worker, txControl, false);
}
Also used : InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) TypeToken(com.google.common.reflect.TypeToken) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) Worker(io.cdap.cdap.api.worker.Worker)

Example 5 with PropertyFieldSetter

use of io.cdap.cdap.common.lang.PropertyFieldSetter in project cdap by caskdata.

the class MapReduceProgramRunner method run.

@Override
public ProgramController run(final Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MAPREDUCE process type is supported.");
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
    DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
    // Setup dataset framework context, if required
    if (programDatasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    MapReduce mapReduce;
    try {
        mapReduce = new InstantiatorFactory(false).get(TypeToken.of(program.<MapReduce>getMainClass())).create();
    } catch (Exception e) {
        LOG.error("Failed to instantiate MapReduce class for {}", spec.getClassName(), e);
        throw Throwables.propagate(e);
    }
    // List of all Closeable resources that needs to be cleanup
    List<Closeable> closeables = new ArrayList<>();
    try {
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.add(pluginInstantiator);
        }
        final BasicMapReduceContext context = new BasicMapReduceContext(program, options, cConf, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txSystemClient, programDatasetFramework, getPluginArchive(options), pluginInstantiator, secureStore, secureStoreManager, messagingService, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
        closeables.add(context);
        Reflections.visit(mapReduce, mapReduce.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new MetricsFieldSetter(context.getMetrics()), new DataSetFieldSetter(context));
        // note: this sets logging context on the thread level
        LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
        // Set the job queue to hConf if it is provided
        Configuration hConf = new Configuration(this.hConf);
        String schedulerQueue = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
        if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
            hConf.set(JobContext.QUEUE_NAME, schedulerQueue);
        }
        ClusterMode clusterMode = ProgramRunners.getClusterMode(options);
        Service mapReduceRuntimeService = new MapReduceRuntimeService(injector, cConf, hConf, mapReduce, spec, context, program.getJarLocation(), locationFactory, clusterMode, fieldLineageWriter);
        mapReduceRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new MapReduceProgramController(mapReduceRuntimeService, context);
        LOG.debug("Starting MapReduce Job: {}", context);
        // be running the job, but the data directory will be owned by cdap.
        if (MapReduceTaskContextProvider.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) {
            mapReduceRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), mapReduceRuntimeService);
        }
        return controller;
    } catch (Exception e) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(e);
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) MapReduce(io.cdap.cdap.api.mapreduce.MapReduce) DatasetFramework(io.cdap.cdap.data2.dataset2.DatasetFramework) NameMappedDatasetFramework(io.cdap.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework) InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) ProgramType(io.cdap.cdap.proto.ProgramType) RunId(org.apache.twill.api.RunId) ProgramController(io.cdap.cdap.app.runtime.ProgramController) MapReduceSpecification(io.cdap.cdap.api.mapreduce.MapReduceSpecification) Arguments(io.cdap.cdap.app.runtime.Arguments) MessagingService(io.cdap.cdap.messaging.MessagingService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ProgramId(io.cdap.cdap.proto.id.ProgramId) BasicProgramContext(io.cdap.cdap.internal.app.runtime.BasicProgramContext) DataSetFieldSetter(io.cdap.cdap.internal.app.runtime.DataSetFieldSetter) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) WorkflowProgramInfo(io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) PluginInstantiator(io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator) ProgramContextAware(io.cdap.cdap.data.ProgramContextAware)

Aggregations

PropertyFieldSetter (io.cdap.cdap.common.lang.PropertyFieldSetter)7 MetricsFieldSetter (io.cdap.cdap.internal.app.runtime.MetricsFieldSetter)7 DataSetFieldSetter (io.cdap.cdap.internal.app.runtime.DataSetFieldSetter)6 IOException (java.io.IOException)3 ProgramLifecycle (io.cdap.cdap.api.ProgramLifecycle)2 RuntimeContext (io.cdap.cdap.api.RuntimeContext)2 TransactionControl (io.cdap.cdap.api.annotation.TransactionControl)2 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)2 InstantiatorFactory (io.cdap.cdap.common.lang.InstantiatorFactory)2 WeakReferenceDelegatorClassLoader (io.cdap.cdap.common.lang.WeakReferenceDelegatorClassLoader)2 ArrayList (java.util.ArrayList)2 Configuration (org.apache.hadoop.conf.Configuration)2 Function (com.google.common.base.Function)1 Joiner (com.google.common.base.Joiner)1 TypeToken (com.google.common.reflect.TypeToken)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 Service (com.google.common.util.concurrent.Service)1 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)1 AbstractCustomAction (io.cdap.cdap.api.customaction.AbstractCustomAction)1 CustomAction (io.cdap.cdap.api.customaction.CustomAction)1