Examples with ProgramLifecycle co.cask.cdap.api.ProgramLifecycle used on opensource projects

Example 6 with ProgramLifecycle

use of co.cask.cdap.api.ProgramLifecycle in project cdap by caskdata.

the class ReducerWrapper method run.

@SuppressWarnings("unchecked")
@Override
public void run(Context context) throws IOException, InterruptedException {
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(context.getConfiguration());
    ClassLoader weakReferenceClassLoader = new WeakReferenceDelegatorClassLoader(classLoader);
    BasicMapReduceTaskContext basicMapReduceContext = classLoader.getTaskContextProvider().get(context);
    // this is a hook for periodic flushing of changes buffered by datasets (to avoid OOME)
    WrappedReducer.Context flushingContext = createAutoFlushingContext(context, basicMapReduceContext);
    basicMapReduceContext.setHadoopContext(flushingContext);
    String userReducer = context.getConfiguration().get(ATTR_REDUCER_CLASS);
    ClassLoader programClassLoader = classLoader.getProgramClassLoader();
    Reducer delegate = createReducerInstance(programClassLoader, userReducer);
    // injecting runtime components, like datasets, etc.
    try {
        Reflections.visit(delegate, delegate.getClass(), new PropertyFieldSetter(basicMapReduceContext.getSpecification().getProperties()), new MetricsFieldSetter(basicMapReduceContext.getMetrics()), new DataSetFieldSetter(basicMapReduceContext));
    } catch (Throwable t) {
        LOG.error("Failed to inject fields to {}.", delegate.getClass(), t);
        throw Throwables.propagate(t);
    }
    ClassLoader oldClassLoader;
    if (delegate instanceof ProgramLifecycle) {
        oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
        try {
            ((ProgramLifecycle) delegate).initialize(new MapReduceLifecycleContext(basicMapReduceContext));
        } catch (Exception e) {
            LOG.error("Failed to initialize reducer with {}", basicMapReduceContext, e);
            throw Throwables.propagate(e);
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
    }
    oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
    try {
        delegate.run(flushingContext);
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
    // memory by tx agent)
    try {
        basicMapReduceContext.flushOperations();
    } catch (Exception e) {
        LOG.error("Failed to flush operations at the end of reducer of " + basicMapReduceContext, e);
        throw Throwables.propagate(e);
    }
    // Close all writers created by MultipleOutputs
    basicMapReduceContext.closeMultiOutputs();
    if (delegate instanceof ProgramLifecycle) {
        oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
        try {
            ((ProgramLifecycle<? extends RuntimeContext>) delegate).destroy();
        } catch (Exception e) {
            LOG.error("Error during destroy of reducer {}", basicMapReduceContext, e);
        // Do nothing, try to finish
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
    }
}

Example 7 with ProgramLifecycle

use of co.cask.cdap.api.ProgramLifecycle in project cdap by caskdata.

the class MapReduceRuntimeService method doInitialize.

private void doInitialize(Job job) throws Exception {
    context.setState(new ProgramState(ProgramStatus.INITIALIZING, null));
    ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(job.getConfiguration().getClassLoader());
    try {
        if (mapReduce instanceof ProgramLifecycle) {
            //noinspection unchecked
            ((ProgramLifecycle) mapReduce).initialize(context);
        } else {
            mapReduce.beforeSubmit(context);
        }
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
    // once the initialize method is executed, set the status of the MapReduce to RUNNING
    context.setState(new ProgramState(ProgramStatus.RUNNING, null));
}

Example 8 with ProgramLifecycle

use of co.cask.cdap.api.ProgramLifecycle in project cdap by caskdata.

the class MapReduceRuntimeService method beforeSubmit.

/**
   * For pre 3.5 MapReduce programs, calls the {@link MapReduce#beforeSubmit(MapReduceContext)} method.
   * For MapReduce programs created after 3.5, calls the initialize method of the {@link ProgramLifecycle}.
   * This method also sets up the Input/Output within the same transaction.
   */
@SuppressWarnings("unchecked")
private void beforeSubmit(final Job job) throws Exception {
    // AbstractMapReduce implements final initialize(context) and requires subclass to
    // implement initialize(), whereas programs that directly implement MapReduce have
    // the option to override initialize(context) (if they implement ProgramLifeCycle)
    final TransactionControl txControl = mapReduce instanceof AbstractMapReduce ? Transactions.getTransactionControl(TransactionControl.IMPLICIT, AbstractMapReduce.class, mapReduce, "initialize") : mapReduce instanceof ProgramLifecycle ? Transactions.getTransactionControl(TransactionControl.IMPLICIT, MapReduce.class, mapReduce, "initialize", MapReduceContext.class) : TransactionControl.IMPLICIT;
    if (TransactionControl.EXPLICIT == txControl) {
        doInitialize(job);
    } else {
        Transactionals.execute(context, new TxRunnable() {

            @Override
            public void run(DatasetContext context) throws Exception {
                doInitialize(job);
            }
        }, Exception.class);
    }
    ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(job.getConfiguration().getClassLoader());
    try {
        // set input/outputs info, and get one of the configured mapper's TypeToken
        TypeToken<?> mapperTypeToken = setInputsIfNeeded(job);
        setOutputsIfNeeded(job);
        setOutputClassesIfNeeded(job, mapperTypeToken);
        setMapOutputClassesIfNeeded(job, mapperTypeToken);
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
}

Example 9 with ProgramLifecycle

use of co.cask.cdap.api.ProgramLifecycle in project cdap by caskdata.

the class BasicMapReduceTaskContext method close.

@Override
public void close() {
    super.close();
    // destroy all of the ProgramLifeCycles registered
    RuntimeException ex = null;
    for (ProgramLifecycle programLifecycle : programLifecycles) {
        try {
            programLifecycle.destroy();
        } catch (RuntimeException e) {
            if (ex == null) {
                ex = new RuntimeException(e);
            } else {
                ex.addSuppressed(e);
            }
        }
    }
    if (ex != null) {
        throw ex;
    }
}

Example 10 with ProgramLifecycle

use of co.cask.cdap.api.ProgramLifecycle in project cdap by caskdata.

the class WrapperUtil method createDelegate.

// instantiates and initializes (if its a ProgramLifeCycle) the class specified by the attribute
static <T> T createDelegate(Configuration conf, String attrClass) {
    String delegateClassName = conf.get(attrClass);
    Class<?> delegateClass = conf.getClassByNameOrNull(delegateClassName);
    Preconditions.checkNotNull(delegateClass, "Class could not be found: ", delegateClassName);
    T delegate = (T) ReflectionUtils.newInstance(delegateClass, conf);
    if (!(delegate instanceof ProgramLifecycle)) {
        return delegate;
    }
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(conf);
    BasicMapReduceTaskContext basicMapReduceContext = classLoader.getTaskContextProvider().get(conf);
    ClassLoader programClassLoader = classLoader.getProgramClassLoader();
    ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(programClassLoader);
    try {
        ProgramLifecycle programLifecycle = (ProgramLifecycle) delegate;
        programLifecycle.initialize(new MapReduceLifecycleContext(basicMapReduceContext));
        // register it so that its destroy method can get called when the BasicMapReduceTaskContext is closed
        basicMapReduceContext.registerProgramLifecycle(programLifecycle);
        return delegate;
    } catch (Exception e) {
        LOG.error("Failed to initialize delegate with {}", basicMapReduceContext, e);
        throw Throwables.propagate(e);
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
}