Search in sources :

Example 1 with RetryingLongTransactionSystemClient

use of io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient in project cdap by caskdata.

the class MainOutputCommitter method setupJob.

@Override
public void setupJob(JobContext jobContext) throws IOException {
    Configuration configuration = jobContext.getConfiguration();
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration);
    MapReduceTaskContextProvider taskContextProvider = classLoader.getTaskContextProvider();
    Injector injector = taskContextProvider.getInjector();
    cConf = injector.getInstance(CConfiguration.class);
    MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
    ProgramId programId = contextConfig.getProgramId();
    LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
    RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf);
    this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), retryStrategy);
    // We start long-running tx to be used by mapreduce job tasks when running on premise
    if (ProgramRunners.getClusterMode(contextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) {
        this.transaction = txClient.startLong();
        // Write the tx somewhere, so that we can re-use it in mapreduce tasks
        Path txFile = getTxFile(configuration, jobContext.getJobID());
        FileSystem fs = txFile.getFileSystem(configuration);
        try (FSDataOutputStream fsDataOutputStream = fs.create(txFile, false)) {
            fsDataOutputStream.write(new TransactionCodec().encode(transaction));
        }
    }
    // we can instantiate the TaskContext after we set the tx above. It's used by the operations below
    taskContext = taskContextProvider.get(taskAttemptContext);
    this.outputs = Outputs.transform(contextConfig.getOutputs(), taskContext);
    super.setupJob(jobContext);
}
Also used : Path(org.apache.hadoop.fs.Path) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ProgramId(io.cdap.cdap.proto.id.ProgramId) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) RetryingLongTransactionSystemClient(io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient) Injector(com.google.inject.Injector) TransactionCodec(org.apache.tephra.TransactionCodec) FileSystem(org.apache.hadoop.fs.FileSystem) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy)

Example 2 with RetryingLongTransactionSystemClient

use of io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient in project cdap by caskdata.

the class MainOutputCommitter method setupJob.

@Override
public void setupJob(JobContext jobContext) throws IOException {
    Configuration configuration = jobContext.getConfiguration();
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration);
    MapReduceTaskContextProvider taskContextProvider = classLoader.getTaskContextProvider();
    Injector injector = taskContextProvider.getInjector();
    cConf = injector.getInstance(CConfiguration.class);
    MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
    ProgramId programId = contextConfig.getProgramId();
    LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
    RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf);
    this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), retryStrategy);
    // We start long-running tx to be used by mapreduce job tasks.
    this.transaction = txClient.startLong();
    // Write the tx somewhere, so that we can re-use it in mapreduce tasks
    Path txFile = getTxFile(configuration, jobContext.getJobID());
    FileSystem fs = txFile.getFileSystem(configuration);
    try (FSDataOutputStream fsDataOutputStream = fs.create(txFile, false)) {
        fsDataOutputStream.write(new TransactionCodec().encode(transaction));
    }
    // we can instantiate the TaskContext after we set the tx above. It's used by the operations below
    taskContext = taskContextProvider.get(taskAttemptContext);
    this.outputs = Outputs.transform(contextConfig.getOutputs(), taskContext);
    super.setupJob(jobContext);
}
Also used : Path(org.apache.hadoop.fs.Path) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) ProgramId(co.cask.cdap.proto.id.ProgramId) CConfiguration(co.cask.cdap.common.conf.CConfiguration) RetryingLongTransactionSystemClient(co.cask.cdap.data2.transaction.RetryingLongTransactionSystemClient) Injector(com.google.inject.Injector) TransactionCodec(org.apache.tephra.TransactionCodec) FileSystem(org.apache.hadoop.fs.FileSystem) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) RetryStrategy(co.cask.cdap.common.service.RetryStrategy)

Aggregations

Injector (com.google.inject.Injector)2 Configuration (org.apache.hadoop.conf.Configuration)2 FSDataOutputStream (org.apache.hadoop.fs.FSDataOutputStream)2 FileSystem (org.apache.hadoop.fs.FileSystem)2 Path (org.apache.hadoop.fs.Path)2 TransactionCodec (org.apache.tephra.TransactionCodec)2 CConfiguration (co.cask.cdap.common.conf.CConfiguration)1 RetryStrategy (co.cask.cdap.common.service.RetryStrategy)1 RetryingLongTransactionSystemClient (co.cask.cdap.data2.transaction.RetryingLongTransactionSystemClient)1 ProgramId (co.cask.cdap.proto.id.ProgramId)1 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)1 RetryStrategy (io.cdap.cdap.common.service.RetryStrategy)1 RetryingLongTransactionSystemClient (io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient)1 ProgramId (io.cdap.cdap.proto.id.ProgramId)1