Examples with RetryingLongTransactionSystemClient io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient used on opensource projects

Example 1 with RetryingLongTransactionSystemClient

use of io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient in project cdap by caskdata.

the class MainOutputCommitter method setupJob.

@Override
public void setupJob(JobContext jobContext) throws IOException {
    Configuration configuration = jobContext.getConfiguration();
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration);
    MapReduceTaskContextProvider taskContextProvider = classLoader.getTaskContextProvider();
    Injector injector = taskContextProvider.getInjector();
    cConf = injector.getInstance(CConfiguration.class);
    MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
    ProgramId programId = contextConfig.getProgramId();
    LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
    RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf);
    this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), retryStrategy);
    // We start long-running tx to be used by mapreduce job tasks when running on premise
    if (ProgramRunners.getClusterMode(contextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) {
        this.transaction = txClient.startLong();
        // Write the tx somewhere, so that we can re-use it in mapreduce tasks
        Path txFile = getTxFile(configuration, jobContext.getJobID());
        FileSystem fs = txFile.getFileSystem(configuration);
        try (FSDataOutputStream fsDataOutputStream = fs.create(txFile, false)) {
            fsDataOutputStream.write(new TransactionCodec().encode(transaction));
        }
    }
    // we can instantiate the TaskContext after we set the tx above. It's used by the operations below
    taskContext = taskContextProvider.get(taskAttemptContext);
    this.outputs = Outputs.transform(contextConfig.getOutputs(), taskContext);
    super.setupJob(jobContext);
}

Example 2 with RetryingLongTransactionSystemClient

use of io.cdap.cdap.data2.transaction.RetryingLongTransactionSystemClient in project cdap by caskdata.

the class MainOutputCommitter method setupJob.

@Override
public void setupJob(JobContext jobContext) throws IOException {
    Configuration configuration = jobContext.getConfiguration();
    MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration);
    MapReduceTaskContextProvider taskContextProvider = classLoader.getTaskContextProvider();
    Injector injector = taskContextProvider.getInjector();
    cConf = injector.getInstance(CConfiguration.class);
    MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
    ProgramId programId = contextConfig.getProgramId();
    LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
    RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf);
    this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), retryStrategy);
    // We start long-running tx to be used by mapreduce job tasks.
    this.transaction = txClient.startLong();
    // Write the tx somewhere, so that we can re-use it in mapreduce tasks
    Path txFile = getTxFile(configuration, jobContext.getJobID());
    FileSystem fs = txFile.getFileSystem(configuration);
    try (FSDataOutputStream fsDataOutputStream = fs.create(txFile, false)) {
        fsDataOutputStream.write(new TransactionCodec().encode(transaction));
    }
    // we can instantiate the TaskContext after we set the tx above. It's used by the operations below
    taskContext = taskContextProvider.get(taskAttemptContext);
    this.outputs = Outputs.transform(contextConfig.getOutputs(), taskContext);
    super.setupJob(jobContext);
}