Examples with TransactionFailureException org.apache.tephra.TransactionFailureException used on opensource projects

Example 26 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class MapReduceRuntimeService method startUp.

@Override
protected void startUp() throws Exception {
    // Creates a temporary directory locally for storing all generated files.
    File tempDir = createTempDirectory();
    cleanupTask = createCleanupTask(tempDir, context);
    try {
        Job job = createJob(new File(tempDir, "mapreduce"));
        Configuration mapredConf = job.getConfiguration();
        MapReduceClassLoader classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getApplicationSpecification().getPlugins(), context.getPluginInstantiator());
        cleanupTask = createCleanupTask(cleanupTask, classLoader);
        context.setMapReduceClassLoader(classLoader);
        context.setJob(job);
        mapredConf.setClassLoader(context.getProgramInvocationClassLoader());
        beforeSubmit(job);
        // Localize additional resources that users have requested via BasicMapReduceContext.localize methods
        Map<String, String> localizedUserResources = localizeUserResources(job, tempDir);
        // Override user-defined job name, since we set it and depend on the name.
        // https://issues.cask.co/browse/CDAP-2441
        String jobName = job.getJobName();
        if (!jobName.isEmpty()) {
            LOG.warn("Job name {} is being overridden.", jobName);
        }
        job.setJobName(getJobName(context));
        // Create a temporary location for storing all generated files through the LocationFactory.
        Location tempLocation = createTempLocationDirectory();
        cleanupTask = createCleanupTask(cleanupTask, tempLocation);
        // For local mode, everything is in the configuration classloader already, hence no need to create new jar
        if (!MapReduceTaskContextProvider.isLocal(mapredConf)) {
            // After calling initialize, we know what plugins are needed for the program, hence construct the proper
            // ClassLoader from here and use it for setting up the job
            Location pluginArchive = createPluginArchive(tempLocation);
            if (pluginArchive != null) {
                job.addCacheArchive(pluginArchive.toURI());
                mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName());
            }
        }
        // set resources for the job
        TaskType.MAP.configure(mapredConf, cConf, context.getMapperRuntimeArguments(), context.getMapperResources());
        TaskType.REDUCE.configure(mapredConf, cConf, context.getReducerRuntimeArguments(), context.getReducerResources());
        // replace user's Mapper, Reducer, Partitioner, and Comparator classes with our wrappers in job config
        MapperWrapper.wrap(job);
        ReducerWrapper.wrap(job);
        PartitionerWrapper.wrap(job);
        RawComparatorWrapper.CombinerGroupComparatorWrapper.wrap(job);
        RawComparatorWrapper.GroupComparatorWrapper.wrap(job);
        RawComparatorWrapper.KeyComparatorWrapper.wrap(job);
        // packaging job jar which includes cdap classes with dependencies
        File jobJar = buildJobJar(job, tempDir);
        job.setJar(jobJar.toURI().toString());
        Location programJar = programJarLocation;
        String hbaseDDLExecutorDirectory = null;
        if (!MapReduceTaskContextProvider.isLocal(mapredConf)) {
            // Copy and localize the program jar in distributed mode
            programJar = copyProgramJar(tempLocation);
            job.addCacheFile(programJar.toURI());
            // Generate and localize the launcher jar to control the classloader of MapReduce containers processes
            Location launcherJar = createLauncherJar(tempLocation);
            job.addCacheFile(launcherJar.toURI());
            // Launcher.jar should be the first one in the classpath
            List<String> classpath = new ArrayList<>();
            classpath.add(launcherJar.getName());
            // Localize logback.xml
            Location logbackLocation = ProgramRunners.createLogbackJar(tempLocation.append("logback.xml.jar"));
            if (logbackLocation != null) {
                job.addCacheFile(logbackLocation.toURI());
                classpath.add(logbackLocation.getName());
                mapredConf.set("yarn.app.mapreduce.am.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
                mapredConf.set("mapreduce.map.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
                mapredConf.set("mapreduce.reduce.env", "CDAP_LOG_DIR=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
            }
            // (CDAP-7052) Locate the name of the javax.ws.rs-api jar to make it last in the classpath
            URL jaxrsURL = ClassLoaders.getClassPathURL(Path.class);
            String jaxrsClassPath = null;
            // Get all the jars in jobJar and sort them lexically before adding to the classpath
            // This allows CDAP classes to be picked up first before the Twill classes
            Set<String> jarFiles = new TreeSet<>();
            try (JarFile jobJarFile = new JarFile(jobJar)) {
                Enumeration<JarEntry> entries = jobJarFile.entries();
                while (entries.hasMoreElements()) {
                    JarEntry entry = entries.nextElement();
                    String entryName = entry.getName();
                    if (entryName.startsWith("lib/") && entryName.endsWith(".jar")) {
                        // Skip the jaxrs jar
                        if (jaxrsURL != null && jaxrsURL.getPath().endsWith(entryName)) {
                            jaxrsClassPath = "job.jar/" + entryName;
                        } else {
                            jarFiles.add("job.jar/" + entryName);
                        }
                    }
                }
            }
            classpath.addAll(jarFiles);
            classpath.add("job.jar/classes");
            // Add extra jars set in cConf
            for (URI jarURI : CConfigurationUtil.getExtraJars(cConf)) {
                if ("file".equals(jarURI.getScheme())) {
                    Location extraJarLocation = copyFileToLocation(new File(jarURI.getPath()), tempLocation);
                    job.addCacheFile(extraJarLocation.toURI());
                } else {
                    job.addCacheFile(jarURI);
                }
                classpath.add(LocalizationUtils.getLocalizedName(jarURI));
            }
            hbaseDDLExecutorDirectory = getLocalizedHBaseDDLExecutorDir(tempDir, cConf, job, tempLocation);
            // Add the mapreduce application classpath, followed by the javax.ws.rs-api at last.
            MapReduceContainerHelper.addMapReduceClassPath(mapredConf, classpath);
            if (jaxrsClassPath != null) {
                classpath.add(jaxrsClassPath);
            }
            mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, Joiner.on(",").join(classpath));
            mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(classpath));
        }
        MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf);
        CConfiguration cConfCopy = CConfiguration.copy(cConf);
        if (hbaseDDLExecutorDirectory != null) {
            cConfCopy.set(Constants.HBaseDDLExecutor.EXTENSIONS_DIR, hbaseDDLExecutorDirectory);
        }
        contextConfig.set(context, cConfCopy, programJar.toURI(), localizedUserResources);
        // submits job and returns immediately.
        // Set the context classloader to the program invocation one (which is a weak reference wrapped MRClassloader)
        ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(context.getProgramInvocationClassLoader());
        try {
            job.submit();
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
        // log after the job.submit(), because the jobId is not assigned before then
        LOG.info("Submitted MapReduce Job: {}.", context);
        this.job = job;
    } catch (Throwable t) {
        cleanupTask.run();
        if (t instanceof Error) {
            // Guava 15.0+ have this condition fixed, hence wrapping is no longer needed if upgrade to later Guava.
            throw new Exception(t);
        }
        // don't log the error. It will be logged by the ProgramControllerServiceAdapter.failed()
        if (t instanceof TransactionFailureException) {
            throw Transactionals.propagate((TransactionFailureException) t, Exception.class);
        }
        throw t;
    }
}

Example 27 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class DatasetBasedTimeScheduleStore method upgrade.

/**
 * Method to add version to row key in SchedulerStore.
 *
 * @throws InterruptedException
 * @throws IOException
 * @throws DatasetManagementException
 */
public void upgrade() throws InterruptedException, IOException, DatasetManagementException {
    while (true) {
        try {
            initializeScheduleTable();
            break;
        } catch (Exception ex) {
            // Expected if the cdap services are not up.
            TimeUnit.SECONDS.sleep(10);
        }
    }
    if (isUpgradeComplete()) {
        LOG.info("{} is already upgraded.", NAME);
        return;
    }
    final AtomicInteger sleepTimeInSecs = new AtomicInteger(60);
    final AtomicInteger tries = new AtomicInteger(0);
    LOG.info("Starting upgrade of {}.", NAME);
    while (!isUpgradeComplete()) {
        sleepTimeInSecs.set(60);
        try {
            factory.createExecutor(ImmutableList.of((TransactionAware) table)).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() {
                    upgradeJobs(table);
                    upgradeTriggers(table);
                    // Upgrade is complete. Mark that app version upgrade is complete in the table.
                    table.put(APP_VERSION_UPGRADE_KEY, COLUMN, Bytes.toBytes(ProjectInfo.getVersion().toString()));
                }
            });
        } catch (TransactionFailureException e) {
            if (e instanceof TransactionConflictException) {
                LOG.debug("Upgrade step faced Transaction Conflict exception. Retrying operation now.", e);
                sleepTimeInSecs.set(10);
            } else {
                LOG.error("Upgrade step faced exception. Will retry operation after some delay.", e);
                sleepTimeInSecs.set(60);
            }
        }
        if (tries.incrementAndGet() > 500) {
            LOG.warn("Could not complete upgrade of {}, tried for 500 times", NAME);
            return;
        }
        TimeUnit.SECONDS.sleep(sleepTimeInSecs.get());
    }
    LOG.info("Upgrade of {} is complete.", NAME);
}

Example 28 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project phoenix by apache.

the class TephraTransactionContext method commitDDLFence.

@Override
public void commitDDLFence(PTable dataTable, Logger logger) throws SQLException {
    byte[] key = dataTable.getName().getBytes();
    try {
        FenceWait fenceWait = VisibilityFence.prepareWait(key, txServiceClient);
        fenceWait.await(10000, TimeUnit.MILLISECONDS);
        if (logger.isInfoEnabled()) {
            logger.info("Added write fence at ~" + getCurrentTransaction().getReadPointer());
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new SQLExceptionInfo.Builder(SQLExceptionCode.INTERRUPTED_EXCEPTION).setRootCause(e).build().buildException();
    } catch (TimeoutException | TransactionFailureException e) {
        throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_UNABLE_TO_GET_WRITE_FENCE).setSchemaName(dataTable.getSchemaName().getString()).setTableName(dataTable.getTableName().getString()).build().buildException();
    }
}

Example 29 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project phoenix by apache.

the class TephraTransactionContext method checkpoint.

@Override
public void checkpoint(boolean hasUncommittedData) throws SQLException {
    if (hasUncommittedData) {
        try {
            if (txContext == null) {
                tx = txServiceClient.checkpoint(tx);
            } else {
                assert (txContext != null);
                txContext.checkpoint();
                tx = txContext.getCurrentTransaction();
            }
        } catch (TransactionFailureException e) {
            throw new SQLException(e);
        }
    }
    // or get into an infinite loop (for UPSERT SELECT).
    if (txContext == null) {
        tx.setVisibility(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
    } else {
        assert (txContext != null);
        txContext.getCurrentTransaction().setVisibility(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
    }
}

Example 30 with TransactionFailureException

use of org.apache.tephra.TransactionFailureException in project cdap by caskdata.

the class CoreSchedulerService method cleanupJobs.

// Attempts to remove all jobs that are in PENDING_LAUNCH state.
// These are jobs that were about to be launched, but the scheduler shut down or crashed after the job was marked
// PENDING_LAUNCH, but before they were actually launched.
// This should only be called at startup.
private void cleanupJobs() {
    try {
        transactional.execute(context -> {
            JobQueueDataset jobQueue = Schedulers.getJobQueue(context, datasetFramework);
            try (CloseableIterator<Job> jobIter = jobQueue.fullScan()) {
                LOG.info("Cleaning up jobs in state {}.", Job.State.PENDING_LAUNCH);
                while (jobIter.hasNext()) {
                    Job job = jobIter.next();
                    if (job.getState() == Job.State.PENDING_LAUNCH) {
                        LOG.warn("Removing job because it was left in state {} from a previous run of the scheduler: {} .", Job.State.PENDING_LAUNCH, job);
                        jobQueue.deleteJob(job);
                    }
                }
            }
        });
    } catch (TransactionFailureException exception) {
        LOG.warn("Failed to cleanup jobs upon startup.", exception);
    }
}