Examples with Private org.apache.hadoop.classification.InterfaceAudience.Private used on opensource projects

Example 31 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project hadoop by apache.

the class RMContainerAllocator method getResourceLimit.

@Private
public Resource getResourceLimit() {
    Resource headRoom = getAvailableResources();
    Resource assignedMapResource = Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
    Resource assignedReduceResource = Resources.multiply(reduceResourceRequest, assignedRequests.reduces.size());
    return Resources.add(headRoom, Resources.add(assignedMapResource, assignedReduceResource));
}

Example 32 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project hadoop by apache.

the class FileOutputFormat method setWorkOutputPath.

/**
   * Set the {@link Path} of the task's temporary output directory 
   * for the map-reduce job.
   * 
   * <p><i>Note</i>: Task output path is set by the framework.
   * </p>
   * @param conf The configuration of the job.
   * @param outputDir the {@link Path} of the output directory 
   * for the map-reduce job.
   */
@Private
public static void setWorkOutputPath(JobConf conf, Path outputDir) {
    outputDir = new Path(conf.getWorkingDirectory(), outputDir);
    conf.set(JobContext.TASK_OUTPUT_DIR, outputDir.toString());
}

Example 33 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project hadoop by apache.

the class JobConf method parseMaximumHeapSizeMB.

/**
   * Parse the Maximum heap size from the java opts as specified by the -Xmx option
   * Format: -Xmx<size>[g|G|m|M|k|K]
   * @param javaOpts String to parse to read maximum heap size
   * @return Maximum heap size in MB or -1 if not specified
   */
@Private
@VisibleForTesting
public static int parseMaximumHeapSizeMB(String javaOpts) {
    // Find the last matching -Xmx following word boundaries
    Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts);
    if (m.matches()) {
        long size = Long.parseLong(m.group(1));
        if (size <= 0) {
            return -1;
        }
        if (m.group(2).isEmpty()) {
            // -Xmx specified in bytes
            return (int) (size / (1024 * 1024));
        }
        char unit = m.group(2).charAt(0);
        switch(unit) {
            case 'g':
            case 'G':
                // -Xmx specified in GB
                return (int) (size * 1024);
            case 'm':
            case 'M':
                // -Xmx specified in MB
                return (int) size;
            case 'k':
            case 'K':
                // -Xmx specified in KB
                return (int) (size / 1024);
        }
    }
    // -Xmx not specified
    return -1;
}

Example 34 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project hadoop by apache.

the class FileOutputCommitter method commitTask.

@Private
public void commitTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    TaskAttemptID attemptId = context.getTaskAttemptID();
    if (hasOutputPath()) {
        context.progress();
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        FileStatus taskAttemptDirStatus;
        try {
            taskAttemptDirStatus = fs.getFileStatus(taskAttemptPath);
        } catch (FileNotFoundException e) {
            taskAttemptDirStatus = null;
        }
        if (taskAttemptDirStatus != null) {
            if (algorithmVersion == 1) {
                Path committedTaskPath = getCommittedTaskPath(context);
                if (fs.exists(committedTaskPath)) {
                    if (!fs.delete(committedTaskPath, true)) {
                        throw new IOException("Could not delete " + committedTaskPath);
                    }
                }
                if (!fs.rename(taskAttemptPath, committedTaskPath)) {
                    throw new IOException("Could not rename " + taskAttemptPath + " to " + committedTaskPath);
                }
                LOG.info("Saved output of task '" + attemptId + "' to " + committedTaskPath);
            } else {
                // directly merge everything from taskAttemptPath to output directory
                mergePaths(fs, taskAttemptDirStatus, outputPath);
                LOG.info("Saved output of task '" + attemptId + "' to " + outputPath);
            }
        } else {
            LOG.warn("No Output found for " + attemptId);
        }
    } else {
        LOG.warn("Output Path is null in commitTask()");
    }
}

Example 35 with Private

use of org.apache.hadoop.classification.InterfaceAudience.Private in project hadoop by apache.

the class FileOutputCommitter method abortTask.

@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
    if (hasOutputPath()) {
        context.progress();
        if (taskAttemptPath == null) {
            taskAttemptPath = getTaskAttemptPath(context);
        }
        FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
        if (!fs.delete(taskAttemptPath, true)) {
            LOG.warn("Could not delete " + taskAttemptPath);
        }
    } else {
        LOG.warn("Output Path is null in abortTask()");
    }
}