Examples with VisibleForTesting org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 81 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class HadoopArchiveLogs method generateScript.

/*
  The generated script looks like this:
  #!/bin/bash
  set -e
  set -x
  if [ "$YARN_SHELL_ID" == "1" ]; then
        appId="application_1440448768987_0001"
        user="rkanter"
  elif [ "$YARN_SHELL_ID" == "2" ]; then
        appId="application_1440448768987_0002"
        user="rkanter"
  else
        echo "Unknown Mapping!"
        exit 1
  fi
  export HADOOP_CLIENT_OPTS="-Xmx1024m"
  export HADOOP_CLASSPATH=/dist/share/hadoop/tools/lib/hadoop-archive-logs-2.8.0-SNAPSHOT.jar:/dist/share/hadoop/tools/lib/hadoop-archives-2.8.0-SNAPSHOT.jar
  "$HADOOP_HOME"/bin/hadoop org.apache.hadoop.tools.HadoopArchiveLogsRunner -appId "$appId" -user "$user" -workingDir /tmp/logs/archive-logs-work -remoteRootLogDir /tmp/logs -suffix logs
   */
@VisibleForTesting
void generateScript(File localScript, Path workingDir, Path remoteRootLogDir, String suffix) throws IOException {
    if (verbose) {
        LOG.info("Generating script at: " + localScript.getAbsolutePath());
    }
    String halrJarPath = HadoopArchiveLogsRunner.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    String harJarPath = HadoopArchives.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    String classpath = halrJarPath + File.pathSeparator + harJarPath;
    FileWriterWithEncoding fw = null;
    try {
        fw = new FileWriterWithEncoding(localScript, "UTF-8");
        fw.write("#!/bin/bash\nset -e\nset -x\n");
        int containerCount = 1;
        for (AppInfo app : eligibleApplications) {
            fw.write("if [ \"$YARN_SHELL_ID\" == \"");
            fw.write(Integer.toString(containerCount));
            fw.write("\" ]; then\n\tappId=\"");
            fw.write(app.getAppId());
            fw.write("\"\n\tuser=\"");
            fw.write(app.getUser());
            fw.write("\"\nel");
            containerCount++;
        }
        fw.write("se\n\techo \"Unknown Mapping!\"\n\texit 1\nfi\n");
        fw.write("export HADOOP_CLIENT_OPTS=\"-Xmx");
        fw.write(Long.toString(memory));
        fw.write("m\"\n");
        fw.write("export HADOOP_CLASSPATH=");
        fw.write(classpath);
        fw.write("\n\"$HADOOP_HOME\"/bin/hadoop ");
        fw.write(HadoopArchiveLogsRunner.class.getName());
        fw.write(" -appId \"$appId\" -user \"$user\" -workingDir ");
        fw.write(workingDir.toString());
        fw.write(" -remoteRootLogDir ");
        fw.write(remoteRootLogDir.toString());
        fw.write(" -suffix ");
        fw.write(suffix);
        if (!proxy) {
            fw.write(" -noProxy\n");
        }
        fw.write("\n");
    } finally {
        if (fw != null) {
            fw.close();
        }
    }
}

Example 82 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class HadoopArchiveLogs method filterAppsByAggregatedStatus.

@VisibleForTesting
void filterAppsByAggregatedStatus() throws IOException, YarnException {
    YarnClient client = YarnClient.createYarnClient();
    try {
        client.init(getConf());
        client.start();
        for (Iterator<AppInfo> it = eligibleApplications.iterator(); it.hasNext(); ) {
            AppInfo app = it.next();
            try {
                ApplicationReport report = client.getApplicationReport(ApplicationId.fromString(app.getAppId()));
                LogAggregationStatus aggStatus = report.getLogAggregationStatus();
                if (aggStatus.equals(LogAggregationStatus.RUNNING) || aggStatus.equals(LogAggregationStatus.RUNNING_WITH_FAILURE) || aggStatus.equals(LogAggregationStatus.NOT_START) || aggStatus.equals(LogAggregationStatus.DISABLED) || aggStatus.equals(LogAggregationStatus.FAILED)) {
                    if (verbose) {
                        LOG.info("Skipping " + app.getAppId() + " due to aggregation status being " + aggStatus);
                    }
                    it.remove();
                } else {
                    if (verbose) {
                        LOG.info(app.getAppId() + " has aggregation status " + aggStatus);
                    }
                    app.setFinishTime(report.getFinishTime());
                }
            } catch (ApplicationNotFoundException e) {
                // Assume the aggregation has finished
                if (verbose) {
                    LOG.info(app.getAppId() + " not in the ResourceManager");
                }
            }
        }
    } finally {
        if (client != null) {
            client.stop();
        }
    }
}

Example 83 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class HadoopArchiveLogs method prepareWorkingDir.

@VisibleForTesting
boolean prepareWorkingDir(FileSystem fs, Path workingDir) throws IOException {
    if (fs.exists(workingDir)) {
        if (force) {
            LOG.info("Existing Working Dir detected: -" + FORCE_OPTION + " specified -> recreating Working Dir");
            fs.delete(workingDir, true);
        } else {
            LOG.info("Existing Working Dir detected: -" + FORCE_OPTION + " not specified -> exiting");
            return false;
        }
    }
    fs.mkdirs(workingDir);
    fs.setPermission(workingDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true));
    return true;
}

Example 84 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class RetriableFileCopyCommand method copyBytes.

@VisibleForTesting
long copyBytes(CopyListingFileStatus source2, long sourceOffset, OutputStream outStream, int bufferSize, Mapper.Context context) throws IOException {
    Path source = source2.getPath();
    byte[] buf = new byte[bufferSize];
    ThrottledInputStream inStream = null;
    long totalBytesRead = 0;
    try {
        inStream = getInputStream(source, context.getConfiguration());
        int bytesRead = readBytes(inStream, buf, sourceOffset);
        while (bytesRead >= 0) {
            totalBytesRead += bytesRead;
            if (action == FileAction.APPEND) {
                sourceOffset += bytesRead;
            }
            outStream.write(buf, 0, bytesRead);
            updateContextStatus(totalBytesRead, context, source2);
            bytesRead = readBytes(inStream, buf, sourceOffset);
        }
        outStream.close();
        outStream = null;
    } finally {
        IOUtils.cleanup(LOG, outStream, inStream);
    }
    return totalBytesRead;
}

Example 85 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class SimpleCopyListing method doBuildListingWithSnapshotDiff.

/**
   * Build a copy list based on the snapshot diff report.
   *
   * Any file/directory changed or created will be in the list. Deleted
   * files/directories will not be in the list, since they are handled by
   * {@link org.apache.hadoop.tools.DistCpSync#sync}. An item can be
   * created/modified and renamed, in which case, the target path is put
   * into the list.
   * @throws IOException
   */
@VisibleForTesting
protected void doBuildListingWithSnapshotDiff(SequenceFile.Writer fileListWriter, DistCpOptions options) throws IOException {
    ArrayList<DiffInfo> diffList = distCpSync.prepareDiffListForCopyListing();
    Path sourceRoot = options.getSourcePaths().get(0);
    FileSystem sourceFS = sourceRoot.getFileSystem(getConf());
    try {
        List<FileStatusInfo> fileStatuses = Lists.newArrayList();
        for (DiffInfo diff : diffList) {
            // add snapshot paths prefix
            diff.setTarget(new Path(options.getSourcePaths().get(0), diff.getTarget()));
            if (diff.getType() == SnapshotDiffReport.DiffType.MODIFY) {
                addToFileListing(fileListWriter, sourceRoot, diff.getTarget(), options);
            } else if (diff.getType() == SnapshotDiffReport.DiffType.CREATE) {
                addToFileListing(fileListWriter, sourceRoot, diff.getTarget(), options);
                FileStatus sourceStatus = sourceFS.getFileStatus(diff.getTarget());
                if (sourceStatus.isDirectory()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath());
                    }
                    HashSet<String> excludeList = distCpSync.getTraverseExcludeList(diff.getSource(), options.getSourcePaths().get(0));
                    ArrayList<FileStatus> sourceDirs = new ArrayList<>();
                    sourceDirs.add(sourceStatus);
                    traverseDirectory(fileListWriter, sourceFS, sourceDirs, sourceRoot, options, excludeList, fileStatuses);
                }
            }
        }
        if (randomizeFileListing) {
            writeToFileListing(fileStatuses, fileListWriter);
        }
        fileListWriter.close();
        fileListWriter = null;
    } finally {
        IOUtils.cleanup(LOG, fileListWriter);
    }
}