Examples with VisibleForTesting org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 86 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class SimpleCopyListing method doBuildListing.

/**
   * Collect the list of 
   *   {@literal <sourceRelativePath, sourceFileStatus>}
   * to be copied and write to the sequence file. In essence, any file or
   * directory that need to be copied or sync-ed is written as an entry to the
   * sequence file, with the possible exception of the source root:
   *     when either -update (sync) or -overwrite switch is specified, and if
   *     the the source root is a directory, then the source root entry is not 
   *     written to the sequence file, because only the contents of the source
   *     directory need to be copied in this case.
   * See {@link org.apache.hadoop.tools.util.DistCpUtils#getRelativePath} for
   *     how relative path is computed.
   * See computeSourceRootPath method for how the root path of the source is
   *     computed.
   * @param fileListWriter
   * @param options
   * @throws IOException
   */
@VisibleForTesting
protected void doBuildListing(SequenceFile.Writer fileListWriter, DistCpOptions options) throws IOException {
    if (options.getNumListstatusThreads() > 0) {
        numListstatusThreads = options.getNumListstatusThreads();
    }
    try {
        List<FileStatusInfo> statusList = Lists.newArrayList();
        for (Path path : options.getSourcePaths()) {
            FileSystem sourceFS = path.getFileSystem(getConf());
            final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
            final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
            final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs();
            path = makeQualified(path);
            FileStatus rootStatus = sourceFS.getFileStatus(path);
            Path sourcePathRoot = computeSourceRootPath(rootStatus, options);
            FileStatus[] sourceFiles = sourceFS.listStatus(path);
            boolean explore = (sourceFiles != null && sourceFiles.length > 0);
            if (!explore || rootStatus.isDirectory()) {
                CopyListingFileStatus rootCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, preserveAcls, preserveXAttrs, preserveRawXAttrs);
                writeToFileListingRoot(fileListWriter, rootCopyListingStatus, sourcePathRoot, options);
            }
            if (explore) {
                ArrayList<FileStatus> sourceDirs = new ArrayList<FileStatus>();
                for (FileStatus sourceStatus : sourceFiles) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
                    }
                    CopyListingFileStatus sourceCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, preserveAcls && sourceStatus.isDirectory(), preserveXAttrs && sourceStatus.isDirectory(), preserveRawXAttrs && sourceStatus.isDirectory());
                    if (randomizeFileListing) {
                        addToFileListing(statusList, new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot), fileListWriter);
                    } else {
                        writeToFileListing(fileListWriter, sourceCopyListingStatus, sourcePathRoot);
                    }
                    if (sourceStatus.isDirectory()) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath());
                        }
                        sourceDirs.add(sourceStatus);
                    }
                }
                traverseDirectory(fileListWriter, sourceFS, sourceDirs, sourcePathRoot, options, null, statusList);
            }
        }
        if (randomizeFileListing) {
            writeToFileListing(statusList, fileListWriter);
        }
        fileListWriter.close();
        printStats();
        LOG.info("Build file listing completed.");
        fileListWriter = null;
    } finally {
        IOUtils.cleanup(LOG, fileListWriter);
    }
}

Example 87 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class ContainerManagerImpl method authorizeGetAndStopContainerRequest.

@Private
@VisibleForTesting
protected void authorizeGetAndStopContainerRequest(ContainerId containerId, Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException {
    if (identifier == null) {
        throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
    }
    /*
     * For get/stop container status; we need to verify that 1) User (NMToken)
     * application attempt only has started container. 2) Requested containerId
     * belongs to the same application attempt (NMToken) which was used. (Note:-
     * This will prevent user in knowing another application's containers).
     */
    ApplicationId nmTokenAppId = identifier.getApplicationAttemptId().getApplicationId();
    if ((!nmTokenAppId.equals(containerId.getApplicationAttemptId().getApplicationId())) || (container != null && !nmTokenAppId.equals(container.getContainerId().getApplicationAttemptId().getApplicationId()))) {
        String msg;
        if (stopRequest) {
            msg = identifier.getApplicationAttemptId() + " attempted to stop non-application container : " + containerId;
            NMAuditLogger.logFailure("UnknownUser", AuditConstants.STOP_CONTAINER, "ContainerManagerImpl", "Trying to stop unknown container!", nmTokenAppId, containerId);
        } else {
            msg = identifier.getApplicationAttemptId() + " attempted to get status for non-application container : " + containerId;
        }
        LOG.warn(msg);
        throw RPCUtil.getRemoteException(msg);
    }
}

Example 88 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class ContainerManagerImpl method createNMTimelinePublisher.

@VisibleForTesting
protected NMTimelinePublisher createNMTimelinePublisher(Context ctxt) {
    NMTimelinePublisher nmTimelinePublisherLocal = new NMTimelinePublisher(ctxt);
    addIfService(nmTimelinePublisherLocal);
    return nmTimelinePublisherLocal;
}

Example 89 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class SharedCacheUploader method getActualPath.

@VisibleForTesting
Path getActualPath() throws IOException {
    Path path = localPath;
    FileStatus status = localFs.getFileStatus(path);
    if (status != null && status.isDirectory()) {
        // for certain types of resources that get unpacked, the original file may
        // be found under the directory with the same name (see
        // FSDownload.unpack); check if the path is a directory and if so look
        // under it
        path = new Path(path, path.getName());
    }
    return path;
}

Example 90 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class SharedCacheUploader method notifySharedCacheManager.

@VisibleForTesting
boolean notifySharedCacheManager(String checksumVal, String fileName) throws IOException {
    try {
        SCMUploaderNotifyRequest request = recordFactory.newRecordInstance(SCMUploaderNotifyRequest.class);
        request.setResourceKey(checksumVal);
        request.setFilename(fileName);
        return scmClient.notify(request).getAccepted();
    } catch (YarnException e) {
        throw new IOException(e);
    } catch (UndeclaredThrowableException e) {
        // retrieve the cause of the exception and throw it as an IOException
        throw new IOException(e.getCause() == null ? e : e.getCause());
    }
}