Examples with VisibleForTesting org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 56 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class FadvisedFileRegion method customShuffleTransfer.

/**
   * This method transfers data using local buffer. It transfers data from 
   * a disk to a local buffer in memory, and then it transfers data from the 
   * buffer to the target. This is used only if transferTo is disallowed in
   * the configuration file. super.TransferTo does not perform well on Windows 
   * due to a small IO request generated. customShuffleTransfer can control 
   * the size of the IO requests by changing the size of the intermediate 
   * buffer.
   */
@VisibleForTesting
long customShuffleTransfer(WritableByteChannel target, long position) throws IOException {
    long actualCount = this.count - position;
    if (actualCount < 0 || position < 0) {
        throw new IllegalArgumentException("position out of range: " + position + " (expected: 0 - " + (this.count - 1) + ')');
    }
    if (actualCount == 0) {
        return 0L;
    }
    long trans = actualCount;
    int readSize;
    ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize);
    while (trans > 0L && (readSize = fileChannel.read(byteBuffer, this.position + position)) > 0) {
        //adjust counters and buffer limit
        if (readSize < trans) {
            trans -= readSize;
            position += readSize;
            byteBuffer.flip();
        } else {
            //We can read more than we need if the actualCount is not multiple 
            //of the byteBuffer size and file is big enough. In that case we cannot
            //use flip method but we need to set buffer limit manually to trans.
            byteBuffer.limit((int) trans);
            byteBuffer.position(0);
            position += trans;
            trans = 0;
        }
        //write data to the target
        while (byteBuffer.hasRemaining()) {
            target.write(byteBuffer);
        }
        byteBuffer.clear();
    }
    return actualCount - trans;
}

Example 57 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class LogsCLI method createYarnClient.

@VisibleForTesting
protected YarnClient createYarnClient() {
    YarnClient client = YarnClient.createYarnClient();
    client.init(getConf());
    client.start();
    return client;
}

Example 58 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class NodeStatusUpdaterImpl method getContainerStatuses.

// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
    List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
    for (Container container : this.context.getContainers().values()) {
        ContainerId containerId = container.getContainerId();
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
        if (containerStatus.getState() == ContainerState.COMPLETE) {
            if (isApplicationStopped(applicationId)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(applicationId + " is completing, " + " remove " + containerId + " from NM context.");
                }
                context.getContainers().remove(containerId);
                pendingCompletedContainers.put(containerId, containerStatus);
            } else {
                if (!isContainerRecentlyStopped(containerId)) {
                    pendingCompletedContainers.put(containerId, containerStatus);
                }
            }
            // Adding to finished containers cache. Cache will keep it around at
            // least for #durationToTrackStoppedContainers duration. In the
            // subsequent call to stop container it will get removed from cache.
            addCompletedContainer(containerId);
        } else {
            containerStatuses.add(containerStatus);
        }
    }
    containerStatuses.addAll(pendingCompletedContainers.values());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Sending out " + containerStatuses.size() + " container statuses: " + containerStatuses);
    }
    return containerStatuses;
}

Example 59 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class CapacityScheduler method getUserGroupMappingPlacementRule.

@VisibleForTesting
public UserGroupMappingPlacementRule getUserGroupMappingPlacementRule() throws IOException {
    try {
        readLock.lock();
        boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
        LOG.info("Initialized queue mappings, override: " + overrideWithQueueMappings);
        // Get new user/group mappings
        List<QueueMapping> newMappings = conf.getQueueMappings();
        // check if mappings refer to valid queues
        for (QueueMapping mapping : newMappings) {
            String mappingQueue = mapping.getQueue();
            if (!mappingQueue.equals(UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mappingQueue.equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) {
                CSQueue queue = getQueue(mappingQueue);
                if (queue == null || !(queue instanceof LeafQueue)) {
                    throw new IOException("mapping contains invalid or non-leaf queue " + mappingQueue);
                }
            }
        }
        // initialize groups if mappings are present
        if (newMappings.size() > 0) {
            Groups groups = new Groups(conf);
            return new UserGroupMappingPlacementRule(overrideWithQueueMappings, newMappings, groups);
        }
        return null;
    } finally {
        readLock.unlock();
    }
}

Example 60 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class HistoryFileManager method tryCreatingHistoryDirs.

/**
   * Returns TRUE if the history dirs were created, FALSE if they could not
   * be created because the FileSystem is not reachable or in safe mode and
   * throws and exception otherwise.
   */
@VisibleForTesting
boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
    boolean succeeded = true;
    String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    try {
        doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
        doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
        doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
        mkdir(doneDirFc, doneDirPrefixPath, new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
    } catch (ConnectException ex) {
        if (logWait) {
            LOG.info("Waiting for FileSystem at " + doneDirPrefixPath.toUri().getAuthority() + "to be available");
        }
        succeeded = false;
    } catch (IOException e) {
        if (isNameNodeStillNotStarted(e)) {
            succeeded = false;
            if (logWait) {
                LOG.info("Waiting for FileSystem at " + doneDirPrefixPath.toUri().getAuthority() + "to be out of safe mode");
            }
        } else {
            throw new YarnRuntimeException("Error creating done directory: [" + doneDirPrefixPath + "]", e);
        }
    }
    if (succeeded) {
        String intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
        try {
            intermediateDoneDirPath = FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
            intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
            mkdir(intermediateDoneDirFc, intermediateDoneDirPath, new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
        } catch (ConnectException ex) {
            succeeded = false;
            if (logWait) {
                LOG.info("Waiting for FileSystem at " + intermediateDoneDirPath.toUri().getAuthority() + "to be available");
            }
        } catch (IOException e) {
            if (isNameNodeStillNotStarted(e)) {
                succeeded = false;
                if (logWait) {
                    LOG.info("Waiting for FileSystem at " + intermediateDoneDirPath.toUri().getAuthority() + "to be out of safe mode");
                }
            } else {
                throw new YarnRuntimeException("Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
            }
        }
    }
    return succeeded;
}