Examples with VisibleForTesting org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 76 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hbase by apache.

the class FSTableDescriptors method getTableInfoSequenceId.

/**
   * @param p Path to a <code>.tableinfo</code> file.
   * @return The current editid or 0 if none found.
   */
@VisibleForTesting
static int getTableInfoSequenceId(final Path p) {
    if (p == null)
        return 0;
    Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName());
    if (!m.matches())
        throw new IllegalArgumentException(p.toString());
    String suffix = m.group(2);
    if (suffix == null || suffix.length() <= 0)
        return 0;
    return Integer.parseInt(m.group(2));
}

Example 77 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hbase by apache.

the class IdReadWriteLock method waitForWaiters.

@VisibleForTesting
public void waitForWaiters(long id, int numWaiters) throws InterruptedException {
    for (ReentrantReadWriteLock readWriteLock; ; ) {
        readWriteLock = lockPool.get(id);
        if (readWriteLock != null) {
            synchronized (readWriteLock) {
                if (readWriteLock.getQueueLength() >= numWaiters) {
                    return;
                }
            }
        }
        Thread.sleep(50);
    }
}

Example 78 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class Fetcher method copyFromHost.

/**
   * The crux of the matter...
   * 
   * @param host {@link MapHost} from which we need to  
   *              shuffle available map-outputs.
   */
@VisibleForTesting
protected void copyFromHost(MapHost host) throws IOException {
    // reset retryStartTime for a new host
    retryStartTime = 0;
    // Get completed maps on 'host'
    List<TaskAttemptID> maps = scheduler.getMapsForHost(host);
    // especially at the tail of large jobs
    if (maps.size() == 0) {
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + maps);
    }
    // List of maps to be fetched yet
    Set<TaskAttemptID> remaining = new HashSet<TaskAttemptID>(maps);
    // Construct the url and connect
    URL url = getMapOutputURL(host, maps);
    DataInputStream input = openShuffleUrl(host, remaining, url);
    if (input == null) {
        return;
    }
    try {
        // Loop through available map-outputs and fetch them
        // On any error, faildTasks is not null and we exit
        // after putting back the remaining maps to the 
        // yet_to_be_fetched list and marking the failed tasks.
        TaskAttemptID[] failedTasks = null;
        while (!remaining.isEmpty() && failedTasks == null) {
            try {
                failedTasks = copyMapOutput(host, input, remaining, fetchRetryEnabled);
            } catch (IOException e) {
                IOUtils.cleanup(LOG, input);
                //
                // Setup connection again if disconnected by NM
                connection.disconnect();
                // Get map output from remaining tasks only.
                url = getMapOutputURL(host, remaining);
                input = openShuffleUrl(host, remaining, url);
                if (input == null) {
                    return;
                }
            }
        }
        if (failedTasks != null && failedTasks.length > 0) {
            LOG.warn("copyMapOutput failed for tasks " + Arrays.toString(failedTasks));
            scheduler.hostFailed(host.getHostName());
            for (TaskAttemptID left : failedTasks) {
                scheduler.copyFailed(left, host, true, false);
            }
        }
        // Sanity check
        if (failedTasks == null && !remaining.isEmpty()) {
            throw new IOException("server didn't return all expected map outputs: " + remaining.size() + " left.");
        }
        input.close();
        input = null;
    } finally {
        if (input != null) {
            IOUtils.cleanup(LOG, input);
            input = null;
        }
        for (TaskAttemptID left : remaining) {
            scheduler.putBackKnownMapOutput(host, left);
        }
    }
}

Example 79 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class Fetcher method openConnection.

@VisibleForTesting
protected synchronized void openConnection(URL url) throws IOException {
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    if (sslShuffle) {
        HttpsURLConnection httpsConn = (HttpsURLConnection) conn;
        try {
            httpsConn.setSSLSocketFactory(sslFactory.createSSLSocketFactory());
        } catch (GeneralSecurityException ex) {
            throw new IOException(ex);
        }
        httpsConn.setHostnameVerifier(sslFactory.getHostnameVerifier());
    }
    connection = conn;
}

Example 80 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class FileOutputCommitter method commitJobInternal.

/**
   * The job has completed, so do following commit job, include:
   * Move all committed tasks to the final output dir (algorithm 1 only).
   * Delete the temporary directory, including all of the work directories.
   * Create a _SUCCESS file to make it as successful.
   * @param context the job's context
   */
@VisibleForTesting
protected void commitJobInternal(JobContext context) throws IOException {
    if (hasOutputPath()) {
        Path finalOutput = getOutputPath();
        FileSystem fs = finalOutput.getFileSystem(context.getConfiguration());
        if (algorithmVersion == 1) {
            for (FileStatus stat : getAllCommittedTaskPaths(context)) {
                mergePaths(fs, stat, finalOutput);
            }
        }
        if (skipCleanup) {
            LOG.info("Skip cleanup the _temporary folders under job's output " + "directory in commitJob.");
        } else {
            // folder
            try {
                cleanupJob(context);
            } catch (IOException e) {
                if (ignoreCleanupFailures) {
                    // swallow exceptions in cleanup as user configure to make sure
                    // commitJob could be success even when cleanup get failure.
                    LOG.error("Error in cleanup job, manually cleanup is needed.", e);
                } else {
                    // throw back exception to fail commitJob.
                    throw e;
                }
            }
        }
        // Note that by default it is set to true.
        if (context.getConfiguration().getBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true)) {
            Path markerPath = new Path(outputPath, SUCCEEDED_FILE_NAME);
            // in case other FS implementations don't overwritten by default.
            if (isCommitJobRepeatable(context)) {
                fs.create(markerPath, true).close();
            } else {
                fs.create(markerPath).close();
            }
        }
    } else {
        LOG.warn("Output Path is null in commitJob()");
    }
}