Examples with VisibleForTesting org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 21 with VisibleForTesting

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting in project beam by apache.

the class NameUtils method approximateSimpleName.

@VisibleForTesting
static String approximateSimpleName(String fullName, boolean dropOuterClassNames) {
    String shortName = fullName.substring(fullName.lastIndexOf('.') + 1);
    // Drop common suffixes for each named component.
    String[] names = shortName.split("\\$");
    for (int i = 0; i < names.length; i++) {
        names[i] = simplifyNameComponent(names[i]);
    }
    shortName = Joiner.on('$').join(names);
    if (dropOuterClassNames) {
        // Simplify inner class name by dropping outer class prefixes.
        Matcher m = NAMED_INNER_CLASS.matcher(shortName);
        if (m.matches()) {
            shortName = m.group("INNER");
        }
    } else {
        // Dropping anonymous outer classes
        shortName = shortName.replaceAll(ANONYMOUS_CLASS_REGEX, ".");
        shortName = shortName.replaceAll("\\$", ".");
    }
    return shortName;
}

Example 22 with VisibleForTesting

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting in project beam by apache.

the class NumberedShardedFile method checkTotalNumOfFiles.

/**
 * Check if total number of files is correct by comparing with the number that is parsed from
 * shard name using a name template. If no template is specified, "SSSS-of-NNNN" will be used as
 * default, and "NNNN" will be the expected total number of files.
 *
 * @return {@code true} if at least one shard name matches template and total number of given
 *     files equals the number that is parsed from shard name.
 */
@VisibleForTesting
boolean checkTotalNumOfFiles(Collection<Metadata> files) {
    for (Metadata fileMedadata : files) {
        String fileName = fileMedadata.resourceId().getFilename();
        if (fileName == null) {
            // this path has zero elements
            continue;
        }
        Matcher matcher = shardTemplate.matcher(fileName);
        if (!matcher.matches()) {
            // shard name doesn't match the pattern, check with the next shard
            continue;
        }
        // once match, extract total number of shards and compare to file list
        return files.size() == Integer.parseInt(matcher.group("numshards"));
    }
    return false;
}

Example 23 with VisibleForTesting

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting in project beam by apache.

the class NumberedShardedFile method readLines.

/**
 * Reads all the lines of all the files.
 *
 * <p>Not suitable for use except in testing of small data, since the data size may be far more
 * than can be reasonably processed serially, in-memory, by a single thread.
 */
@VisibleForTesting
List<String> readLines(Collection<Metadata> files) throws IOException {
    List<String> allLines = Lists.newArrayList();
    int i = 1;
    for (Metadata file : files) {
        try (Reader reader = Channels.newReader(FileSystems.open(file.resourceId()), StandardCharsets.UTF_8.name())) {
            List<String> lines = CharStreams.readLines(reader);
            allLines.addAll(lines);
            LOG.debug("[{} of {}] Read {} lines from file: {}", i, files.size(), lines.size(), file);
        }
        i++;
    }
    return allLines;
}

Example 24 with VisibleForTesting

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting in project beam by apache.

the class AzureBlobStoreFileSystem method expand.

/**
 * Expands a pattern into {@link MatchResult}.
 */
@VisibleForTesting
MatchResult expand(AzfsResourceId azfsPattern) {
    checkArgument(azfsPattern.isWildcard(), "The resource id should be a wildcard.");
    String blobPrefix = azfsPattern.getBlobNonWildcardPrefix();
    Pattern wildcardAsRegexp = Pattern.compile(wildcardToRegexp(azfsPattern.getBlob()));
    LOG.debug("matching files in container {}, prefix {} against pattern {}", azfsPattern.getContainer(), blobPrefix, wildcardAsRegexp.toString());
    ListBlobsOptions listOptions = new ListBlobsOptions().setPrefix(blobPrefix);
    Duration timeout = Duration.ofMinutes(1);
    String account = azfsPattern.getAccount();
    String container = azfsPattern.getContainer();
    BlobContainerClient blobContainerClient = client.get().getBlobContainerClient(container);
    PagedIterable<BlobItem> blobs = blobContainerClient.listBlobs(listOptions, timeout);
    List<MatchResult.Metadata> results = new ArrayList<>();
    blobs.forEach(blob -> {
        String name = blob.getName();
        if (wildcardAsRegexp.matcher(name).matches() && !name.endsWith("/")) {
            LOG.debug("Matched object: azfs://{}/{}/{}", account, container, name);
            BlobProperties properties = blobContainerClient.getBlobClient(name).getProperties();
            AzfsResourceId rid = AzfsResourceId.fromComponents(account, container, name).withSize(properties.getBlobSize()).withLastModified(Date.from(properties.getLastModified().toInstant()));
            results.add(toMetadata(rid, properties.getContentEncoding(), properties.getETag()));
        }
    });
    return MatchResult.create(MatchResult.Status.OK, results);
}

Example 25 with VisibleForTesting

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting in project beam by apache.

the class AzureBlobStoreFileSystem method copy.

@VisibleForTesting
void copy(AzfsResourceId sourcePath, AzfsResourceId destinationPath) throws IOException {
    checkArgument(sourcePath.getBlob() != null && destinationPath.getBlob() != null, "This method is intended to copy file-like resources, not directories.");
    // get source blob client
    BlobClient srcBlobClient = client.get().getBlobContainerClient(sourcePath.getContainer()).getBlobClient(sourcePath.getBlob());
    if (!srcBlobClient.exists()) {
        throw new FileNotFoundException("The copy source does not exist.");
    }
    // get destination blob client
    BlobContainerClient destBlobContainerClient = client.get().getBlobContainerClient(destinationPath.getContainer());
    if (!destBlobContainerClient.exists()) {
        client.get().createBlobContainer(destinationPath.getContainer());
        LOG.info("Created a container called {}", destinationPath.getContainer());
    }
    BlobClient destBlobClient = destBlobContainerClient.getBlobClient(destinationPath.getBlob());
    destBlobClient.copyFromUrl(srcBlobClient.getBlobUrl() + generateSasToken());
}