Examples with Processor co.cask.cdap.common.io.Processor used on opensource projects

Example 1 with Processor

use of co.cask.cdap.common.io.Processor in project cdap by caskdata.

the class StreamUtils method fetchStreamFilesSize.

/**
   * Get the size of the data persisted for the stream under the given stream location.
   *
   * @param streamLocation stream to get data size of
   * @return the size of the data persisted for the stream which config is the {@code streamName}
   * @throws IOException in case of any error in fetching the size
   */
public static long fetchStreamFilesSize(Location streamLocation) throws IOException {
    Processor<LocationStatus, Long> processor = new Processor<LocationStatus, Long>() {

        private long size = 0;

        @Override
        public boolean process(LocationStatus input) {
            if (!input.isDir() && StreamFileType.EVENT.isMatched(input.getUri().getPath())) {
                size += input.getLength();
            }
            return true;
        }

        @Override
        public Long getResult() {
            return size;
        }
    };
    List<Location> locations = streamLocation.list();
    // All directories are partition directories
    for (Location location : locations) {
        if (!location.isDirectory() || !isPartition(location.getName())) {
            continue;
        }
        Locations.processLocations(location, false, processor);
    }
    return processor.getResult();
}

Example 2 with Processor

use of co.cask.cdap.common.io.Processor in project cdap by caskdata.

the class SparkCredentialsUpdater method cleanup.

private void cleanup() {
    try {
        // Locations.processLocations is more efficient in getting the location last modified time
        Locations.processLocations(credentialsDir, false, new Processor<LocationStatus, Runnable>() {

            private final List<LocationStatus> cleanupFiles = new ArrayList<>();

            @Override
            public boolean process(LocationStatus status) {
                if (!status.isDir()) {
                    cleanupFiles.add(status);
                }
                return true;
            }

            @Override
            public Runnable getResult() {
                return new Runnable() {

                    @Override
                    public void run() {
                        if (cleanupFiles.size() <= minFilesToKeep) {
                            return;
                        }
                        // Sort the list of files in descending order of last modified time.
                        Collections.sort(cleanupFiles, new Comparator<LocationStatus>() {

                            @Override
                            public int compare(LocationStatus o1, LocationStatus o2) {
                                return Long.compare(o2.getLastModified(), o1.getLastModified());
                            }
                        });
                        final long thresholdTime = System.currentTimeMillis() - cleanupExpireMs;
                        // Remove the tail of the list, keep the top "minFilesToKeep"
                        for (LocationStatus locationStatus : cleanupFiles.subList(minFilesToKeep, cleanupFiles.size())) {
                            if (locationStatus.getLastModified() >= thresholdTime) {
                                continue;
                            }
                            Location location = credentialsDir.getLocationFactory().create(locationStatus.getUri());
                            try {
                                location.delete();
                                LOG.debug("Removed old credential file {}", location);
                            } catch (Exception e) {
                                LOG.warn("Failed to cleanup old credential file {}", location, e);
                            }
                        }
                    }
                };
            }
        }).run();
    } catch (Exception e) {
        LOG.warn("Exception raised when cleaning up credential files in {}", credentialsDir, e);
    }
}