Examples with LineIterator org.apache.commons.io.LineIterator used on opensource projects

Example 21 with LineIterator

use of org.apache.commons.io.LineIterator in project druid by druid-io.

the class StaticCloudFilesFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException {
    Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi");
    final LinkedList<CloudFilesBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final CloudFilesBlob nextURI = objectQueue.poll();
            final String region = nextURI.getRegion();
            final String container = nextURI.getContainer();
            final String path = nextURI.getPath();
            log.info("Retrieving file from region[%s], container[%s] and path [%s]", region, container, path);
            CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy(cloudFilesApi, region, container);
            final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path);
            try {
                final InputStream innerInputStream = byteSource.openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (IOException e) {
                log.error(e, "Exception opening container[%s] blob[%s] from region[%s]", container, path, region);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}

Example 22 with LineIterator

use of org.apache.commons.io.LineIterator in project druid by druid-io.

the class StaticS3FirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");
    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();
            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());
            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);
            try {
                final InputStream innerInputStream = s3Client.getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();
                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 23 with LineIterator

use of org.apache.commons.io.LineIterator in project es6draft by anba.

the class Test262Info method readTagged.

private void readTagged(String descriptor, boolean lenient) throws MalformedDataException {
    assert descriptor != null && !descriptor.isEmpty();
    for (LineIterator lines = new LineIterator(new StringReader(descriptor)); lines.hasNext(); ) {
        String line = lines.next();
        Matcher m = tags.matcher(line);
        if (m.matches()) {
            String type = m.group(1);
            String val = m.group(2);
            switch(type) {
                case "description":
                    this.description = requireNonNull(val, "description must not be null");
                    break;
                case "noStrict":
                    requireNull(val);
                    this.noStrict = true;
                    break;
                case "onlyStrict":
                    requireNull(val);
                    this.onlyStrict = true;
                    break;
                case "negative":
                    this.negative = true;
                    this.errorType = Objects.toString(val, this.errorType);
                    break;
                case "hostObject":
                case "reviewers":
                case "generator":
                case "verbatim":
                case "noHelpers":
                case "bestPractice":
                case "implDependent":
                case "author":
                    // ignore for now
                    break;
                // legacy
                case "strict_mode_negative":
                    this.negative = true;
                    this.onlyStrict = true;
                    this.errorType = Objects.toString(val, this.errorType);
                    break;
                case "strict_only":
                    requireNull(val);
                    this.onlyStrict = true;
                    break;
                case "errortype":
                    this.errorType = requireNonNull(val, "error-type must not be null");
                    break;
                case "assertion":
                case "section":
                case "path":
                case "comment":
                case "name":
                    // ignore for now
                    break;
                default:
                    // error
                    if (lenient) {
                        break;
                    }
                    throw new MalformedDataException(String.format("unhandled type '%s' (%s)\n", type, this));
            }
        }
    }
}

Example 24 with LineIterator

use of org.apache.commons.io.LineIterator in project voldemort by voldemort.

the class StringSorter method main.

public static void main(String[] args) throws Exception {
    if (args.length != 3)
        Utils.croak("USAGE: java StringSorter inputfile internal_sort_size num_threads");
    String input = args[0];
    int internalSortSize = Integer.parseInt(args[1]);
    int numThreads = Integer.parseInt(args[2]);
    ExternalSorter<String> sorter = new ExternalSorter<String>(new StringSerializer(), internalSortSize, numThreads);
    @SuppressWarnings("unchecked") Iterator<String> it = new LineIterator(new BufferedReader(new FileReader(input), 10 * 1024 * 1024));
    String separator = Utils.NEWLINE;
    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(System.out), 10 * 1024 * 1024);
    for (String line : sorter.sorted(it)) {
        writer.write(line);
        writer.write(separator);
    }
}

Example 25 with LineIterator

use of org.apache.commons.io.LineIterator in project opennms by OpenNMS.

the class JmxRrdMigratorOffline method fixJmxGraphTemplateFile.

/**
     * Fixes a JMX graph template file.
     *
     * @param jmxTemplateFile the JMX template file
     * @throws OnmsUpgradeException the OpenNMS upgrade exception
     */
private void fixJmxGraphTemplateFile(File jmxTemplateFile) throws OnmsUpgradeException {
    try {
        log("Updating JMX graph templates on %s\n", jmxTemplateFile);
        zipFile(jmxTemplateFile);
        backupFiles.add(new File(jmxTemplateFile.getAbsolutePath() + ZIP_EXT));
        File outputFile = new File(jmxTemplateFile.getCanonicalFile() + ".temp");
        FileWriter w = new FileWriter(outputFile);
        Pattern defRegex = Pattern.compile("DEF:.+:(.+\\..+):");
        Pattern colRegex = Pattern.compile("\\.columns=(.+)$");
        Pattern incRegex = Pattern.compile("^include.directory=(.+)$");
        List<File> externalFiles = new ArrayList<File>();
        boolean override = false;
        LineIterator it = FileUtils.lineIterator(jmxTemplateFile);
        while (it.hasNext()) {
            String line = it.next();
            Matcher m = incRegex.matcher(line);
            if (m.find()) {
                File includeDirectory = new File(jmxTemplateFile.getParentFile(), m.group(1));
                if (includeDirectory.isDirectory()) {
                    FilenameFilter propertyFilesFilter = new FilenameFilter() {

                        @Override
                        public boolean accept(File dir, String name) {
                            return (name.endsWith(".properties"));
                        }
                    };
                    for (File file : includeDirectory.listFiles(propertyFilesFilter)) {
                        externalFiles.add(file);
                    }
                }
            }
            m = colRegex.matcher(line);
            if (m.find()) {
                String[] badColumns = m.group(1).split(",(\\s)?");
                for (String badDs : badColumns) {
                    String fixedDs = getFixedDsName(badDs);
                    if (fixedDs.equals(badDs)) {
                        continue;
                    }
                    if (badMetrics.contains(badDs)) {
                        override = true;
                        log("  Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line);
                        line = line.replaceAll(badDs, fixedDs);
                    } else {
                        log("  Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs);
                    }
                }
            }
            m = defRegex.matcher(line);
            if (m.find()) {
                String badDs = m.group(1);
                if (badMetrics.contains(badDs)) {
                    override = true;
                    String fixedDs = getFixedDsName(badDs);
                    log("  Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line);
                    line = line.replaceAll(badDs, fixedDs);
                } else {
                    log("  Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs);
                }
            }
            w.write(line + "\n");
        }
        LineIterator.closeQuietly(it);
        w.close();
        if (override) {
            FileUtils.deleteQuietly(jmxTemplateFile);
            FileUtils.moveFile(outputFile, jmxTemplateFile);
        } else {
            FileUtils.deleteQuietly(outputFile);
        }
        if (!externalFiles.isEmpty()) {
            for (File configFile : externalFiles) {
                fixJmxGraphTemplateFile(configFile);
            }
        }
    } catch (Exception e) {
        throw new OnmsUpgradeException("Can't fix " + jmxTemplateFile + " because " + e.getMessage(), e);
    }
}