Examples with LineIterator org.apache.commons.io.LineIterator used on opensource projects

Example 16 with LineIterator

use of org.apache.commons.io.LineIterator in project opennms by OpenNMS.

the class JmxRrdMigratorOffline method fixJmxConfigurationFile.

/**
 * Fixes a JMX configuration file.
 *
 * @param jmxConfigFile the JMX configuration file
 * @throws OnmsUpgradeException the OpenNMS upgrade exception
 */
private void fixJmxConfigurationFile(File jmxConfigFile) throws OnmsUpgradeException {
    try {
        log("Updating JMX metric definitions on %s\n", jmxConfigFile);
        zipFile(jmxConfigFile);
        backupFiles.add(new File(jmxConfigFile.getAbsolutePath() + ZIP_EXT));
        File outputFile = new File(jmxConfigFile.getCanonicalFile() + ".temp");
        FileWriter w = new FileWriter(outputFile);
        Pattern extRegex = Pattern.compile("import-mbeans[>](.+)[<]");
        Pattern aliasRegex = Pattern.compile("alias=\"([^\"]+\\.[^\"]+)\"");
        List<File> externalFiles = new ArrayList<>();
        LineIterator it = FileUtils.lineIterator(jmxConfigFile);
        while (it.hasNext()) {
            String line = it.next();
            Matcher m = extRegex.matcher(line);
            if (m.find()) {
                externalFiles.add(new File(jmxConfigFile.getParentFile(), m.group(1)));
            }
            m = aliasRegex.matcher(line);
            if (m.find()) {
                String badDs = m.group(1);
                String fixedDs = getFixedDsName(badDs);
                log("  Replacing bad alias %s with %s on %s\n", badDs, fixedDs, line.trim());
                line = line.replaceAll(badDs, fixedDs);
                if (badMetrics.contains(badDs) == false) {
                    badMetrics.add(badDs);
                }
            }
            w.write(line + "\n");
        }
        LineIterator.closeQuietly(it);
        w.close();
        FileUtils.deleteQuietly(jmxConfigFile);
        FileUtils.moveFile(outputFile, jmxConfigFile);
        if (!externalFiles.isEmpty()) {
            for (File configFile : externalFiles) {
                fixJmxConfigurationFile(configFile);
            }
        }
    } catch (Exception e) {
        throw new OnmsUpgradeException("Can't fix " + jmxConfigFile + " because " + e.getMessage(), e);
    }
}

Example 17 with LineIterator

use of org.apache.commons.io.LineIterator in project opennms by OpenNMS.

the class JmxRrdMigratorOffline method fixJmxGraphTemplateFile.

/**
 * Fixes a JMX graph template file.
 *
 * @param jmxTemplateFile the JMX template file
 * @throws OnmsUpgradeException the OpenNMS upgrade exception
 */
private void fixJmxGraphTemplateFile(File jmxTemplateFile) throws OnmsUpgradeException {
    try {
        log("Updating JMX graph templates on %s\n", jmxTemplateFile);
        zipFile(jmxTemplateFile);
        backupFiles.add(new File(jmxTemplateFile.getAbsolutePath() + ZIP_EXT));
        File outputFile = new File(jmxTemplateFile.getCanonicalFile() + ".temp");
        FileWriter w = new FileWriter(outputFile);
        Pattern defRegex = Pattern.compile("DEF:.+:(.+\\..+):");
        Pattern colRegex = Pattern.compile("\\.columns=(.+)$");
        Pattern incRegex = Pattern.compile("^include.directory=(.+)$");
        List<File> externalFiles = new ArrayList<>();
        boolean override = false;
        LineIterator it = FileUtils.lineIterator(jmxTemplateFile);
        while (it.hasNext()) {
            String line = it.next();
            Matcher m = incRegex.matcher(line);
            if (m.find()) {
                File includeDirectory = new File(jmxTemplateFile.getParentFile(), m.group(1));
                if (includeDirectory.isDirectory()) {
                    FilenameFilter propertyFilesFilter = new FilenameFilter() {

                        @Override
                        public boolean accept(File dir, String name) {
                            return (name.endsWith(".properties"));
                        }
                    };
                    for (File file : includeDirectory.listFiles(propertyFilesFilter)) {
                        externalFiles.add(file);
                    }
                }
            }
            m = colRegex.matcher(line);
            if (m.find()) {
                String[] badColumns = m.group(1).split(",(\\s)?");
                for (String badDs : badColumns) {
                    String fixedDs = getFixedDsName(badDs);
                    if (fixedDs.equals(badDs)) {
                        continue;
                    }
                    if (badMetrics.contains(badDs)) {
                        override = true;
                        log("  Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line);
                        line = line.replaceAll(badDs, fixedDs);
                    } else {
                        log("  Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs);
                    }
                }
            }
            m = defRegex.matcher(line);
            if (m.find()) {
                String badDs = m.group(1);
                if (badMetrics.contains(badDs)) {
                    override = true;
                    String fixedDs = getFixedDsName(badDs);
                    log("  Replacing bad data source %s with %s on %s\n", badDs, fixedDs, line);
                    line = line.replaceAll(badDs, fixedDs);
                } else {
                    log("  Warning: a bad data source not related with JMX has been found: %s (this won't be updated)\n", badDs);
                }
            }
            w.write(line + "\n");
        }
        LineIterator.closeQuietly(it);
        w.close();
        if (override) {
            FileUtils.deleteQuietly(jmxTemplateFile);
            FileUtils.moveFile(outputFile, jmxTemplateFile);
        } else {
            FileUtils.deleteQuietly(outputFile);
        }
        if (!externalFiles.isEmpty()) {
            for (File configFile : externalFiles) {
                fixJmxGraphTemplateFile(configFile);
            }
        }
    } catch (Exception e) {
        throw new OnmsUpgradeException("Can't fix " + jmxTemplateFile + " because " + e.getMessage(), e);
    }
}

Example 18 with LineIterator

use of org.apache.commons.io.LineIterator in project druid by druid-io.

the class StaticS3FirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");
    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();
            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());
            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);
            try {
                final InputStream innerInputStream = s3Client.getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();
                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 19 with LineIterator

use of org.apache.commons.io.LineIterator in project druid by druid-io.

the class FileIteratingFirehoseTest method testFirehose.

@Test
public void testFirehose() throws Exception {
    for (Pair<String[], ImmutableList<String>> fixture : fixtures) {
        final List<LineIterator> lineIterators = Lists.transform(Arrays.asList(fixture.lhs), new Function<String, LineIterator>() {

            @Override
            public LineIterator apply(String s) {
                return new LineIterator(new StringReader(s));
            }
        });
        final StringInputRowParser parser = new StringInputRowParser(new CSVParseSpec(new TimestampSpec("ts", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("x")), null, null), ",", ImmutableList.of("ts", "x")), null);
        final FileIteratingFirehose firehose = new FileIteratingFirehose(lineIterators.iterator(), parser);
        final List<String> results = Lists.newArrayList();
        while (firehose.hasMore()) {
            results.add(Joiner.on("|").join(firehose.nextRow().getDimension("x")));
        }
        Assert.assertEquals(fixture.rhs, results);
    }
}

Example 20 with LineIterator

use of org.apache.commons.io.LineIterator in project druid by druid-io.

the class StaticGoogleBlobStoreFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException {
    Preconditions.checkNotNull(storage, "null storage");
    final LinkedList<GoogleBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final GoogleBlob nextURI = objectQueue.poll();
            final String bucket = nextURI.getBucket();
            final String path = nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath();
            try {
                final InputStream innerInputStream = new GoogleByteSource(storage, bucket, path).openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                LOG.error(e, "Exception opening bucket[%s] blob[%s]", bucket, path);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}