Examples with CountingInputStream org.apache.commons.io.input.CountingInputStream used on opensource projects

Example 16 with CountingInputStream

use of org.apache.commons.io.input.CountingInputStream in project pwm by pwm-project.

the class LocalDBUtility method importLocalDB.

private void importLocalDB(final InputStream inputStream, final Appendable out, final long totalBytes) throws PwmOperationalException, IOException {
    this.prepareForImport();
    importLineCounter = 0;
    if (totalBytes > 0) {
        writeStringToOut(out, "total bytes in localdb import source: " + totalBytes);
    }
    writeStringToOut(out, "beginning localdb import...");
    final Instant startTime = Instant.now();
    final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(new TransactionSizeCalculator.SettingsBuilder().setDurationGoal(new TimeDuration(100, TimeUnit.MILLISECONDS)).setMinTransactions(50).setMaxTransactions(5 * 1000).createSettings());
    final Map<LocalDB.DB, Map<String, String>> transactionMap = new HashMap<>();
    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
        transactionMap.put(loopDB, new TreeMap<>());
    }
    final CountingInputStream countingInputStream = new CountingInputStream(inputStream);
    final EventRateMeter eventRateMeter = new EventRateMeter(TimeDuration.MINUTE);
    final Timer statTimer = new Timer(true);
    statTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            String output = "";
            if (totalBytes > 0) {
                final ProgressInfo progressInfo = new ProgressInfo(startTime, totalBytes, countingInputStream.getByteCount());
                output += progressInfo.debugOutput();
            } else {
                output += "recordsImported=" + importLineCounter;
            }
            output += ", avgTransactionSize=" + transactionCalculator.getTransactionSize() + ", recordsPerMinute=" + eventRateMeter.readEventRate().setScale(2, BigDecimal.ROUND_DOWN);
            writeStringToOut(out, output);
        }
    }, 30 * 1000, 30 * 1000);
    Reader csvReader = null;
    try {
        csvReader = new InputStreamReader(new GZIPInputStream(countingInputStream, GZIP_BUFFER_SIZE), PwmConstants.DEFAULT_CHARSET);
        for (final CSVRecord record : PwmConstants.DEFAULT_CSV_FORMAT.parse(csvReader)) {
            importLineCounter++;
            eventRateMeter.markEvents(1);
            final String dbNameRecordStr = record.get(0);
            final LocalDB.DB db = JavaHelper.readEnumFromString(LocalDB.DB.class, null, dbNameRecordStr);
            final String key = record.get(1);
            final String value = record.get(2);
            if (db == null) {
                writeStringToOut(out, "ignoring localdb import record #" + importLineCounter + ", invalid DB name '" + dbNameRecordStr + "'");
            } else {
                transactionMap.get(db).put(key, value);
                int cachedTransactions = 0;
                for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
                    cachedTransactions += transactionMap.get(loopDB).size();
                }
                if (cachedTransactions >= transactionCalculator.getTransactionSize()) {
                    final long startTxnTime = System.currentTimeMillis();
                    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
                        localDB.putAll(loopDB, transactionMap.get(loopDB));
                        transactionMap.get(loopDB).clear();
                    }
                    transactionCalculator.recordLastTransactionDuration(TimeDuration.fromCurrent(startTxnTime));
                }
            }
        }
    } finally {
        LOGGER.trace("import process completed");
        statTimer.cancel();
        IOUtils.closeQuietly(csvReader);
        IOUtils.closeQuietly(countingInputStream);
    }
    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
        localDB.putAll(loopDB, transactionMap.get(loopDB));
        transactionMap.get(loopDB).clear();
    }
    this.markImportComplete();
    writeStringToOut(out, "restore complete, restored " + importLineCounter + " records in " + TimeDuration.fromCurrent(startTime).asLongString());
    statTimer.cancel();
}

Example 17 with CountingInputStream

use of org.apache.commons.io.input.CountingInputStream in project hudson-2.x by hudson.

the class FilePath method installIfNecessaryFrom.

/**
 * Given a tgz/zip file, extracts it to the given target directory, if necessary.
 *
 * <p>
 * This method is a convenience method designed for installing a binary package to a location
 * that supports upgrade and downgrade. Specifically,
 *
 * <ul>
 * <li>If the target directory doesn't exist {@linkplain #mkdirs() it'll be created}.
 * <li>The timestamp of the .tgz file is left in the installation directory upon extraction.
 * <li>If the timestamp left in the directory doesn't match with the timestamp of the current archive file,
 *     the directory contents will be discarded and the archive file will be re-extracted.
 * <li>If the connection is refused but the target directory already exists, it is left alone.
 * </ul>
 *
 * @param archive
 *      The resource that represents the tgz/zip file. This URL must support the "Last-Modified" header.
 *      (Most common usage is to get this from {@link ClassLoader#getResource(String)})
 * @param listener
 *      If non-null, a message will be printed to this listener once this method decides to
 *      extract an archive.
 * @return
 *      true if the archive was extracted. false if the extraction was skipped because the target directory
 *      was considered up to date.
 * @since 1.299
 */
public boolean installIfNecessaryFrom(URL archive, TaskListener listener, String message) throws IOException, InterruptedException {
    try {
        URLConnection con;
        try {
            con = ProxyConfiguration.open(archive);
            con.connect();
        } catch (IOException x) {
            if (this.exists()) {
                // Cannot connect now, so assume whatever was last unpacked is still OK.
                if (listener != null) {
                    listener.getLogger().println("Skipping installation of " + archive + " to " + remote + ": " + x);
                }
                return false;
            } else {
                throw x;
            }
        }
        long sourceTimestamp = con.getLastModified();
        FilePath timestamp = this.child(".timestamp");
        if (this.exists()) {
            if (timestamp.exists() && sourceTimestamp == timestamp.lastModified())
                // already up to date
                return false;
            this.deleteContents();
        } else {
            this.mkdirs();
        }
        if (listener != null)
            listener.getLogger().println(message);
        InputStream in = con.getInputStream();
        CountingInputStream cis = new CountingInputStream(in);
        try {
            if (archive.toExternalForm().endsWith(".zip"))
                unzipFrom(cis);
            else
                untarFrom(cis, GZIP);
        } catch (IOException e) {
            throw new IOException2(String.format("Failed to unpack %s (%d bytes read of total %d)", archive, cis.getByteCount(), con.getContentLength()), e);
        }
        timestamp.touch(sourceTimestamp);
        return true;
    } catch (IOException e) {
        throw new IOException2("Failed to install " + archive + " to " + remote, e);
    }
}

Example 18 with CountingInputStream

use of org.apache.commons.io.input.CountingInputStream in project AntennaPod by AntennaPod.

the class ChapterReaderTest method testRealFileAuphonic.

@Test
public void testRealFileAuphonic() throws IOException, ID3ReaderException {
    CountingInputStream inputStream = new CountingInputStream(getClass().getClassLoader().getResource("auphonic.mp3").openStream());
    ChapterReader reader = new ChapterReader(inputStream);
    reader.readInputStream();
    List<Chapter> chapters = reader.getChapters();
    assertEquals(4, chapters.size());
    assertEquals(0, chapters.get(0).getStart());
    assertEquals(3000, chapters.get(1).getStart());
    assertEquals(6000, chapters.get(2).getStart());
    assertEquals(9000, chapters.get(3).getStart());
    assertEquals("Chapter 1 - ❤️😊", chapters.get(0).getTitle());
    assertEquals("Chapter 2 - ßöÄ", chapters.get(1).getTitle());
    assertEquals("Chapter 3 - 爱", chapters.get(2).getTitle());
    assertEquals("Chapter 4", chapters.get(3).getTitle());
    assertEquals("https://example.com", chapters.get(0).getLink());
    assertEquals("https://example.com", chapters.get(1).getLink());
    assertEquals("https://example.com", chapters.get(2).getLink());
    assertEquals("https://example.com", chapters.get(3).getLink());
    assertEquals(EmbeddedChapterImage.makeUrl(765, 308), chapters.get(0).getImageUrl());
    assertEquals(EmbeddedChapterImage.makeUrl(1271, 308), chapters.get(1).getImageUrl());
    assertEquals(EmbeddedChapterImage.makeUrl(1771, 308), chapters.get(2).getImageUrl());
    assertEquals(EmbeddedChapterImage.makeUrl(2259, 308), chapters.get(3).getImageUrl());
}

Example 19 with CountingInputStream

use of org.apache.commons.io.input.CountingInputStream in project AntennaPod by AntennaPod.

the class ChapterReaderTest method testReadTitleWithGarbage.

@Test
public void testReadTitleWithGarbage() throws IOException, ID3ReaderException {
    byte[] titleSubframeContent = { ID3Reader.ENCODING_ISO, // Title
    'A', // Null-terminated
    0, // Garbage, should be ignored
    42, // Garbage, should be ignored
    42, // Garbage, should be ignored
    42, // Garbage, should be ignored
    42 };
    FrameHeader header = new FrameHeader(ChapterReader.FRAME_ID_TITLE, titleSubframeContent.length, (short) 0);
    CountingInputStream inputStream = new CountingInputStream(new ByteArrayInputStream(titleSubframeContent));
    ChapterReader reader = new ChapterReader(inputStream);
    Chapter chapter = new ID3Chapter("", 0);
    reader.readChapterSubFrame(header, chapter);
    assertEquals("A", chapter.getTitle());
    // Should skip the garbage and point to the next frame
    assertEquals(titleSubframeContent.length, reader.getPosition());
}

Example 20 with CountingInputStream

use of org.apache.commons.io.input.CountingInputStream in project AntennaPod by AntennaPod.

the class ChapterReaderTest method testReadChapterWithoutSubframes.

@Test
public void testReadChapterWithoutSubframes() throws IOException, ID3ReaderException {
    FrameHeader header = new FrameHeader(ChapterReader.FRAME_ID_CHAPTER, CHAPTER_WITHOUT_SUBFRAME.length, (short) 0);
    CountingInputStream inputStream = new CountingInputStream(new ByteArrayInputStream(CHAPTER_WITHOUT_SUBFRAME));
    Chapter chapter = new ChapterReader(inputStream).readChapter(header);
    assertEquals(CHAPTER_WITHOUT_SUBFRAME_START_TIME, chapter.getStart());
}