Search in sources :

Example 1 with TransactionSizeCalculator

use of password.pwm.util.TransactionSizeCalculator in project pwm by pwm-project.

the class LocalDBUtility method importLocalDB.

private void importLocalDB(final InputStream inputStream, final Appendable out, final long totalBytes) throws PwmOperationalException, IOException {
    this.prepareForImport();
    importLineCounter = 0;
    if (totalBytes > 0) {
        writeStringToOut(out, "total bytes in localdb import source: " + totalBytes);
    }
    writeStringToOut(out, "beginning localdb import...");
    final Instant startTime = Instant.now();
    final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(new TransactionSizeCalculator.SettingsBuilder().setDurationGoal(new TimeDuration(100, TimeUnit.MILLISECONDS)).setMinTransactions(50).setMaxTransactions(5 * 1000).createSettings());
    final Map<LocalDB.DB, Map<String, String>> transactionMap = new HashMap<>();
    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
        transactionMap.put(loopDB, new TreeMap<>());
    }
    final CountingInputStream countingInputStream = new CountingInputStream(inputStream);
    final EventRateMeter eventRateMeter = new EventRateMeter(TimeDuration.MINUTE);
    final Timer statTimer = new Timer(true);
    statTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            String output = "";
            if (totalBytes > 0) {
                final ProgressInfo progressInfo = new ProgressInfo(startTime, totalBytes, countingInputStream.getByteCount());
                output += progressInfo.debugOutput();
            } else {
                output += "recordsImported=" + importLineCounter;
            }
            output += ", avgTransactionSize=" + transactionCalculator.getTransactionSize() + ", recordsPerMinute=" + eventRateMeter.readEventRate().setScale(2, BigDecimal.ROUND_DOWN);
            writeStringToOut(out, output);
        }
    }, 30 * 1000, 30 * 1000);
    Reader csvReader = null;
    try {
        csvReader = new InputStreamReader(new GZIPInputStream(countingInputStream, GZIP_BUFFER_SIZE), PwmConstants.DEFAULT_CHARSET);
        for (final CSVRecord record : PwmConstants.DEFAULT_CSV_FORMAT.parse(csvReader)) {
            importLineCounter++;
            eventRateMeter.markEvents(1);
            final String dbNameRecordStr = record.get(0);
            final LocalDB.DB db = JavaHelper.readEnumFromString(LocalDB.DB.class, null, dbNameRecordStr);
            final String key = record.get(1);
            final String value = record.get(2);
            if (db == null) {
                writeStringToOut(out, "ignoring localdb import record #" + importLineCounter + ", invalid DB name '" + dbNameRecordStr + "'");
            } else {
                transactionMap.get(db).put(key, value);
                int cachedTransactions = 0;
                for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
                    cachedTransactions += transactionMap.get(loopDB).size();
                }
                if (cachedTransactions >= transactionCalculator.getTransactionSize()) {
                    final long startTxnTime = System.currentTimeMillis();
                    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
                        localDB.putAll(loopDB, transactionMap.get(loopDB));
                        transactionMap.get(loopDB).clear();
                    }
                    transactionCalculator.recordLastTransactionDuration(TimeDuration.fromCurrent(startTxnTime));
                }
            }
        }
    } finally {
        LOGGER.trace("import process completed");
        statTimer.cancel();
        IOUtils.closeQuietly(csvReader);
        IOUtils.closeQuietly(countingInputStream);
    }
    for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
        localDB.putAll(loopDB, transactionMap.get(loopDB));
        transactionMap.get(loopDB).clear();
    }
    this.markImportComplete();
    writeStringToOut(out, "restore complete, restored " + importLineCounter + " records in " + TimeDuration.fromCurrent(startTime).asLongString());
    statTimer.cancel();
}
Also used : InputStreamReader(java.io.InputStreamReader) TransactionSizeCalculator(password.pwm.util.TransactionSizeCalculator) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Instant(java.time.Instant) CountingInputStream(org.apache.commons.io.input.CountingInputStream) Reader(java.io.Reader) InputStreamReader(java.io.InputStreamReader) EventRateMeter(password.pwm.svc.stats.EventRateMeter) GZIPInputStream(java.util.zip.GZIPInputStream) Timer(java.util.Timer) TimerTask(java.util.TimerTask) ProgressInfo(password.pwm.util.ProgressInfo) TimeDuration(password.pwm.util.java.TimeDuration) CSVRecord(org.apache.commons.csv.CSVRecord) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) TreeMap(java.util.TreeMap)

Aggregations

InputStreamReader (java.io.InputStreamReader)1 Reader (java.io.Reader)1 Instant (java.time.Instant)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1 Map (java.util.Map)1 Timer (java.util.Timer)1 TimerTask (java.util.TimerTask)1 TreeMap (java.util.TreeMap)1 GZIPInputStream (java.util.zip.GZIPInputStream)1 CSVRecord (org.apache.commons.csv.CSVRecord)1 CountingInputStream (org.apache.commons.io.input.CountingInputStream)1 EventRateMeter (password.pwm.svc.stats.EventRateMeter)1 ProgressInfo (password.pwm.util.ProgressInfo)1 TransactionSizeCalculator (password.pwm.util.TransactionSizeCalculator)1 TimeDuration (password.pwm.util.java.TimeDuration)1