Examples with ProcessLog org.xipki.common.ProcessLog used on opensource projects

Example 6 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class CaCertStoreDbExporter method exportEntries.

private void exportEntries(CaDbEntryType type, CertStoreType certstore, File processLogFile, FileOutputStream filenameListOs, Long idProcessedInLastProcess) throws Exception {
    final int numEntriesPerSelect = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsPerSelect));
    final int numEntriesPerZip = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsInBundle));
    final File entriesDir = new File(baseDir, type.getDirName());
    final String tableName = type.getTableName();
    int numProcessedBefore;
    String coreSql;
    switch(type) {
        case CERT:
            numProcessedBefore = certstore.getCountCerts();
            coreSql = "ID,SN,CA_ID,PID,RID,ART,RTYPE,TID,UID,EE,LUPDATE,REV,RR,RT,RIT,FP_RS," + "REQ_SUBJECT,CERT FROM CERT INNER JOIN CRAW ON CERT.ID>=? AND CERT.ID=CRAW.CID";
            break;
        case CRL:
            numProcessedBefore = certstore.getCountCrls();
            coreSql = "ID,CA_ID,CRL FROM CRL WHERE ID>=?";
            break;
        case REQUEST:
            numProcessedBefore = certstore.getCountRequests();
            coreSql = "ID,LUPDATE,DATA FROM REQUEST WHERE ID>=?";
            break;
        case REQCERT:
            numProcessedBefore = certstore.getCountReqCerts();
            coreSql = "ID,RID,CID FROM REQCERT WHERE ID>=?";
            break;
        default:
            throw new RuntimeException("unknown CaDbEntryType " + type);
    }
    Long minId = (idProcessedInLastProcess != null) ? idProcessedInLastProcess + 1 : min(tableName, "ID");
    String tablesText = (CaDbEntryType.CERT == type) ? "tables " + tableName + " and CRAW" : "table " + type.getTableName();
    System.out.println(exportingText() + tablesText + " from ID " + minId);
    final long maxId = max(tableName, "ID");
    long total = count(tableName) - numProcessedBefore;
    if (total < 1) {
        // to avoid exception
        total = 1;
    }
    String sql = datasource.buildSelectFirstSql(numEntriesPerSelect, "ID ASC", coreSql);
    DbiXmlWriter entriesInCurrentFile = createWriter(type);
    PreparedStatement ps = prepareStatement(sql.toString());
    int numEntriesInCurrentFile = 0;
    int sum = 0;
    File currentEntriesZipFile = new File(baseDir, "tmp-" + type.getDirName() + "-" + System.currentTimeMillis() + ".zip");
    ZipOutputStream currentEntriesZip = getZipOutputStream(currentEntriesZipFile);
    long minIdOfCurrentFile = -1;
    long maxIdOfCurrentFile = -1;
    ProcessLog processLog = new ProcessLog(total);
    processLog.printHeader();
    try {
        Long id = null;
        boolean interrupted = false;
        long lastMaxId = minId - 1;
        while (true) {
            if (stopMe.get()) {
                interrupted = true;
                break;
            }
            ps.setLong(1, lastMaxId + 1);
            ResultSet rs = ps.executeQuery();
            // no entries anymore
            if (!rs.next()) {
                break;
            }
            do {
                id = rs.getLong("ID");
                if (lastMaxId < id) {
                    lastMaxId = id;
                }
                if (minIdOfCurrentFile == -1) {
                    minIdOfCurrentFile = id;
                } else if (minIdOfCurrentFile > id) {
                    minIdOfCurrentFile = id;
                }
                if (maxIdOfCurrentFile == -1) {
                    maxIdOfCurrentFile = id;
                } else if (maxIdOfCurrentFile < id) {
                    maxIdOfCurrentFile = id;
                }
                if (CaDbEntryType.CERT == type) {
                    String b64Cert = rs.getString("CERT");
                    byte[] certBytes = Base64.decodeFast(b64Cert);
                    String sha1 = HashAlgo.SHA1.hexHash(certBytes);
                    String certFileName = sha1 + ".der";
                    if (!evaulateOnly) {
                        ZipEntry certZipEntry = new ZipEntry(certFileName);
                        currentEntriesZip.putNextEntry(certZipEntry);
                        try {
                            currentEntriesZip.write(certBytes);
                        } finally {
                            currentEntriesZip.closeEntry();
                        }
                    }
                    CertType cert = new CertType();
                    cert.setId(id);
                    cert.setArt(rs.getInt("ART"));
                    cert.setCaId(rs.getInt("CA_ID"));
                    cert.setEe(rs.getBoolean("EE"));
                    cert.setFile(certFileName);
                    long fpReqSubject = rs.getLong("FP_RS");
                    if (fpReqSubject != 0) {
                        cert.setFpRs(fpReqSubject);
                        cert.setRs(rs.getString("REQ_SUBJECT"));
                    }
                    cert.setPid(rs.getInt("PID"));
                    cert.setReqType(rs.getInt("RTYPE"));
                    cert.setRid(rs.getInt("RID"));
                    cert.setSn(rs.getString("SN"));
                    String str = rs.getString("TID");
                    if (StringUtil.isNotBlank(str)) {
                        cert.setTid(str);
                    }
                    int userId = rs.getInt("UID");
                    if (userId != 0) {
                        cert.setUid(userId);
                    }
                    cert.setUpdate(rs.getLong("LUPDATE"));
                    boolean revoked = rs.getBoolean("REV");
                    cert.setRev(revoked);
                    if (revoked) {
                        cert.setRr(rs.getInt("RR"));
                        cert.setRt(rs.getLong("RT"));
                        long revInvTime = rs.getLong("RIT");
                        if (revInvTime != 0) {
                            cert.setRit(revInvTime);
                        }
                    }
                    ((CertsWriter) entriesInCurrentFile).add(cert);
                } else if (CaDbEntryType.CRL == type) {
                    String b64Crl = rs.getString("CRL");
                    byte[] crlBytes = Base64.decodeFast(b64Crl);
                    X509CRL x509Crl = null;
                    try {
                        x509Crl = X509Util.parseCrl(crlBytes);
                    } catch (CRLException ex) {
                        LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
                        throw ex;
                    } catch (Exception ex) {
                        LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
                        throw new CRLException(ex.getMessage(), ex);
                    }
                    byte[] octetString = x509Crl.getExtensionValue(Extension.cRLNumber.getId());
                    if (octetString == null) {
                        LOG.warn("CRL without CRL number, ignore it");
                        continue;
                    }
                    String sha1 = HashAlgo.SHA1.hexHash(crlBytes);
                    final String crlFilename = sha1 + ".crl";
                    if (!evaulateOnly) {
                        ZipEntry certZipEntry = new ZipEntry(crlFilename);
                        currentEntriesZip.putNextEntry(certZipEntry);
                        try {
                            currentEntriesZip.write(crlBytes);
                        } finally {
                            currentEntriesZip.closeEntry();
                        }
                    }
                    CrlType crl = new CrlType();
                    crl.setId(id);
                    crl.setCaId(rs.getInt("CA_ID"));
                    byte[] extnValue = DEROctetString.getInstance(octetString).getOctets();
                    BigInteger crlNumber = ASN1Integer.getInstance(extnValue).getPositiveValue();
                    crl.setCrlNo(crlNumber.toString());
                    crl.setFile(crlFilename);
                    ((CrlsWriter) entriesInCurrentFile).add(crl);
                } else if (CaDbEntryType.REQUEST == type) {
                    long update = rs.getLong("LUPDATE");
                    String b64Data = rs.getString("DATA");
                    byte[] dataBytes = Base64.decodeFast(b64Data);
                    String sha1 = HashAlgo.SHA1.hexHash(dataBytes);
                    final String dataFilename = sha1 + ".req";
                    if (!evaulateOnly) {
                        ZipEntry certZipEntry = new ZipEntry(dataFilename);
                        currentEntriesZip.putNextEntry(certZipEntry);
                        try {
                            currentEntriesZip.write(dataBytes);
                        } finally {
                            currentEntriesZip.closeEntry();
                        }
                    }
                    RequestType entry = new RequestType();
                    entry.setId(id);
                    entry.setUpdate(update);
                    entry.setFile(dataFilename);
                    ((RequestsWriter) entriesInCurrentFile).add(entry);
                } else if (CaDbEntryType.REQCERT == type) {
                    long cid = rs.getLong("CID");
                    long rid = rs.getLong("RID");
                    RequestCertType entry = new RequestCertType();
                    entry.setId(id);
                    entry.setCid(cid);
                    entry.setRid(rid);
                    ((RequestCertsWriter) entriesInCurrentFile).add(entry);
                } else {
                    throw new RuntimeException("unknown CaDbEntryType " + type);
                }
                numEntriesInCurrentFile++;
                sum++;
                if (numEntriesInCurrentFile == numEntriesPerZip) {
                    String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip", minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
                    finalizeZip(currentEntriesZip, "overview.xml", entriesInCurrentFile);
                    currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));
                    writeLine(filenameListOs, currentEntriesFilename);
                    setCount(type, certstore, numProcessedBefore + sum);
                    echoToFile(tableName + ":" + Long.toString(id), processLogFile);
                    processLog.addNumProcessed(numEntriesInCurrentFile);
                    processLog.printStatus();
                    // reset
                    entriesInCurrentFile = createWriter(type);
                    numEntriesInCurrentFile = 0;
                    minIdOfCurrentFile = -1;
                    maxIdOfCurrentFile = -1;
                    currentEntriesZipFile = new File(baseDir, "tmp-" + type.getDirName() + "-" + System.currentTimeMillis() + ".zip");
                    currentEntriesZip = getZipOutputStream(currentEntriesZipFile);
                }
            } while (rs.next());
            rs.close();
        }
        if (interrupted) {
            currentEntriesZip.close();
            throw new InterruptedException("interrupted by the user");
        }
        if (numEntriesInCurrentFile > 0) {
            finalizeZip(currentEntriesZip, "overview.xml", entriesInCurrentFile);
            String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip", minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
            currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));
            writeLine(filenameListOs, currentEntriesFilename);
            setCount(type, certstore, numProcessedBefore + sum);
            if (id != null) {
                echoToFile(Long.toString(id), processLogFile);
            }
            processLog.addNumProcessed(numEntriesInCurrentFile);
        } else {
            currentEntriesZip.close();
            currentEntriesZipFile.delete();
        }
    } catch (SQLException ex) {
        throw translate(null, ex);
    } finally {
        releaseResources(ps, null);
    }
    // end try
    processLog.printTrailer();
    // all successful, delete the processLogFile
    processLogFile.delete();
    System.out.println(exportedText() + sum + " entries from " + tablesText);
}

Example 7 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class CaCertStoreDbImporter method importEntries.

// method importDeltaCRLCache
private Exception importEntries(CaDbEntryType type, CertStoreType certstore, File processLogFile, Integer numProcessedInLastProcess, Long idProcessedInLastProcess) {
    String tablesText = (CaDbEntryType.CERT == type) ? "tables CERT and CRAW" : "table " + type.getTableName();
    try {
        int numProcessedBefore = 0;
        long minId = 1;
        if (idProcessedInLastProcess != null) {
            minId = idProcessedInLastProcess + 1;
            numProcessedBefore = numProcessedInLastProcess;
        }
        deleteFromTableWithLargerId(type.getTableName(), "ID", minId - 1, LOG);
        if (type == CaDbEntryType.CERT) {
            deleteFromTableWithLargerId("CRAW", "CID", minId - 1, LOG);
        }
        final long total;
        String[] sqls;
        switch(type) {
            case CERT:
                total = certstore.getCountCerts();
                sqls = new String[] { SQL_ADD_CERT, SQL_ADD_CRAW };
                break;
            case CRL:
                total = certstore.getCountCrls();
                sqls = new String[] { SQL_ADD_CRL };
                break;
            case REQUEST:
                total = certstore.getCountRequests();
                sqls = new String[] { SQL_ADD_REQUEST };
                break;
            case REQCERT:
                total = certstore.getCountReqCerts();
                sqls = new String[] { SQL_ADD_REQCERT };
                break;
            default:
                throw new RuntimeException("unsupported DbEntryType " + type);
        }
        final long remainingTotal = total - numProcessedBefore;
        final ProcessLog processLog = new ProcessLog(remainingTotal);
        System.out.println(importingText() + "entries to " + tablesText + " from ID " + minId);
        processLog.printHeader();
        DbPortFileNameIterator entriesFileIterator = null;
        PreparedStatement[] statements = null;
        try {
            entriesFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
            statements = new PreparedStatement[sqls.length];
            for (int i = 0; i < sqls.length; i++) {
                statements[i] = prepareStatement(sqls[i]);
            }
            while (entriesFileIterator.hasNext()) {
                String entriesFile = baseDir + File.separator + type.getDirName() + File.separator + entriesFileIterator.next();
                // extract the toId from the filename
                int fromIdx = entriesFile.indexOf('-');
                int toIdx = entriesFile.indexOf(".zip");
                if (fromIdx != -1 && toIdx != -1) {
                    try {
                        long toId = Integer.parseInt(entriesFile.substring(fromIdx + 1, toIdx));
                        if (toId < minId) {
                            // try next file
                            continue;
                        }
                    } catch (Exception ex) {
                        LOG.warn("invalid file name '{}', but will still be processed", entriesFile);
                    }
                } else {
                    LOG.warn("invalid file name '{}', but will still be processed", entriesFile);
                }
                try {
                    long lastId = importEntries(type, entriesFile, minId, processLogFile, processLog, numProcessedBefore, statements, sqls);
                    minId = lastId + 1;
                } catch (Exception ex) {
                    System.err.println("\ncould not import entries from file " + entriesFile + ".\nplease continue with the option '--resume'");
                    LOG.error("Exception", ex);
                    return ex;
                }
            }
        // end for
        } finally {
            if (statements != null) {
                for (PreparedStatement stmt : statements) {
                    if (stmt != null) {
                        releaseResources(stmt, null);
                    }
                }
            }
            if (entriesFileIterator != null) {
                entriesFileIterator.close();
            }
        }
        processLog.printTrailer();
        echoToFile(type + ":" + (numProcessedBefore + processLog.numProcessed()) + ":-1", processLogFile);
        System.out.println(importedText() + processLog.numProcessed() + " entries");
        return null;
    } catch (Exception ex) {
        System.err.println("\nimporting " + tablesText + " has been cancelled due to error,\n" + "please continue with the option '--resume'");
        LOG.error("Exception", ex);
        return ex;
    }
}