Examples with ProcessLog org.xipki.common.ProcessLog used on opensource projects

Example 1 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class CertRepublisher method republish0.

private boolean republish0() {
    long total;
    try {
        total = certstore.getCountOfCerts(ca, onlyRevokedCerts);
    } catch (OperationException ex) {
        LogUtil.error(LOG, ex, "could not getCountOfCerts");
        return false;
    }
    processLog = new ProcessLog(total);
    processLog.printHeader();
    ExecutorService executor = Executors.newFixedThreadPool(numThreads + 1);
    List<CertRepublishConsumer> consumers = new ArrayList<>(numThreads);
    AtomicBoolean stopMe = new AtomicBoolean(false);
    for (int i = 0; i < numThreads; i++) {
        CertRepublishConsumer consumer = new CertRepublishConsumer();
        consumers.add(consumer);
    }
    CertRepublishProducer producer = new CertRepublishProducer();
    executor.execute(producer);
    for (CertRepublishConsumer consumer : consumers) {
        executor.execute(consumer);
    }
    executor.shutdown();
    boolean successful = true;
    while (true) {
        processLog.printStatus();
        if (successful) {
            if (producer.failed) {
                successful = false;
            }
            if (successful) {
                for (CertRepublishConsumer consumer : consumers) {
                    if (consumer.failed) {
                        successful = false;
                        break;
                    }
                }
            }
            if (!successful) {
                stopMe.set(true);
                LOG.warn("failed");
            }
        }
        try {
            boolean terminated = executor.awaitTermination(1, TimeUnit.SECONDS);
            if (terminated) {
                break;
            }
        } catch (InterruptedException ex) {
            stopMe.set(true);
            LogUtil.warn(LOG, ex, "interrupted: " + ex.getMessage());
        }
    }
    if (successful) {
        if (producer.failed) {
            successful = false;
        }
        if (successful) {
            for (CertRepublishConsumer consumer : consumers) {
                if (consumer.failed) {
                    successful = false;
                    break;
                }
            }
        }
        if (!successful) {
            LOG.warn("failed");
        }
    }
    return successful;
}

Example 2 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class OcspCertStoreFromCaDbImporter method importCert.

// method importIssuer0
private void importCert(CertStoreType certstore, Map<Integer, String> profileMap, boolean revokedOnly, List<Integer> caIds, File processLogFile) throws Exception {
    HashAlgo certhashAlgo = getCertHashAlgo(datasource);
    int numProcessedBefore = 0;
    long minId = 1;
    if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 2) {
            String str = new String(content);
            if (str.trim().equalsIgnoreCase(MSG_CERTS_FINISHED)) {
                return;
            }
            StringTokenizer st = new StringTokenizer(str, ":");
            numProcessedBefore = Integer.parseInt(st.nextToken());
            minId = Long.parseLong(st.nextToken());
            minId++;
        }
    }
    deleteCertGreatherThan(minId - 1, LOG);
    final long total = certstore.getCountCerts() - numProcessedBefore;
    final ProcessLog processLog = new ProcessLog(total);
    // all initial values for importLog will be not evaluated, so just any number
    final ProcessLog importLog = new ProcessLog(total);
    System.out.println(importingText() + "certificates from ID " + minId);
    processLog.printHeader();
    PreparedStatement psCert = prepareStatement(SQL_ADD_CERT);
    CaDbEntryType type = CaDbEntryType.CERT;
    DbPortFileNameIterator certsFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
    try {
        while (certsFileIterator.hasNext()) {
            String certsFile = baseDir + File.separator + type.getDirName() + File.separator + certsFileIterator.next();
            // extract the toId from the filename
            int fromIdx = certsFile.indexOf('-');
            int toIdx = certsFile.indexOf(".zip");
            if (fromIdx != -1 && toIdx != -1) {
                try {
                    long toId = Integer.parseInt(certsFile.substring(fromIdx + 1, toIdx));
                    if (toId < minId) {
                        // try next file
                        continue;
                    }
                } catch (Exception ex) {
                    LOG.warn("invalid file name '{}', but will still be processed", certsFile);
                }
            } else {
                LOG.warn("invalid file name '{}', but will still be processed", certsFile);
            }
            try {
                long lastId = importCert0(certhashAlgo, psCert, certsFile, profileMap, revokedOnly, caIds, minId, processLogFile, processLog, numProcessedBefore, importLog);
                minId = lastId + 1;
            } catch (Exception ex) {
                System.err.println("\ncould not import certificates from file " + certsFile + ".\nplease continue with the option '--resume'");
                LOG.error("Exception", ex);
                throw ex;
            }
        }
    } finally {
        releaseResources(psCert, null);
        certsFileIterator.close();
    }
    processLog.printTrailer();
    DbPorter.echoToFile(MSG_CERTS_FINISHED, processLogFile);
    System.out.println("processed " + processLog.numProcessed() + " and " + importedText() + importLog.numProcessed() + " certificates");
}

Example 3 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class DigestDiff method diffSingleCa.

// method diff
private void diffSingleCa(RefDigestReader refReader, Map<Integer, byte[]> caIdCertBytesMap) throws CertificateException, IOException, InterruptedException {
    X509Certificate caCert = refReader.getCaCert();
    byte[] caCertBytes = caCert.getEncoded();
    if (includeCaCerts != null && !includeCaCerts.isEmpty()) {
        boolean include = false;
        for (byte[] m : includeCaCerts) {
            if (Arrays.equals(m, caCertBytes)) {
                include = true;
                break;
            }
        }
        if (!include) {
            System.out.println("skipped CA " + refReader.getCaSubjectName());
        }
    }
    String commonName = X509Util.getCommonName(caCert.getSubjectX500Principal());
    File caReportDir = new File(reportDirName, "ca-" + commonName);
    int idx = 2;
    while (caReportDir.exists()) {
        caReportDir = new File(reportDirName, "ca-" + commonName + "-" + (idx++));
    }
    DigestDiffReporter reporter = new DigestDiffReporter(caReportDir.getPath(), caCertBytes);
    Integer caId = null;
    for (Integer i : caIdCertBytesMap.keySet()) {
        if (Arrays.equals(caCertBytes, caIdCertBytesMap.get(i))) {
            caId = i;
        }
    }
    if (caId == null) {
        reporter.addNoCaMatch();
        refReader.close();
        reporter.close();
        return;
    }
    TargetDigestRetriever target = null;
    try {
        reporter.start();
        ProcessLog processLog = new ProcessLog(refReader.getTotalAccount());
        System.out.println("Processing certificates of CA \n\t'" + refReader.getCaSubjectName() + "'");
        processLog.printHeader();
        target = new TargetDigestRetriever(revokedOnly, processLog, refReader, reporter, targetDatasource, targetDbControl, certhashAlgo, caId, numPerSelect, numTargetThreads, stopMe);
        target.awaitTerminiation();
        processLog.printTrailer();
    } catch (InterruptedException ex) {
        throw ex;
    } catch (Exception ex) {
        reporter.addError("Exception thrown: " + ex.getClass().getName() + ": " + ex.getMessage());
        LOG.error("exception in diffSingleCa", ex);
    } finally {
        reporter.close();
        refReader.close();
        if (target != null) {
            target.close();
        }
    }
}

Example 4 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class OcspCertStoreDbImporter method importCert.

// method importIssuer0
private void importCert(CertStoreType certstore, File processLogFile) throws Exception {
    int numProcessedBefore = 0;
    long minId = 1;
    if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 2) {
            String str = new String(content);
            if (str.trim().equalsIgnoreCase(MSG_CERTS_FINISHED)) {
                return;
            }
            StringTokenizer st = new StringTokenizer(str, ":");
            numProcessedBefore = Integer.parseInt(st.nextToken());
            minId = Long.parseLong(st.nextToken());
            minId++;
        }
    }
    deleteCertGreatherThan(minId - 1, LOG);
    final long total = certstore.getCountCerts() - numProcessedBefore;
    final ProcessLog processLog = new ProcessLog(total);
    System.out.println(importingText() + "certificates from ID " + minId);
    processLog.printHeader();
    PreparedStatement psCert = prepareStatement(SQL_ADD_CERT);
    OcspDbEntryType type = OcspDbEntryType.CERT;
    DbPortFileNameIterator certsFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
    try {
        while (certsFileIterator.hasNext()) {
            String certsFile = baseDir + File.separator + type.getDirName() + File.separator + certsFileIterator.next();
            // extract the toId from the filename
            int fromIdx = certsFile.indexOf('-');
            int toIdx = certsFile.indexOf(".zip");
            if (fromIdx != -1 && toIdx != -1) {
                try {
                    long toId = Long.parseLong(certsFile.substring(fromIdx + 1, toIdx));
                    if (toId < minId) {
                        // try next file
                        continue;
                    }
                } catch (Exception ex) {
                    LOG.warn("invalid file name '{}', but will still be processed", certsFile);
                }
            } else {
                LOG.warn("invalid file name '{}', but will still be processed", certsFile);
            }
            try {
                long lastId = importCert0(psCert, certsFile, minId, processLogFile, processLog, numProcessedBefore);
                minId = lastId + 1;
            } catch (Exception ex) {
                System.err.println("\ncould not import certificates from file " + certsFile + ".\nplease continue with the option '--resume'");
                LOG.error("Exception", ex);
                throw ex;
            }
        }
    // end for
    } finally {
        releaseResources(psCert, null);
        certsFileIterator.close();
    }
    processLog.printTrailer();
    echoToFile(MSG_CERTS_FINISHED, processLogFile);
    System.out.println(importedText() + processLog.numProcessed() + " certificates");
}

Example 5 with ProcessLog

use of org.xipki.common.ProcessLog in project xipki by xipki.

the class OcspCertStoreDbExporter method exportCert0.

// method exportCert
private void exportCert0(CertStoreType certstore, File processLogFile, FileOutputStream certsFileOs) throws Exception {
    File certsDir = new File(baseDir, OcspDbEntryType.CERT.getDirName());
    Long minId = null;
    if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 0) {
            minId = Long.parseLong(new String(content).trim());
            minId++;
        }
    }
    if (minId == null) {
        minId = min("CERT", "ID");
    }
    System.out.println(exportingText() + "table CERT from ID " + minId);
    final String coreSql = "ID,SN,IID,LUPDATE,REV,RR,RT,RIT,PN,NAFTER,NBEFORE,HASH,SUBJECT " + "FROM CERT WHERE ID>=?";
    final String certSql = datasource.buildSelectFirstSql(numCertsPerSelect, "ID ASC", coreSql);
    final long maxId = max("CERT", "ID");
    int numProcessedBefore = certstore.getCountCerts();
    final long total = count("CERT") - numProcessedBefore;
    ProcessLog processLog = new ProcessLog(total);
    PreparedStatement certPs = prepareStatement(certSql);
    int sum = 0;
    int numCertInCurrentFile = 0;
    OcspCertsWriter certsInCurrentFile = new OcspCertsWriter();
    File currentCertsZipFile = new File(baseDir, "tmp-certs-" + System.currentTimeMillis() + ".zip");
    ZipOutputStream currentCertsZip = getZipOutputStream(currentCertsZipFile);
    long minCertIdOfCurrentFile = -1;
    long maxCertIdOfCurrentFile = -1;
    processLog.printHeader();
    String sql = null;
    Long id = null;
    try {
        boolean interrupted = false;
        long lastMaxId = minId - 1;
        while (true) {
            if (stopMe.get()) {
                interrupted = true;
                break;
            }
            sql = certSql;
            certPs.setLong(1, lastMaxId + 1);
            ResultSet rs = certPs.executeQuery();
            if (!rs.next()) {
                break;
            }
            do {
                id = rs.getLong("ID");
                if (lastMaxId < id) {
                    lastMaxId = id;
                }
                if (minCertIdOfCurrentFile == -1) {
                    minCertIdOfCurrentFile = id;
                } else if (minCertIdOfCurrentFile > id) {
                    minCertIdOfCurrentFile = id;
                }
                if (maxCertIdOfCurrentFile == -1) {
                    maxCertIdOfCurrentFile = id;
                } else if (maxCertIdOfCurrentFile < id) {
                    maxCertIdOfCurrentFile = id;
                }
                OcspCertType cert = new OcspCertType();
                cert.setId(id);
                cert.setIid(rs.getInt("IID"));
                cert.setSn(rs.getString("SN"));
                cert.setUpdate(rs.getLong("LUPDATE"));
                boolean revoked = rs.getBoolean("REV");
                cert.setRev(revoked);
                if (revoked) {
                    cert.setRr(rs.getInt("RR"));
                    cert.setRt(rs.getLong("RT"));
                    long rit = rs.getLong("RIT");
                    if (rit != 0) {
                        cert.setRit(rit);
                    }
                }
                cert.setProfile(rs.getString("PN"));
                String hash = rs.getString("HASH");
                if (hash != null) {
                    cert.setHash(hash);
                }
                String subject = rs.getString("SUBJECT");
                if (subject != null) {
                    cert.setSubject(subject);
                }
                long nafter = rs.getLong("NAFTER");
                if (nafter != 0) {
                    cert.setNafter(nafter);
                }
                long nbefore = rs.getLong("NBEFORE");
                if (nbefore != 0) {
                    cert.setNbefore(nbefore);
                }
                certsInCurrentFile.add(cert);
                numCertInCurrentFile++;
                sum++;
                if (numCertInCurrentFile == numCertsInBundle) {
                    finalizeZip(currentCertsZip, certsInCurrentFile);
                    String currentCertsFilename = buildFilename("certs_", ".zip", minCertIdOfCurrentFile, maxCertIdOfCurrentFile, maxId);
                    currentCertsZipFile.renameTo(new File(certsDir, currentCertsFilename));
                    writeLine(certsFileOs, currentCertsFilename);
                    certstore.setCountCerts(numProcessedBefore + sum);
                    echoToFile(Long.toString(id), processLogFile);
                    processLog.addNumProcessed(numCertInCurrentFile);
                    processLog.printStatus();
                    // reset
                    certsInCurrentFile = new OcspCertsWriter();
                    numCertInCurrentFile = 0;
                    minCertIdOfCurrentFile = -1;
                    maxCertIdOfCurrentFile = -1;
                    currentCertsZipFile = new File(baseDir, "tmp-certs-" + System.currentTimeMillis() + ".zip");
                    currentCertsZip = getZipOutputStream(currentCertsZipFile);
                }
            // end if
            } while (rs.next());
            rs.close();
        }
        if (interrupted) {
            throw new InterruptedException("interrupted by the user");
        }
        if (numCertInCurrentFile > 0) {
            finalizeZip(currentCertsZip, certsInCurrentFile);
            String currentCertsFilename = buildFilename("certs_", ".zip", minCertIdOfCurrentFile, maxCertIdOfCurrentFile, maxId);
            currentCertsZipFile.renameTo(new File(certsDir, currentCertsFilename));
            writeLine(certsFileOs, currentCertsFilename);
            certstore.setCountCerts(numProcessedBefore + sum);
            if (id != null) {
                echoToFile(Long.toString(id), processLogFile);
            }
            processLog.addNumProcessed(numCertInCurrentFile);
        } else {
            currentCertsZip.close();
            currentCertsZipFile.delete();
        }
    } catch (SQLException ex) {
        throw translate(sql, ex);
    } finally {
        releaseResources(certPs, null);
    }
    processLog.printTrailer();
    // all successful, delete the processLogFile
    processLogFile.delete();
    System.out.println(exportedText() + processLog.numProcessed() + " certificates from tables CERT");
}