Examples with DbPortFileNameIterator org.xipki.ca.dbtool.port.DbPortFileNameIterator used on opensource projects

Example 1 with DbPortFileNameIterator

use of org.xipki.ca.dbtool.port.DbPortFileNameIterator in project xipki by xipki.

the class OcspCertStoreFromCaDbImporter method importCert.

// method importIssuer0
private void importCert(CertStoreType certstore, Map<Integer, String> profileMap, boolean revokedOnly, List<Integer> caIds, File processLogFile) throws Exception {
    HashAlgo certhashAlgo = getCertHashAlgo(datasource);
    int numProcessedBefore = 0;
    long minId = 1;
    if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 2) {
            String str = new String(content);
            if (str.trim().equalsIgnoreCase(MSG_CERTS_FINISHED)) {
                return;
            }
            StringTokenizer st = new StringTokenizer(str, ":");
            numProcessedBefore = Integer.parseInt(st.nextToken());
            minId = Long.parseLong(st.nextToken());
            minId++;
        }
    }
    deleteCertGreatherThan(minId - 1, LOG);
    final long total = certstore.getCountCerts() - numProcessedBefore;
    final ProcessLog processLog = new ProcessLog(total);
    // all initial values for importLog will be not evaluated, so just any number
    final ProcessLog importLog = new ProcessLog(total);
    System.out.println(importingText() + "certificates from ID " + minId);
    processLog.printHeader();
    PreparedStatement psCert = prepareStatement(SQL_ADD_CERT);
    CaDbEntryType type = CaDbEntryType.CERT;
    DbPortFileNameIterator certsFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
    try {
        while (certsFileIterator.hasNext()) {
            String certsFile = baseDir + File.separator + type.getDirName() + File.separator + certsFileIterator.next();
            // extract the toId from the filename
            int fromIdx = certsFile.indexOf('-');
            int toIdx = certsFile.indexOf(".zip");
            if (fromIdx != -1 && toIdx != -1) {
                try {
                    long toId = Integer.parseInt(certsFile.substring(fromIdx + 1, toIdx));
                    if (toId < minId) {
                        // try next file
                        continue;
                    }
                } catch (Exception ex) {
                    LOG.warn("invalid file name '{}', but will still be processed", certsFile);
                }
            } else {
                LOG.warn("invalid file name '{}', but will still be processed", certsFile);
            }
            try {
                long lastId = importCert0(certhashAlgo, psCert, certsFile, profileMap, revokedOnly, caIds, minId, processLogFile, processLog, numProcessedBefore, importLog);
                minId = lastId + 1;
            } catch (Exception ex) {
                System.err.println("\ncould not import certificates from file " + certsFile + ".\nplease continue with the option '--resume'");
                LOG.error("Exception", ex);
                throw ex;
            }
        }
    } finally {
        releaseResources(psCert, null);
        certsFileIterator.close();
    }
    processLog.printTrailer();
    DbPorter.echoToFile(MSG_CERTS_FINISHED, processLogFile);
    System.out.println("processed " + processLog.numProcessed() + " and " + importedText() + importLog.numProcessed() + " certificates");
}

Example 2 with DbPortFileNameIterator

use of org.xipki.ca.dbtool.port.DbPortFileNameIterator in project xipki by xipki.

the class OcspCertStoreDbImporter method importCert.

// method importIssuer0
private void importCert(CertStoreType certstore, File processLogFile) throws Exception {
    int numProcessedBefore = 0;
    long minId = 1;
    if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 2) {
            String str = new String(content);
            if (str.trim().equalsIgnoreCase(MSG_CERTS_FINISHED)) {
                return;
            }
            StringTokenizer st = new StringTokenizer(str, ":");
            numProcessedBefore = Integer.parseInt(st.nextToken());
            minId = Long.parseLong(st.nextToken());
            minId++;
        }
    }
    deleteCertGreatherThan(minId - 1, LOG);
    final long total = certstore.getCountCerts() - numProcessedBefore;
    final ProcessLog processLog = new ProcessLog(total);
    System.out.println(importingText() + "certificates from ID " + minId);
    processLog.printHeader();
    PreparedStatement psCert = prepareStatement(SQL_ADD_CERT);
    OcspDbEntryType type = OcspDbEntryType.CERT;
    DbPortFileNameIterator certsFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
    try {
        while (certsFileIterator.hasNext()) {
            String certsFile = baseDir + File.separator + type.getDirName() + File.separator + certsFileIterator.next();
            // extract the toId from the filename
            int fromIdx = certsFile.indexOf('-');
            int toIdx = certsFile.indexOf(".zip");
            if (fromIdx != -1 && toIdx != -1) {
                try {
                    long toId = Long.parseLong(certsFile.substring(fromIdx + 1, toIdx));
                    if (toId < minId) {
                        // try next file
                        continue;
                    }
                } catch (Exception ex) {
                    LOG.warn("invalid file name '{}', but will still be processed", certsFile);
                }
            } else {
                LOG.warn("invalid file name '{}', but will still be processed", certsFile);
            }
            try {
                long lastId = importCert0(psCert, certsFile, minId, processLogFile, processLog, numProcessedBefore);
                minId = lastId + 1;
            } catch (Exception ex) {
                System.err.println("\ncould not import certificates from file " + certsFile + ".\nplease continue with the option '--resume'");
                LOG.error("Exception", ex);
                throw ex;
            }
        }
    // end for
    } finally {
        releaseResources(psCert, null);
        certsFileIterator.close();
    }
    processLog.printTrailer();
    echoToFile(MSG_CERTS_FINISHED, processLogFile);
    System.out.println(importedText() + processLog.numProcessed() + " certificates");
}

Example 3 with DbPortFileNameIterator

use of org.xipki.ca.dbtool.port.DbPortFileNameIterator in project xipki by xipki.

the class CaCertStoreDbImporter method importEntries.

// method importDeltaCRLCache
private Exception importEntries(CaDbEntryType type, CertStoreType certstore, File processLogFile, Integer numProcessedInLastProcess, Long idProcessedInLastProcess) {
    String tablesText = (CaDbEntryType.CERT == type) ? "tables CERT and CRAW" : "table " + type.getTableName();
    try {
        int numProcessedBefore = 0;
        long minId = 1;
        if (idProcessedInLastProcess != null) {
            minId = idProcessedInLastProcess + 1;
            numProcessedBefore = numProcessedInLastProcess;
        }
        deleteFromTableWithLargerId(type.getTableName(), "ID", minId - 1, LOG);
        if (type == CaDbEntryType.CERT) {
            deleteFromTableWithLargerId("CRAW", "CID", minId - 1, LOG);
        }
        final long total;
        String[] sqls;
        switch(type) {
            case CERT:
                total = certstore.getCountCerts();
                sqls = new String[] { SQL_ADD_CERT, SQL_ADD_CRAW };
                break;
            case CRL:
                total = certstore.getCountCrls();
                sqls = new String[] { SQL_ADD_CRL };
                break;
            case REQUEST:
                total = certstore.getCountRequests();
                sqls = new String[] { SQL_ADD_REQUEST };
                break;
            case REQCERT:
                total = certstore.getCountReqCerts();
                sqls = new String[] { SQL_ADD_REQCERT };
                break;
            default:
                throw new RuntimeException("unsupported DbEntryType " + type);
        }
        final long remainingTotal = total - numProcessedBefore;
        final ProcessLog processLog = new ProcessLog(remainingTotal);
        System.out.println(importingText() + "entries to " + tablesText + " from ID " + minId);
        processLog.printHeader();
        DbPortFileNameIterator entriesFileIterator = null;
        PreparedStatement[] statements = null;
        try {
            entriesFileIterator = new DbPortFileNameIterator(baseDir + File.separator + type.getDirName() + ".mf");
            statements = new PreparedStatement[sqls.length];
            for (int i = 0; i < sqls.length; i++) {
                statements[i] = prepareStatement(sqls[i]);
            }
            while (entriesFileIterator.hasNext()) {
                String entriesFile = baseDir + File.separator + type.getDirName() + File.separator + entriesFileIterator.next();
                // extract the toId from the filename
                int fromIdx = entriesFile.indexOf('-');
                int toIdx = entriesFile.indexOf(".zip");
                if (fromIdx != -1 && toIdx != -1) {
                    try {
                        long toId = Integer.parseInt(entriesFile.substring(fromIdx + 1, toIdx));
                        if (toId < minId) {
                            // try next file
                            continue;
                        }
                    } catch (Exception ex) {
                        LOG.warn("invalid file name '{}', but will still be processed", entriesFile);
                    }
                } else {
                    LOG.warn("invalid file name '{}', but will still be processed", entriesFile);
                }
                try {
                    long lastId = importEntries(type, entriesFile, minId, processLogFile, processLog, numProcessedBefore, statements, sqls);
                    minId = lastId + 1;
                } catch (Exception ex) {
                    System.err.println("\ncould not import entries from file " + entriesFile + ".\nplease continue with the option '--resume'");
                    LOG.error("Exception", ex);
                    return ex;
                }
            }
        // end for
        } finally {
            if (statements != null) {
                for (PreparedStatement stmt : statements) {
                    if (stmt != null) {
                        releaseResources(stmt, null);
                    }
                }
            }
            if (entriesFileIterator != null) {
                entriesFileIterator.close();
            }
        }
        processLog.printTrailer();
        echoToFile(type + ":" + (numProcessedBefore + processLog.numProcessed()) + ":-1", processLogFile);
        System.out.println(importedText() + processLog.numProcessed() + " entries");
        return null;
    } catch (Exception ex) {
        System.err.println("\nimporting " + tablesText + " has been cancelled due to error,\n" + "please continue with the option '--resume'");
        LOG.error("Exception", ex);
        return ex;
    }
}