use of org.xipki.ca.dbtool.xmlio.ca.RequestCertType in project xipki by xipki.
the class CaCertStoreDbImporter method importEntries.
private long importEntries(CaDbEntryType type, String entriesZipFile, long minId, File processLogFile, ProcessLog processLog, int numProcessedInLastProcess, PreparedStatement[] statements, String[] sqls) throws Exception {
final int numEntriesPerCommit = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsPerCommit));
ZipFile zipFile = new ZipFile(new File(entriesZipFile));
ZipEntry entriesXmlEntry = zipFile.getEntry("overview.xml");
DbiXmlReader entries;
try {
entries = createReader(type, zipFile.getInputStream(entriesXmlEntry));
} catch (Exception ex) {
try {
zipFile.close();
} catch (Exception e2) {
LOG.error("could not close ZIP file {}: {}", entriesZipFile, e2.getMessage());
LOG.debug("could not close ZIP file " + entriesZipFile, e2);
}
throw ex;
}
disableAutoCommit();
try {
int numEntriesInBatch = 0;
long lastSuccessfulEntryId = 0;
while (entries.hasNext()) {
if (stopMe.get()) {
throw new InterruptedException("interrupted by the user");
}
IdentifidDbObjectType entry = (IdentifidDbObjectType) entries.next();
long id = entry.getId();
if (id < minId) {
continue;
}
numEntriesInBatch++;
if (CaDbEntryType.CERT == type) {
CertType cert = (CertType) entry;
int certArt = (cert.getArt() == null) ? 1 : cert.getArt();
String filename = cert.getFile();
// rawcert
ZipEntry certZipEnty = zipFile.getEntry(filename);
// rawcert
byte[] encodedCert = IoUtil.read(zipFile.getInputStream(certZipEnty));
TBSCertificate tbsCert;
try {
Certificate cc = Certificate.getInstance(encodedCert);
tbsCert = cc.getTBSCertificate();
} catch (RuntimeException ex) {
LOG.error("could not parse certificate in file {}", filename);
LOG.debug("could not parse certificate in file " + filename, ex);
throw new CertificateException(ex.getMessage(), ex);
}
byte[] encodedKey = tbsCert.getSubjectPublicKeyInfo().getPublicKeyData().getBytes();
String b64Sha1FpCert = HashAlgo.SHA1.base64Hash(encodedCert);
// cert
String subjectText = X509Util.cutX500Name(tbsCert.getSubject(), maxX500nameLen);
PreparedStatement psCert = statements[0];
PreparedStatement psRawcert = statements[1];
try {
int idx = 1;
psCert.setLong(idx++, id);
psCert.setInt(idx++, certArt);
psCert.setLong(idx++, cert.getUpdate());
psCert.setString(idx++, tbsCert.getSerialNumber().getPositiveValue().toString(16));
psCert.setString(idx++, subjectText);
long fpSubject = X509Util.fpCanonicalizedName(tbsCert.getSubject());
psCert.setLong(idx++, fpSubject);
if (cert.getFpRs() != null) {
psCert.setLong(idx++, cert.getFpRs());
} else {
psCert.setNull(idx++, Types.BIGINT);
}
psCert.setLong(idx++, tbsCert.getStartDate().getDate().getTime() / 1000);
psCert.setLong(idx++, tbsCert.getEndDate().getDate().getTime() / 1000);
setBoolean(psCert, idx++, cert.getRev());
setInt(psCert, idx++, cert.getRr());
setLong(psCert, idx++, cert.getRt());
setLong(psCert, idx++, cert.getRit());
setInt(psCert, idx++, cert.getPid());
setInt(psCert, idx++, cert.getCaId());
setInt(psCert, idx++, cert.getRid());
setInt(psCert, idx++, cert.getUid());
psCert.setLong(idx++, FpIdCalculator.hash(encodedKey));
Extension extension = tbsCert.getExtensions().getExtension(Extension.basicConstraints);
boolean ee = true;
if (extension != null) {
ASN1Encodable asn1 = extension.getParsedValue();
ee = !BasicConstraints.getInstance(asn1).isCA();
}
psCert.setInt(idx++, ee ? 1 : 0);
psCert.setInt(idx++, cert.getReqType());
String tidS = null;
if (cert.getTid() != null) {
tidS = cert.getTid();
}
psCert.setString(idx++, tidS);
psCert.addBatch();
} catch (SQLException ex) {
throw translate(SQL_ADD_CERT, ex);
}
try {
int idx = 1;
psRawcert.setLong(idx++, cert.getId());
psRawcert.setString(idx++, b64Sha1FpCert);
psRawcert.setString(idx++, cert.getRs());
psRawcert.setString(idx++, Base64.encodeToString(encodedCert));
psRawcert.addBatch();
} catch (SQLException ex) {
throw translate(SQL_ADD_CRAW, ex);
}
} else if (CaDbEntryType.CRL == type) {
PreparedStatement psAddCrl = statements[0];
CrlType crl = (CrlType) entry;
String filename = crl.getFile();
// CRL
ZipEntry zipEnty = zipFile.getEntry(filename);
// rawcert
byte[] encodedCrl = IoUtil.read(zipFile.getInputStream(zipEnty));
X509CRL x509crl = null;
try {
x509crl = X509Util.parseCrl(encodedCrl);
} catch (Exception ex) {
LOG.error("could not parse CRL in file {}", filename);
LOG.debug("could not parse CRL in file " + filename, ex);
if (ex instanceof CRLException) {
throw (CRLException) ex;
} else {
throw new CRLException(ex.getMessage(), ex);
}
}
try {
byte[] octetString = x509crl.getExtensionValue(Extension.cRLNumber.getId());
if (octetString == null) {
LOG.warn("CRL without CRL number, ignore it");
continue;
}
byte[] extnValue = DEROctetString.getInstance(octetString).getOctets();
// CHECKSTYLE:SKIP
BigInteger crlNumber = ASN1Integer.getInstance(extnValue).getPositiveValue();
BigInteger baseCrlNumber = null;
octetString = x509crl.getExtensionValue(Extension.deltaCRLIndicator.getId());
if (octetString != null) {
extnValue = DEROctetString.getInstance(octetString).getOctets();
baseCrlNumber = ASN1Integer.getInstance(extnValue).getPositiveValue();
}
int idx = 1;
psAddCrl.setLong(idx++, crl.getId());
psAddCrl.setInt(idx++, crl.getCaId());
psAddCrl.setLong(idx++, crlNumber.longValue());
psAddCrl.setLong(idx++, x509crl.getThisUpdate().getTime() / 1000);
if (x509crl.getNextUpdate() != null) {
psAddCrl.setLong(idx++, x509crl.getNextUpdate().getTime() / 1000);
} else {
psAddCrl.setNull(idx++, Types.INTEGER);
}
if (baseCrlNumber == null) {
setBoolean(psAddCrl, idx++, false);
psAddCrl.setNull(idx++, Types.BIGINT);
} else {
setBoolean(psAddCrl, idx++, true);
psAddCrl.setLong(idx++, baseCrlNumber.longValue());
}
String str = Base64.encodeToString(encodedCrl);
psAddCrl.setString(idx++, str);
psAddCrl.addBatch();
} catch (SQLException ex) {
System.err.println("could not import CRL with ID=" + crl.getId() + ", message: " + ex.getMessage());
throw ex;
}
} else if (CaDbEntryType.REQUEST == type) {
PreparedStatement psAddRequest = statements[0];
RequestType request = (RequestType) entry;
String filename = request.getFile();
ZipEntry zipEnty = zipFile.getEntry(filename);
byte[] encodedRequest = IoUtil.read(zipFile.getInputStream(zipEnty));
try {
int idx = 1;
psAddRequest.setLong(idx++, request.getId());
psAddRequest.setLong(idx++, request.getUpdate());
psAddRequest.setString(idx++, Base64.encodeToString(encodedRequest));
psAddRequest.addBatch();
} catch (SQLException ex) {
System.err.println("could not import REQUEST with ID=" + request.getId() + ", message: " + ex.getMessage());
throw ex;
}
} else if (CaDbEntryType.REQCERT == type) {
PreparedStatement psAddReqCert = statements[0];
RequestCertType reqCert = (RequestCertType) entry;
try {
int idx = 1;
psAddReqCert.setLong(idx++, reqCert.getId());
psAddReqCert.setLong(idx++, reqCert.getRid());
psAddReqCert.setLong(idx++, reqCert.getCid());
psAddReqCert.addBatch();
} catch (SQLException ex) {
System.err.println("could not import REQUEST with ID=" + reqCert.getId() + ", message: " + ex.getMessage());
throw ex;
}
} else {
throw new RuntimeException("Unknown CaDbEntryType " + type);
}
boolean isLastBlock = !entries.hasNext();
if (numEntriesInBatch > 0 && (numEntriesInBatch % numEntriesPerCommit == 0 || isLastBlock)) {
if (evaulateOnly) {
for (PreparedStatement m : statements) {
m.clearBatch();
}
} else {
String sql = null;
try {
for (int i = 0; i < sqls.length; i++) {
sql = sqls[i];
statements[i].executeBatch();
}
sql = null;
commit("(commit import to CA)");
} catch (Throwable th) {
rollback();
deleteFromTableWithLargerId(type.getTableName(), "ID", id, LOG);
if (CaDbEntryType.CERT == type) {
deleteFromTableWithLargerId("CRAW", "CID", id, LOG);
}
if (th instanceof SQLException) {
throw translate(sql, (SQLException) th);
} else if (th instanceof Exception) {
throw (Exception) th;
} else {
throw new Exception(th);
}
}
}
lastSuccessfulEntryId = id;
processLog.addNumProcessed(numEntriesInBatch);
numEntriesInBatch = 0;
echoToFile(type + ":" + (numProcessedInLastProcess + processLog.numProcessed()) + ":" + lastSuccessfulEntryId, processLogFile);
processLog.printStatus();
}
}
return lastSuccessfulEntryId;
} finally {
recoverAutoCommit();
zipFile.close();
}
}
use of org.xipki.ca.dbtool.xmlio.ca.RequestCertType in project xipki by xipki.
the class CaCertStoreDbExporter method exportEntries.
private void exportEntries(CaDbEntryType type, CertStoreType certstore, File processLogFile, FileOutputStream filenameListOs, Long idProcessedInLastProcess) throws Exception {
final int numEntriesPerSelect = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsPerSelect));
final int numEntriesPerZip = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsInBundle));
final File entriesDir = new File(baseDir, type.getDirName());
final String tableName = type.getTableName();
int numProcessedBefore;
String coreSql;
switch(type) {
case CERT:
numProcessedBefore = certstore.getCountCerts();
coreSql = "ID,SN,CA_ID,PID,RID,ART,RTYPE,TID,UID,EE,LUPDATE,REV,RR,RT,RIT,FP_RS," + "REQ_SUBJECT,CERT FROM CERT INNER JOIN CRAW ON CERT.ID>=? AND CERT.ID=CRAW.CID";
break;
case CRL:
numProcessedBefore = certstore.getCountCrls();
coreSql = "ID,CA_ID,CRL FROM CRL WHERE ID>=?";
break;
case REQUEST:
numProcessedBefore = certstore.getCountRequests();
coreSql = "ID,LUPDATE,DATA FROM REQUEST WHERE ID>=?";
break;
case REQCERT:
numProcessedBefore = certstore.getCountReqCerts();
coreSql = "ID,RID,CID FROM REQCERT WHERE ID>=?";
break;
default:
throw new RuntimeException("unknown CaDbEntryType " + type);
}
Long minId = (idProcessedInLastProcess != null) ? idProcessedInLastProcess + 1 : min(tableName, "ID");
String tablesText = (CaDbEntryType.CERT == type) ? "tables " + tableName + " and CRAW" : "table " + type.getTableName();
System.out.println(exportingText() + tablesText + " from ID " + minId);
final long maxId = max(tableName, "ID");
long total = count(tableName) - numProcessedBefore;
if (total < 1) {
// to avoid exception
total = 1;
}
String sql = datasource.buildSelectFirstSql(numEntriesPerSelect, "ID ASC", coreSql);
DbiXmlWriter entriesInCurrentFile = createWriter(type);
PreparedStatement ps = prepareStatement(sql.toString());
int numEntriesInCurrentFile = 0;
int sum = 0;
File currentEntriesZipFile = new File(baseDir, "tmp-" + type.getDirName() + "-" + System.currentTimeMillis() + ".zip");
ZipOutputStream currentEntriesZip = getZipOutputStream(currentEntriesZipFile);
long minIdOfCurrentFile = -1;
long maxIdOfCurrentFile = -1;
ProcessLog processLog = new ProcessLog(total);
processLog.printHeader();
try {
Long id = null;
boolean interrupted = false;
long lastMaxId = minId - 1;
while (true) {
if (stopMe.get()) {
interrupted = true;
break;
}
ps.setLong(1, lastMaxId + 1);
ResultSet rs = ps.executeQuery();
// no entries anymore
if (!rs.next()) {
break;
}
do {
id = rs.getLong("ID");
if (lastMaxId < id) {
lastMaxId = id;
}
if (minIdOfCurrentFile == -1) {
minIdOfCurrentFile = id;
} else if (minIdOfCurrentFile > id) {
minIdOfCurrentFile = id;
}
if (maxIdOfCurrentFile == -1) {
maxIdOfCurrentFile = id;
} else if (maxIdOfCurrentFile < id) {
maxIdOfCurrentFile = id;
}
if (CaDbEntryType.CERT == type) {
String b64Cert = rs.getString("CERT");
byte[] certBytes = Base64.decodeFast(b64Cert);
String sha1 = HashAlgo.SHA1.hexHash(certBytes);
String certFileName = sha1 + ".der";
if (!evaulateOnly) {
ZipEntry certZipEntry = new ZipEntry(certFileName);
currentEntriesZip.putNextEntry(certZipEntry);
try {
currentEntriesZip.write(certBytes);
} finally {
currentEntriesZip.closeEntry();
}
}
CertType cert = new CertType();
cert.setId(id);
cert.setArt(rs.getInt("ART"));
cert.setCaId(rs.getInt("CA_ID"));
cert.setEe(rs.getBoolean("EE"));
cert.setFile(certFileName);
long fpReqSubject = rs.getLong("FP_RS");
if (fpReqSubject != 0) {
cert.setFpRs(fpReqSubject);
cert.setRs(rs.getString("REQ_SUBJECT"));
}
cert.setPid(rs.getInt("PID"));
cert.setReqType(rs.getInt("RTYPE"));
cert.setRid(rs.getInt("RID"));
cert.setSn(rs.getString("SN"));
String str = rs.getString("TID");
if (StringUtil.isNotBlank(str)) {
cert.setTid(str);
}
int userId = rs.getInt("UID");
if (userId != 0) {
cert.setUid(userId);
}
cert.setUpdate(rs.getLong("LUPDATE"));
boolean revoked = rs.getBoolean("REV");
cert.setRev(revoked);
if (revoked) {
cert.setRr(rs.getInt("RR"));
cert.setRt(rs.getLong("RT"));
long revInvTime = rs.getLong("RIT");
if (revInvTime != 0) {
cert.setRit(revInvTime);
}
}
((CertsWriter) entriesInCurrentFile).add(cert);
} else if (CaDbEntryType.CRL == type) {
String b64Crl = rs.getString("CRL");
byte[] crlBytes = Base64.decodeFast(b64Crl);
X509CRL x509Crl = null;
try {
x509Crl = X509Util.parseCrl(crlBytes);
} catch (CRLException ex) {
LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
throw ex;
} catch (Exception ex) {
LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
throw new CRLException(ex.getMessage(), ex);
}
byte[] octetString = x509Crl.getExtensionValue(Extension.cRLNumber.getId());
if (octetString == null) {
LOG.warn("CRL without CRL number, ignore it");
continue;
}
String sha1 = HashAlgo.SHA1.hexHash(crlBytes);
final String crlFilename = sha1 + ".crl";
if (!evaulateOnly) {
ZipEntry certZipEntry = new ZipEntry(crlFilename);
currentEntriesZip.putNextEntry(certZipEntry);
try {
currentEntriesZip.write(crlBytes);
} finally {
currentEntriesZip.closeEntry();
}
}
CrlType crl = new CrlType();
crl.setId(id);
crl.setCaId(rs.getInt("CA_ID"));
byte[] extnValue = DEROctetString.getInstance(octetString).getOctets();
BigInteger crlNumber = ASN1Integer.getInstance(extnValue).getPositiveValue();
crl.setCrlNo(crlNumber.toString());
crl.setFile(crlFilename);
((CrlsWriter) entriesInCurrentFile).add(crl);
} else if (CaDbEntryType.REQUEST == type) {
long update = rs.getLong("LUPDATE");
String b64Data = rs.getString("DATA");
byte[] dataBytes = Base64.decodeFast(b64Data);
String sha1 = HashAlgo.SHA1.hexHash(dataBytes);
final String dataFilename = sha1 + ".req";
if (!evaulateOnly) {
ZipEntry certZipEntry = new ZipEntry(dataFilename);
currentEntriesZip.putNextEntry(certZipEntry);
try {
currentEntriesZip.write(dataBytes);
} finally {
currentEntriesZip.closeEntry();
}
}
RequestType entry = new RequestType();
entry.setId(id);
entry.setUpdate(update);
entry.setFile(dataFilename);
((RequestsWriter) entriesInCurrentFile).add(entry);
} else if (CaDbEntryType.REQCERT == type) {
long cid = rs.getLong("CID");
long rid = rs.getLong("RID");
RequestCertType entry = new RequestCertType();
entry.setId(id);
entry.setCid(cid);
entry.setRid(rid);
((RequestCertsWriter) entriesInCurrentFile).add(entry);
} else {
throw new RuntimeException("unknown CaDbEntryType " + type);
}
numEntriesInCurrentFile++;
sum++;
if (numEntriesInCurrentFile == numEntriesPerZip) {
String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip", minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
finalizeZip(currentEntriesZip, "overview.xml", entriesInCurrentFile);
currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));
writeLine(filenameListOs, currentEntriesFilename);
setCount(type, certstore, numProcessedBefore + sum);
echoToFile(tableName + ":" + Long.toString(id), processLogFile);
processLog.addNumProcessed(numEntriesInCurrentFile);
processLog.printStatus();
// reset
entriesInCurrentFile = createWriter(type);
numEntriesInCurrentFile = 0;
minIdOfCurrentFile = -1;
maxIdOfCurrentFile = -1;
currentEntriesZipFile = new File(baseDir, "tmp-" + type.getDirName() + "-" + System.currentTimeMillis() + ".zip");
currentEntriesZip = getZipOutputStream(currentEntriesZipFile);
}
} while (rs.next());
rs.close();
}
if (interrupted) {
currentEntriesZip.close();
throw new InterruptedException("interrupted by the user");
}
if (numEntriesInCurrentFile > 0) {
finalizeZip(currentEntriesZip, "overview.xml", entriesInCurrentFile);
String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip", minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));
writeLine(filenameListOs, currentEntriesFilename);
setCount(type, certstore, numProcessedBefore + sum);
if (id != null) {
echoToFile(Long.toString(id), processLogFile);
}
processLog.addNumProcessed(numEntriesInCurrentFile);
} else {
currentEntriesZip.close();
currentEntriesZipFile.delete();
}
} catch (SQLException ex) {
throw translate(null, ex);
} finally {
releaseResources(ps, null);
}
// end try
processLog.printTrailer();
// all successful, delete the processLogFile
processLogFile.delete();
System.out.println(exportedText() + sum + " entries from " + tablesText);
}
Aggregations