use of org.apache.nifi.provenance.toc.TocWriter in project nifi by apache.
the class WriteAheadProvenanceRepository method initialize.
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory, final IdentifierLookup idLookup) throws IOException {
final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
};
final EventFileManager fileManager = new EventFileManager();
final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
fileManager.obtainReadLock(file);
try {
return RecordReaders.newRecordReader(file, logs, maxChars);
} finally {
fileManager.releaseReadLock(file);
}
};
init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}
use of org.apache.nifi.provenance.toc.TocWriter in project nifi by apache.
the class EncryptedWriteAheadProvenanceRepository method initialize.
/**
* This method initializes the repository. It first builds the key provider and event encryptor
* from the config values, then creates the encrypted record writer and reader, then delegates
* back to the superclass for the common implementation.
*
* @param eventReporter the event reporter
* @param authorizer the authorizer
* @param resourceFactory the authorizable factory
* @param idLookup the lookup provider
* @throws IOException if there is an error initializing this repository
*/
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory, final IdentifierLookup idLookup) throws IOException {
// Initialize the encryption-specific fields
ProvenanceEventEncryptor provenanceEventEncryptor;
if (getConfig().supportsEncryption()) {
try {
KeyProvider keyProvider;
if (KeyProviderFactory.requiresMasterKey(getConfig().getKeyProviderImplementation())) {
SecretKey masterKey = getMasterKey();
keyProvider = buildKeyProvider(masterKey);
} else {
keyProvider = buildKeyProvider();
}
provenanceEventEncryptor = new AESProvenanceEventEncryptor();
provenanceEventEncryptor.initialize(keyProvider);
} catch (KeyManagementException e) {
String msg = "Encountered an error building the key provider";
logger.error(msg, e);
throw new IOException(msg, e);
}
} else {
throw new IOException("The provided configuration does not support a encrypted repository");
}
// Build a factory using lambda which injects the encryptor
final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
try {
final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup, provenanceEventEncryptor, getConfig().getDebugFrequency());
} catch (EncryptionException e) {
logger.error("Encountered an error building the schema record writer factory: ", e);
throw new IOException(e);
}
};
// Build a factory using lambda which injects the encryptor
final EventFileManager fileManager = new EventFileManager();
final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
fileManager.obtainReadLock(file);
try {
EncryptedSchemaRecordReader tempReader = (EncryptedSchemaRecordReader) RecordReaders.newRecordReader(file, logs, maxChars);
tempReader.setProvenanceEventEncryptor(provenanceEventEncryptor);
return tempReader;
} finally {
fileManager.releaseReadLock(file);
}
};
// Delegate the init to the parent impl
super.init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}
use of org.apache.nifi.provenance.toc.TocWriter in project nifi by apache.
the class CompressableRecordWriter method writeRecord.
@Override
public synchronized StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException {
if (isDirty()) {
throw new IOException("Cannot update Provenance Repository because this Record Writer has already failed to write to the Repository");
}
try {
final long recordIdentifier = record.getEventId() == -1L ? idGenerator.getAndIncrement() : record.getEventId();
final long startBytes = byteCountingOut.getBytesWritten();
ensureStreamState(recordIdentifier, startBytes);
writeRecord(record, recordIdentifier, out);
recordCount++;
final long bytesWritten = byteCountingOut.getBytesWritten();
final long serializedLength = bytesWritten - startBytes;
final TocWriter tocWriter = getTocWriter();
final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex();
final String storageLocation = getStorageLocation();
return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, bytesWritten);
} catch (final IOException ioe) {
markDirty();
throw ioe;
}
}
Aggregations