use of org.apache.nifi.provenance.index.EventIndexWriter in project nifi by apache.
the class PersistentProvenanceRepository method mergeJournals.
/**
* <p>
* Merges all of the given Journal Files into a single, merged Provenance
* Event Log File. As these records are merged, they will be compressed, if
* the repository is configured to compress records, and will be indexed.
* </p>
*
* <p>
* If the repository is configured to compress the data, the file written to
* may not be the same as the <code>suggestedMergeFile</code>, as a filename
* extension of '.gz' may be appended. If the journals are successfully
* merged, the file that they were merged into will be returned. If unable
* to merge the records (for instance, because the repository has been
* closed or because the list of journal files was empty), this method will
* return <code>null</code>.
* </p>
*
* @param journalFiles the journal files to merge
* @param suggestedMergeFile the file to write the merged records to
* @param eventReporter the event reporter to report any warnings or errors
* to; may be null.
*
* @return the file that the given journals were merged into, or
* <code>null</code> if no records were merged.
*
* @throws IOException if a problem occurs writing to the mergedFile,
* reading from a journal, or updating the Lucene Index.
*/
File mergeJournals(final List<File> journalFiles, final File suggestedMergeFile, final EventReporter eventReporter) throws IOException {
logger.debug("Merging {} to {}", journalFiles, suggestedMergeFile);
if (this.closed.get()) {
logger.info("Provenance Repository has been closed; will not merge journal files to {}", suggestedMergeFile);
return null;
}
if (journalFiles.isEmpty()) {
logger.debug("Couldn't merge journals: Journal Files is empty; won't merge journals");
return null;
}
Collections.sort(journalFiles, new Comparator<File>() {
@Override
public int compare(final File o1, final File o2) {
final String suffix1 = LuceneUtil.substringAfterLast(o1.getName(), ".");
final String suffix2 = LuceneUtil.substringAfterLast(o2.getName(), ".");
try {
final int journalIndex1 = Integer.parseInt(suffix1);
final int journalIndex2 = Integer.parseInt(suffix2);
return Integer.compare(journalIndex1, journalIndex2);
} catch (final NumberFormatException nfe) {
return o1.getName().compareTo(o2.getName());
}
}
});
// Search for any missing files. At this point they should have been written to disk otherwise cannot continue.
// Missing files is most likely due to incomplete cleanup of files post merge
final List<File> availableFiles = filterUnavailableFiles(journalFiles);
final int numAvailableFiles = availableFiles.size();
// check if we have all of the "partial" files for the journal.
if (numAvailableFiles > 0) {
if (suggestedMergeFile.exists()) {
// we have all "partial" files and there is already a merged file. Delete the data from the index
// because the merge file may not be fully merged. We will re-merge.
logger.warn("Merged Journal File {} already exists; however, all partial journal files also exist " + "so assuming that the merge did not finish. Repeating procedure in order to ensure consistency.");
final DeleteIndexAction deleteAction = new DeleteIndexAction(this, indexConfig, getIndexManager());
try {
deleteAction.execute(suggestedMergeFile);
} catch (final Exception e) {
logger.warn("Failed to delete records from Journal File {} from the index; this could potentially result in duplicates. Failure was due to {}", suggestedMergeFile, e.toString());
if (logger.isDebugEnabled()) {
logger.warn("", e);
}
}
// file and the TOC file. Otherwise, we could get the wrong copy and have issues retrieving events.
if (!suggestedMergeFile.delete()) {
logger.error("Failed to delete partially written Provenance Journal File {}. This may result in events from this journal " + "file not being able to be displayed. This file should be deleted manually.", suggestedMergeFile);
}
final File tocFile = TocUtil.getTocFile(suggestedMergeFile);
if (tocFile.exists() && !tocFile.delete()) {
logger.error("Failed to delete .toc file {}; this may result in not being able to read the Provenance Events from the {} Journal File. " + "This can be corrected by manually deleting the {} file", tocFile, suggestedMergeFile, tocFile);
}
}
} else {
logger.warn("Cannot merge journal files {} because they do not exist on disk", journalFiles);
return null;
}
final long startNanos = System.nanoTime();
// Map each journal to a RecordReader
final List<RecordReader> readers = new ArrayList<>();
int records = 0;
final boolean isCompress = configuration.isCompressOnRollover();
final File writerFile = isCompress ? new File(suggestedMergeFile.getParentFile(), suggestedMergeFile.getName() + ".gz") : suggestedMergeFile;
try {
for (final File journalFile : availableFiles) {
try {
// Use MAX_VALUE for number of chars because we don't want to truncate the value as we write it
// out. This allows us to later decide that we want more characters and still be able to retrieve
// the entire event.
readers.add(RecordReaders.newRecordReader(journalFile, null, Integer.MAX_VALUE));
} catch (final EOFException eof) {
// there's nothing here. Skip over it.
} catch (final IOException ioe) {
logger.warn("Unable to merge {} with other Journal Files due to {}", journalFile, ioe.toString());
if (logger.isDebugEnabled()) {
logger.warn("", ioe);
}
if (eventReporter != null) {
eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to merge Journal Files due to " + ioe.toString());
}
}
}
// Create a Map so that the key is the next record available from a reader and the value is the Reader from which
// the record came. This sorted map is then used so that we are able to always get the first entry, which is the next
// lowest record id
final SortedMap<StandardProvenanceEventRecord, RecordReader> recordToReaderMap = new TreeMap<>(new Comparator<StandardProvenanceEventRecord>() {
@Override
public int compare(final StandardProvenanceEventRecord o1, final StandardProvenanceEventRecord o2) {
return Long.compare(o1.getEventId(), o2.getEventId());
}
});
long minEventId = 0L;
long earliestTimestamp = System.currentTimeMillis();
for (final RecordReader reader : readers) {
StandardProvenanceEventRecord record = null;
try {
record = reader.nextRecord();
} catch (final EOFException eof) {
// record will be null and reader can no longer be used
} catch (final Exception e) {
logger.warn("Failed to generate Provenance Event Record from Journal due to " + e + "; it's " + "possible that the record wasn't completely written to the file. This journal will be " + "skipped.");
if (logger.isDebugEnabled()) {
logger.warn("", e);
}
if (eventReporter != null) {
eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to read Provenance Event " + "Record from Journal due to " + e + "; it's possible that the record wasn't " + "completely written to the file. This journal will be skipped.");
}
}
if (record == null) {
continue;
}
if (record.getEventTime() < earliestTimestamp) {
earliestTimestamp = record.getEventTime();
}
if (record.getEventId() < minEventId) {
minEventId = record.getEventId();
}
recordToReaderMap.put(record, reader);
}
// We want to keep track of the last 1000 events in the files so that we can add them to 'ringBuffer'.
// However, we don't want to add them directly to ringBuffer, because once they are added to ringBuffer, they are
// available in query results. As a result, we can have the issue where we've not finished indexing the file
// but we try to create the lineage for events in that file. In order to avoid this, we will add the records
// to a temporary RingBuffer and after we finish merging the records will then copy the data to the
// ringBuffer provided as a method argument.
final RingBuffer<ProvenanceEventRecord> latestRecords = new RingBuffer<>(1000);
// with the next entry from the journal file from which the previous record was written.
try (final RecordWriter writer = RecordWriters.newSchemaRecordWriter(writerFile, idGenerator, configuration.isCompressOnRollover(), true)) {
writer.writeHeader(minEventId);
final IndexingAction indexingAction = createIndexingAction();
final File indexingDirectory = indexConfig.getWritableIndexDirectory(writerFile, earliestTimestamp);
long maxId = 0L;
final BlockingQueue<Tuple<StandardProvenanceEventRecord, Integer>> eventQueue = new LinkedBlockingQueue<>(100);
final AtomicBoolean finishedAdding = new AtomicBoolean(false);
final List<Future<?>> futures = new ArrayList<>();
final EventIndexWriter indexWriter = getIndexManager().borrowIndexWriter(indexingDirectory);
try {
final ExecutorService exec = Executors.newFixedThreadPool(configuration.getIndexThreadPoolSize(), new ThreadFactory() {
@Override
public Thread newThread(final Runnable r) {
final Thread t = Executors.defaultThreadFactory().newThread(r);
t.setName("Index Provenance Events");
return t;
}
});
final AtomicInteger indexingFailureCount = new AtomicInteger(0);
try {
for (int i = 0; i < configuration.getIndexThreadPoolSize(); i++) {
final Callable<Object> callable = new Callable<Object>() {
@Override
public Object call() throws IOException {
while (!eventQueue.isEmpty() || !finishedAdding.get()) {
try {
final Tuple<StandardProvenanceEventRecord, Integer> tuple;
try {
tuple = eventQueue.poll(10, TimeUnit.MILLISECONDS);
} catch (final InterruptedException ie) {
Thread.currentThread().interrupt();
continue;
}
if (tuple == null) {
continue;
}
indexingAction.index(tuple.getKey(), indexWriter.getIndexWriter(), tuple.getValue());
} catch (final Throwable t) {
logger.error("Failed to index Provenance Event for " + writerFile + " to " + indexingDirectory, t);
if (indexingFailureCount.incrementAndGet() >= MAX_INDEXING_FAILURE_COUNT) {
return null;
}
}
}
return null;
}
};
final Future<?> future = exec.submit(callable);
futures.add(future);
}
boolean indexEvents = true;
while (!recordToReaderMap.isEmpty()) {
final StandardProvenanceEventRecord record = recordToReaderMap.firstKey();
final RecordReader reader = recordToReaderMap.get(record);
writer.writeRecord(record);
final int blockIndex = writer.getTocWriter().getCurrentBlockIndex();
boolean accepted = false;
while (!accepted && indexEvents) {
try {
accepted = eventQueue.offer(new Tuple<>(record, blockIndex), 10, TimeUnit.MILLISECONDS);
} catch (final InterruptedException ie) {
Thread.currentThread().interrupt();
}
// So, if the queue is filled, we will check if this is the case.
if (!accepted && indexingFailureCount.get() >= MAX_INDEXING_FAILURE_COUNT) {
// don't add anything else to the queue.
indexEvents = false;
eventQueue.clear();
final String warning = String.format("Indexing Provenance Events for %s has failed %s times. This exceeds the maximum threshold of %s failures, " + "so no more Provenance Events will be indexed for this Provenance file.", writerFile, indexingFailureCount.get(), MAX_INDEXING_FAILURE_COUNT);
logger.warn(warning);
if (eventReporter != null) {
eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, warning);
}
}
}
maxId = record.getEventId();
latestRecords.add(truncateAttributes(record));
records++;
// Remove this entry from the map
recordToReaderMap.remove(record);
// Get the next entry from this reader and add it to the map
StandardProvenanceEventRecord nextRecord = null;
try {
nextRecord = reader.nextRecord();
} catch (final EOFException eof) {
// record will be null and reader can no longer be used
} catch (final Exception e) {
logger.warn("Failed to generate Provenance Event Record from Journal due to " + e + "; it's possible that the record wasn't completely written to the file. " + "The remainder of this journal will be skipped.");
if (logger.isDebugEnabled()) {
logger.warn("", e);
}
if (eventReporter != null) {
eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to read " + "Provenance Event Record from Journal due to " + e + "; it's possible " + "that the record wasn't completely written to the file. The remainder " + "of this journal will be skipped.");
}
}
if (nextRecord != null) {
recordToReaderMap.put(nextRecord, reader);
}
}
} finally {
finishedAdding.set(true);
exec.shutdown();
}
for (final Future<?> future : futures) {
try {
future.get();
} catch (final ExecutionException ee) {
final Throwable t = ee.getCause();
if (t instanceof RuntimeException) {
throw (RuntimeException) t;
}
throw new RuntimeException(t);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Thread interrupted");
}
}
} finally {
getIndexManager().returnIndexWriter(indexWriter);
}
indexConfig.setMaxIdIndexed(maxId);
}
// record should now be available in the repository. We can copy the values from latestRecords to ringBuffer.
final RingBuffer<ProvenanceEventRecord> latestRecordBuffer = this.latestRecords;
latestRecords.forEach(new ForEachEvaluator<ProvenanceEventRecord>() {
@Override
public boolean evaluate(final ProvenanceEventRecord event) {
latestRecordBuffer.add(event);
return true;
}
});
} finally {
for (final RecordReader reader : readers) {
try {
reader.close();
} catch (final IOException ioe) {
}
}
}
// Success. Remove all of the journal files, as they're no longer needed, now that they've been merged.
for (final File journalFile : availableFiles) {
if (!journalFile.delete() && journalFile.exists()) {
logger.warn("Failed to remove temporary journal file {}; this file should be cleaned up manually", journalFile.getAbsolutePath());
if (eventReporter != null) {
eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal file " + journalFile.getAbsolutePath() + "; this file should be cleaned up manually");
}
}
final File tocFile = TocUtil.getTocFile(journalFile);
if (!tocFile.delete() && tocFile.exists()) {
logger.warn("Failed to remove temporary journal TOC file {}; this file should be cleaned up manually", tocFile.getAbsolutePath());
if (eventReporter != null) {
eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal TOC file " + tocFile.getAbsolutePath() + "; this file should be cleaned up manually");
}
}
}
if (records == 0) {
writerFile.delete();
logger.debug("Couldn't merge journals: No Records to merge");
return null;
} else {
final long nanos = System.nanoTime() - startNanos;
final long millis = TimeUnit.MILLISECONDS.convert(nanos, TimeUnit.NANOSECONDS);
logger.info("Successfully merged {} journal files ({} records) into single Provenance Log File {} in {} milliseconds", numAvailableFiles, records, suggestedMergeFile, millis);
}
return writerFile;
}
use of org.apache.nifi.provenance.index.EventIndexWriter in project nifi by apache.
the class EventIndexTask method index.
private void index(final List<StoredDocument> toIndex, final String partitionName) throws IOException {
if (toIndex.isEmpty()) {
return;
}
// Convert the IndexableDocument list into a List of Documents so that we can pass them to the Index Writer.
final List<Document> documents = toIndex.stream().map(doc -> doc.getDocument()).collect(Collectors.toList());
boolean requestClose = false;
boolean requestCommit = false;
final long minEventTime = toIndex.stream().mapToLong(doc -> doc.getDocument().getField(SearchableFields.EventTime.getSearchableFieldName()).numericValue().longValue()).min().getAsLong();
// Synchronize on the directory manager because we don't want the active directory to change
// while we are obtaining an index writer for it. I.e., determining the active directory
// and obtaining an Index Writer for it need to be done atomically.
final EventIndexWriter indexWriter;
final File indexDirectory;
synchronized (directoryManager) {
indexDirectory = directoryManager.getWritableIndexingDirectory(minEventTime, partitionName);
indexWriter = indexManager.borrowIndexWriter(indexDirectory);
}
try {
// Perform the actual indexing.
boolean writerIndicatesCommit = indexWriter.index(documents, commitThreshold);
// If we don't need to commit index based on what index writer tells us, we will still want
// to commit the index if it's assigned to a partition and this is no longer the active index
// for that partition. This prevents the following case:
//
// Thread T1: pulls events from queue
// Maps events to Index Directory D1
// Thread T2: pulls events from queue
// Maps events to Index Directory D1, the active index for Partition P1.
// Writes events to D1.
// Commits Index Writer for D1.
// Closes Index Writer for D1.
// Thread T1: Writes events to D1.
// Determines that Index Writer for D1 does not need to be committed or closed.
//
// In the case outlined above, we would potentially lose those events from the index! To avoid this,
// we simply decide to commit the index if this writer is no longer the active writer for the index.
// However, if we have 10 threads, we don't want all 10 threads trying to commit the index after each
// update. We want to commit when they've all finished. This is what the IndexManager will do if we request
// that it commit the index. It will also close the index if requested, once all writers have finished.
// So when this is the case, we will request that the Index Manager both commit and close the writer.
final Optional<File> activeIndexDirOption = directoryManager.getActiveIndexDirectory(partitionName);
if (!activeIndexDirOption.isPresent() || !activeIndexDirOption.get().equals(indexDirectory)) {
requestCommit = true;
requestClose = true;
}
if (writerIndicatesCommit) {
commit(indexWriter);
// we've already committed the index writer so no need to request that the index manager do so also.
requestCommit = false;
final boolean directoryManagerIndicatesClose = directoryManager.onIndexCommitted(indexDirectory);
requestClose = requestClose || directoryManagerIndicatesClose;
if (logger.isDebugEnabled()) {
final long maxId = documents.stream().mapToLong(doc -> doc.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue()).max().orElse(-1L);
logger.debug("Committed index {} after writing a max Event ID of {}", indexDirectory, maxId);
}
}
} finally {
indexManager.returnIndexWriter(indexWriter, requestCommit, requestClose);
}
}
use of org.apache.nifi.provenance.index.EventIndexWriter in project nifi by apache.
the class CachingIndexManager method borrowIndexWriter.
@Override
public EventIndexWriter borrowIndexWriter(final File indexDirectory) throws IOException {
final File absoluteFile = indexDirectory.getAbsoluteFile();
logger.trace("Borrowing index writer for {}", indexDirectory);
lock.lock();
try {
IndexWriterCount writerCount = writerCounts.remove(absoluteFile);
if (writerCount == null) {
final List<Closeable> closeables = new ArrayList<>();
final Directory directory = FSDirectory.open(indexDirectory);
closeables.add(directory);
try {
final Analyzer analyzer = new StandardAnalyzer();
closeables.add(analyzer);
final IndexWriterConfig config = new IndexWriterConfig(LuceneUtil.LUCENE_VERSION, analyzer);
config.setWriteLockTimeout(300000L);
final IndexWriter indexWriter = new IndexWriter(directory, config);
final EventIndexWriter eventIndexWriter = new LuceneEventIndexWriter(indexWriter, indexDirectory);
writerCount = new IndexWriterCount(eventIndexWriter, analyzer, directory, 1);
logger.debug("Providing new index writer for {}", indexDirectory);
} catch (final IOException ioe) {
for (final Closeable closeable : closeables) {
try {
closeable.close();
} catch (final IOException ioe2) {
ioe.addSuppressed(ioe2);
}
}
throw ioe;
}
writerCounts.put(absoluteFile, writerCount);
// Mark any active searchers as poisoned because we are updating the index
final List<ActiveIndexSearcher> searchers = activeSearchers.get(absoluteFile);
if (searchers != null) {
for (final ActiveIndexSearcher activeSearcher : searchers) {
logger.debug("Poisoning {} because it is searching {}, which is getting updated", activeSearcher, indexDirectory);
activeSearcher.poison();
}
}
} else {
logger.debug("Providing existing index writer for {} and incrementing count to {}", indexDirectory, writerCount.getCount() + 1);
writerCounts.put(absoluteFile, new IndexWriterCount(writerCount.getWriter(), writerCount.getAnalyzer(), writerCount.getDirectory(), writerCount.getCount() + 1));
}
return writerCount.getWriter();
} finally {
lock.unlock();
}
}
use of org.apache.nifi.provenance.index.EventIndexWriter in project nifi by apache.
the class DeleteIndexAction method execute.
@Override
public File execute(final File expiredFile) throws IOException {
// count the number of records and determine the max event id that we are deleting.
final long numDeleted = 0;
long maxEventId = -1L;
try (final RecordReader reader = RecordReaders.newRecordReader(expiredFile, repository.getAllLogFiles(), Integer.MAX_VALUE)) {
maxEventId = reader.getMaxEventId();
} catch (final IOException ioe) {
logger.warn("Failed to obtain max ID present in journal file {}", expiredFile.getAbsolutePath());
}
// remove the records from the index
final List<File> indexDirs = indexConfiguration.getIndexDirectories(expiredFile);
for (final File indexingDirectory : indexDirs) {
final Term term = new Term(FieldNames.STORAGE_FILENAME, LuceneUtil.substringBefore(expiredFile.getName(), "."));
boolean deleteDir = false;
final EventIndexWriter writer = indexManager.borrowIndexWriter(indexingDirectory);
try {
final IndexWriter indexWriter = writer.getIndexWriter();
indexWriter.deleteDocuments(term);
indexWriter.commit();
final int docsLeft = indexWriter.numDocs();
deleteDir = docsLeft <= 0;
logger.debug("After expiring {}, there are {} docs left for index {}", expiredFile, docsLeft, indexingDirectory);
} finally {
indexManager.returnIndexWriter(writer);
}
// we've confirmed that all documents have been removed. Delete the index directory.
if (deleteDir) {
indexManager.removeIndex(indexingDirectory);
indexConfiguration.removeIndexDirectory(indexingDirectory);
deleteDirectory(indexingDirectory);
logger.info("Removed empty index directory {}", indexingDirectory);
}
}
// Update the minimum index to 1 more than the max Event ID in this file.
if (maxEventId > -1L) {
indexConfiguration.setMinIdIndexed(maxEventId + 1L);
}
logger.info("Deleted Indices for Expired Provenance File {} from {} index files; {} documents removed", expiredFile, indexDirs.size(), numDeleted);
return expiredFile;
}
use of org.apache.nifi.provenance.index.EventIndexWriter in project nifi by apache.
the class SimpleIndexManager method borrowIndexSearcher.
@Override
public EventIndexSearcher borrowIndexSearcher(final File indexDir) throws IOException {
final File absoluteFile = indexDir.getAbsoluteFile();
final IndexWriterCount writerCount;
synchronized (writerCounts) {
writerCount = writerCounts.remove(absoluteFile);
if (writerCount != null) {
// Increment writer count and create an Index Searcher based on the writer
writerCounts.put(absoluteFile, new IndexWriterCount(writerCount.getWriter(), writerCount.getAnalyzer(), writerCount.getDirectory(), writerCount.getCount() + 1, writerCount.isCloseableWhenUnused()));
}
}
final DirectoryReader directoryReader;
if (writerCount == null) {
logger.trace("Creating index searcher for {}", indexDir);
final Directory directory = FSDirectory.open(indexDir);
directoryReader = DirectoryReader.open(directory);
} else {
final EventIndexWriter eventIndexWriter = writerCount.getWriter();
directoryReader = DirectoryReader.open(eventIndexWriter.getIndexWriter(), false);
}
final IndexSearcher searcher = new IndexSearcher(directoryReader, this.searchExecutor);
logger.trace("Created index searcher {} for {}", searcher, indexDir);
return new LuceneEventIndexSearcher(searcher, indexDir, null, directoryReader);
}
Aggregations