Examples with RecordWriter org.apache.nifi.provenance.serialization.RecordWriter used on opensource projects

Example 26 with RecordWriter

use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.

the class TestSchemaRecordReaderWriter method testReadPerformance.

@Test
@Ignore("For local performance testing only")
public void testReadPerformance() throws IOException, InterruptedException {
    // This is a simple micro-benchmarking test so that we can determine how fast the serialization/deserialization is before
    // making significant changes. This allows us to ensure that changes that we make do not have significant adverse effects
    // on performance of the repository.
    final ProvenanceEventRecord event = createEvent();
    final TocReader tocReader = null;
    final byte[] header;
    try (final ByteArrayOutputStream headerOut = new ByteArrayOutputStream();
        final DataOutputStream out = new DataOutputStream(headerOut)) {
        final RecordWriter schemaWriter = new ByteArraySchemaRecordWriter(out, "out", idGenerator, null, false, 0);
        schemaWriter.writeHeader(1L);
        header = headerOut.toByteArray();
    }
    final byte[] serializedRecord;
    try (final ByteArrayOutputStream headerOut = new ByteArrayOutputStream();
        final RecordWriter writer = new ByteArraySchemaRecordWriter(headerOut, "out", idGenerator, null, false, 0)) {
        writer.writeHeader(1L);
        headerOut.reset();
        writer.writeRecord(event);
        writer.flush();
        serializedRecord = headerOut.toByteArray();
    }
    final int numEvents = 10_000_000;
    final int recordBytes = serializedRecord.length;
    final long totalRecordBytes = (long) recordBytes * (long) numEvents;
    final long startNanos = System.nanoTime();
    try (final InputStream in = new LoopingInputStream(header, serializedRecord);
        final RecordReader reader = new ByteArraySchemaRecordReader(in, "filename", tocReader, 100000)) {
        for (int i = 0; i < numEvents; i++) {
            reader.nextRecord();
        }
    }
    final long nanos = System.nanoTime() - startNanos;
    final long millis = TimeUnit.NANOSECONDS.toMillis(nanos);
    final double seconds = millis / 1000D;
    final long bytesPerSecond = (long) (totalRecordBytes / seconds);
    final long megaBytesPerSecond = bytesPerSecond / 1024 / 1024;
    System.out.println("Took " + millis + " millis to read " + numEvents + " events or " + megaBytesPerSecond + " MB/sec");
}

Example 27 with RecordWriter

use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.

the class TestStandardRecordReaderWriter method testWritePerformance.

@Test
@Ignore("For local testing only")
public void testWritePerformance() throws IOException {
    // This is a simple micro-benchmarking test so that we can determine how fast the serialization/deserialization is before
    // making significant changes. This allows us to ensure that changes that we make do not have significant adverse effects
    // on performance of the repository.
    final ProvenanceEventRecord event = createEvent();
    final TocWriter tocWriter = new NopTocWriter();
    final int numEvents = 10_000_000;
    final long startNanos = System.nanoTime();
    try (final OutputStream nullOut = new NullOutputStream();
        final RecordWriter writer = new StandardRecordWriter(nullOut, "devnull", idGenerator, tocWriter, false, 100000)) {
        writer.writeHeader(0L);
        for (int i = 0; i < numEvents; i++) {
            writer.writeRecord(event);
        }
    }
    final long nanos = System.nanoTime() - startNanos;
    final long millis = TimeUnit.NANOSECONDS.toMillis(nanos);
    System.out.println("Took " + millis + " millis to write " + numEvents + " events");
}

Example 28 with RecordWriter

use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.

the class PersistentProvenanceRepository method close.

@Override
public synchronized void close() throws IOException {
    this.closed.set(true);
    writeLock.lock();
    try {
        logger.debug("Obtained write lock for close");
        scheduledExecService.shutdownNow();
        rolloverExecutor.shutdownNow();
        queryExecService.shutdownNow();
        getIndexManager().close();
        if (writers != null) {
            for (final RecordWriter writer : writers) {
                writer.close();
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 29 with RecordWriter

use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.

the class WriteAheadStorePartition method tryRollover.

private synchronized boolean tryRollover(final RecordWriterLease lease) throws IOException {
    if (!Objects.equals(lease, eventWriterLeaseRef.get())) {
        return false;
    }
    final long nextEventId = idGenerator.get();
    final File updatedEventFile = new File(partitionDirectory, nextEventId + ".prov");
    final RecordWriter updatedWriter = recordWriterFactory.createWriter(updatedEventFile, idGenerator, false, true);
    updatedWriter.writeHeader(nextEventId);
    final RecordWriterLease updatedLease = new RecordWriterLease(updatedWriter, config.getMaxEventFileCapacity(), config.getMaxEventFileCount());
    final boolean updated = eventWriterLeaseRef.compareAndSet(lease, updatedLease);
    if (!updated) {
        try {
            updatedWriter.close();
        } catch (final Exception e) {
            logger.warn("Failed to close Record Writer {}; some resources may not be cleaned up properly.", updatedWriter, e);
        }
        updatedEventFile.delete();
        return false;
    }
    if (lease != null) {
        lease.close();
    }
    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.put(nextEventId, updatedEventFile);
    }
    if (config.isCompressOnRollover() && lease != null && lease.getWriter() != null) {
        boolean offered = false;
        while (!offered && !closed) {
            try {
                offered = filesToCompress.offer(lease.getWriter().getFile(), 1, TimeUnit.SECONDS);
            } catch (final InterruptedException ie) {
                Thread.currentThread().interrupt();
                throw new IOException("Interrupted while waiting to enqueue " + lease.getWriter().getFile() + " for compression");
            }
        }
    }
    return true;
}

Example 30 with RecordWriter

use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.

the class WriteAheadStorePartition method addEvents.

@Override
public StorageResult addEvents(final Iterable<ProvenanceEventRecord> events) throws IOException {
    if (closed) {
        throw new IOException(this + " is closed");
    }
    // Claim a Record Writer Lease so that we have a writer to persist the events to
    boolean claimed = false;
    RecordWriterLease lease = null;
    while (!claimed) {
        lease = getLease();
        claimed = lease.tryClaim();
        if (claimed) {
            break;
        }
        if (lease.shouldRoll()) {
            tryRollover(lease);
        }
    }
    // Add the events to the writer and ensure that we always
    // relinquish the claim that we've obtained on the writer
    Map<ProvenanceEventRecord, StorageSummary> storageMap;
    final RecordWriter writer = lease.getWriter();
    try {
        storageMap = addEvents(events, writer);
    } finally {
        lease.relinquishClaim();
    }
    // Roll over the writer if necessary
    Integer eventsRolledOver = null;
    final boolean shouldRoll = lease.shouldRoll();
    try {
        if (shouldRoll && tryRollover(lease)) {
            eventsRolledOver = writer.getRecordsWritten();
        }
    } catch (final IOException ioe) {
        logger.error("Updated {} but failed to rollover to a new Event File", this, ioe);
    }
    final Integer rolloverCount = eventsRolledOver;
    return new StorageResult() {

        @Override
        public Map<ProvenanceEventRecord, StorageSummary> getStorageLocations() {
            return storageMap;
        }

        @Override
        public boolean triggeredRollover() {
            return rolloverCount != null;
        }

        @Override
        public Integer getEventsRolledOver() {
            return rolloverCount;
        }

        @Override
        public String toString() {
            return getStorageLocations().toString();
        }
    };
}