use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.
the class TransactionLogFileTest method shouldWaitForOngoingForceToCompleteBeforeForcingAgain.
@Test
void shouldWaitForOngoingForceToCompleteBeforeForcingAgain() throws Throwable {
LogFiles logFiles = buildLogFiles();
life.start();
life.add(logFiles);
LogFile logFile = logFiles.getLogFile();
var capturingChannel = wrappingFileSystem.getCapturingChannel();
ReentrantLock writeAllLock = capturingChannel.getWriteAllLock();
var flushesBefore = capturingChannel.getFlushCounter().get();
var writesBefore = capturingChannel.getWriteAllCounter().get();
writeAllLock.lock();
int executors = 10;
var executorService = Executors.newFixedThreadPool(executors);
try {
var future = executorService.submit(() -> logFile.forceAfterAppend(LogAppendEvent.NULL));
while (!writeAllLock.hasQueuedThreads()) {
parkNanos(100);
}
writeAllLock.unlock();
var future2 = executorService.submit(() -> logFile.forceAfterAppend(LogAppendEvent.NULL));
Futures.getAll(List.of(future, future2));
} finally {
if (writeAllLock.isLocked()) {
writeAllLock.unlock();
}
executorService.shutdownNow();
}
assertThat(capturingChannel.getWriteAllCounter().get() - writesBefore).isEqualTo(2);
assertThat(capturingChannel.getFlushCounter().get() - flushesBefore).isEqualTo(2);
}
use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.
the class TransactionLogFileTest method closeChannelThrowExceptionOnAttemptToAppendTransactionLogRecords.
@Test
void closeChannelThrowExceptionOnAttemptToAppendTransactionLogRecords() throws IOException {
LogFiles logFiles = buildLogFiles();
life.start();
life.add(logFiles);
LogFile logFile = logFiles.getLogFile();
var channel = logFile.getTransactionLogWriter().getChannel();
life.shutdown();
assertThrows(Throwable.class, () -> channel.put((byte) 7));
assertThrows(Throwable.class, () -> channel.putInt(7));
assertThrows(Throwable.class, () -> channel.putLong(7));
assertThrows(Throwable.class, () -> channel.putDouble(7));
assertThrows(Throwable.class, () -> channel.putFloat(7));
assertThrows(Throwable.class, () -> channel.putShort((short) 7));
assertThrows(Throwable.class, () -> channel.put(new byte[] { 1, 2, 3 }, 3));
assertThrows(IllegalStateException.class, logFile::flush);
}
use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.
the class TransactionLogFileRotateAndReadRaceIT method shouldNotSeeEmptyLogFileWhenReadingTransactionStream.
@Test
void shouldNotSeeEmptyLogFileWhenReadingTransactionStream() throws Exception {
// GIVEN
LogVersionRepository logVersionRepository = new SimpleLogVersionRepository();
Config cfg = Config.newBuilder().set(GraphDatabaseSettings.neo4j_home, databaseLayout.getNeo4jLayout().homeDirectory()).set(GraphDatabaseSettings.preallocate_logical_logs, false).set(GraphDatabaseSettings.logical_log_rotation_threshold, ByteUnit.kibiBytes(128)).build();
LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogVersionRepository(logVersionRepository).withTransactionIdStore(new SimpleTransactionIdStore()).withLogEntryReader(new VersionAwareLogEntryReader(new TestCommandReaderFactory())).withConfig(cfg).withStoreId(StoreId.UNKNOWN).build();
life.add(logFiles);
LogFile logFile = logFiles.getLogFile();
var writer = logFile.getTransactionLogWriter();
LogPositionMarker startPosition = new LogPositionMarker();
writer.getCurrentPosition(startPosition);
// WHEN
AtomicBoolean end = new AtomicBoolean();
byte[] dataChunk = new byte[100];
// one thread constantly writing to and rotating the channel
CountDownLatch startSignal = new CountDownLatch(1);
Future<Void> writeFuture = t2.execute(() -> {
ThreadLocalRandom random = ThreadLocalRandom.current();
startSignal.countDown();
int rotations = 0;
while (!end.get()) {
int bytesToWrite = random.nextInt(1, dataChunk.length);
writer.getChannel().put(dataChunk, bytesToWrite);
if (logFile.rotationNeeded()) {
logFile.rotate();
// Let's just close the gap to the reader so that it gets closer to the "hot zone"
// where the rotation happens.
writer.getCurrentPosition(startPosition);
if (++rotations > LIMIT_ROTATIONS) {
end.set(true);
}
}
}
return null;
});
assertTrue(startSignal.await(10, SECONDS));
// one thread reading through the channel
try {
int reads = 0;
while (!end.get()) {
try (ReadableLogChannel reader = logFile.getReader(startPosition.newPosition())) {
deplete(reader);
}
if (++reads > LIMIT_READS) {
end.set(true);
}
}
} finally {
writeFuture.get();
}
// THEN simply getting here means this was successful
}
use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.
the class RecoverIndexDropIT method appendDropTransactionToTransactionLog.
private void appendDropTransactionToTransactionLog(Path transactionLogsDirectory, CommittedTransactionRepresentation dropTransaction, StorageEngineFactory storageEngineFactory) throws IOException {
LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder(transactionLogsDirectory, fs).withCommandReaderFactory(storageEngineFactory.commandReaderFactory()).build();
LogFile logFile = logFiles.getLogFile();
try (ReadableLogChannel reader = logFile.getReader(logFile.extractHeader(0).getStartPosition())) {
LogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
while (logEntryReader.readLogEntry(reader) != null) {
}
LogPosition position = logEntryReader.lastPosition();
StoreChannel storeChannel = fs.write(logFile.getLogFileForVersion(logFile.getHighestLogVersion()));
storeChannel.position(position.getByteOffset());
try (PhysicalFlushableChecksumChannel writeChannel = new PhysicalFlushableChecksumChannel(storeChannel, new HeapScopedBuffer(100, INSTANCE))) {
new LogEntryWriter<>(writeChannel, KernelVersion.LATEST).serialize(dropTransaction);
}
}
}
use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.
the class DetachedLogTailScanner method findLogTail.
@Override
protected LogTailInformation findLogTail() {
LogFile logFile = logFiles.getLogFile();
long highestLogVersion = logFile.getHighestLogVersion();
long lowestLogVersion = logFile.getLowestLogVersion();
try {
var lastAccessibleCheckpoint = checkPointFile.findLatestCheckpoint();
if (lastAccessibleCheckpoint.isEmpty()) {
return noCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion);
}
var checkpoint = lastAccessibleCheckpoint.get();
// found checkpoint pointing to existing position in existing log file
if (isValidCheckpoint(logFile, checkpoint)) {
return validCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion, checkpoint);
}
if (failOnCorruptedLogFiles) {
var exceptionMessage = format("Last available %s checkpoint does not point to a valid location in transaction logs.", checkpoint);
throwUnableToCleanRecover(new RuntimeException(exceptionMessage));
}
// our last checkpoint is not valid (we have a pointer to non existent place) lets try to find last one that looks correct
List<CheckpointInfo> checkpointInfos = checkPointFile.reachableCheckpoints();
// we know that last one is not valid so no reason to double check that again
ListIterator<CheckpointInfo> reverseCheckpoints = checkpointInfos.listIterator(checkpointInfos.size() - 1);
while (reverseCheckpoints.hasPrevious()) {
CheckpointInfo previousCheckpoint = reverseCheckpoints.previous();
if (isValidCheckpoint(logFile, previousCheckpoint)) {
return validCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion, previousCheckpoint);
}
}
// we did not found any valid, we need to restore from the start if possible
return noCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion);
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
Aggregations