use of org.neo4j.helpers.collection.Visitor in project neo4j by neo4j.
the class StoreCopyResponsePacker method packTransactionStreamResponse.
@Override
public <T> Response<T> packTransactionStreamResponse(RequestContext context, T response) {
final long toStartFrom = mandatoryStartTransactionId;
final long toEndAt = transactionIdStore.getLastCommittedTransactionId();
TransactionStream transactions = visitor -> {
if (toStartFrom > BASE_TX_ID && toStartFrom <= toEndAt) {
monitor.startStreamingTransactions(toStartFrom);
extractTransactions(toStartFrom, filterVisitor(visitor, toEndAt));
monitor.finishStreamingTransactions(toEndAt);
}
};
return new TransactionStreamResponse<>(response, storeId.get(), transactions, ResourceReleaser.NO_OP);
}
use of org.neo4j.helpers.collection.Visitor in project neo4j by neo4j.
the class StoreCopyClient method writeTransactionsToActiveLogFile.
private void writeTransactionsToActiveLogFile(File tempStoreDir, Response<?> response) throws Exception {
LifeSupport life = new LifeSupport();
try {
// Start the log and appender
PhysicalLogFiles logFiles = new PhysicalLogFiles(tempStoreDir, fs);
LogHeaderCache logHeaderCache = new LogHeaderCache(10);
ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache, tempStoreDir);
ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore(pageCache, tempStoreDir);
LogFile logFile = life.add(new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
readOnlyTransactionIdStore::getLastCommittedTransactionId, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), logHeaderCache));
life.start();
// Just write all transactions to the active log version. Remember that this is after a store copy
// where there are no logs, and the transaction stream we're about to write will probably contain
// transactions that goes some time back, before the last committed transaction id. So we cannot
// use a TransactionAppender, since it has checks for which transactions one can append.
FlushableChannel channel = logFile.getWriter();
final TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
final AtomicLong firstTxId = new AtomicLong(BASE_TX_ID);
response.accept(new Response.Handler() {
@Override
public void obligation(long txId) throws IOException {
throw new UnsupportedOperationException("Shouldn't be called");
}
@Override
public Visitor<CommittedTransactionRepresentation, Exception> transactions() {
return transaction -> {
long txId = transaction.getCommitEntry().getTxId();
if (firstTxId.compareAndSet(BASE_TX_ID, txId)) {
monitor.startReceivingTransactions(txId);
}
writer.append(transaction.getTransactionRepresentation(), txId);
return false;
};
}
});
long endTxId = firstTxId.get();
if (endTxId != BASE_TX_ID) {
monitor.finishReceivingTransactions(endTxId);
}
long currentLogVersion = logVersionRepository.getCurrentLogVersion();
writer.checkPoint(new LogPosition(currentLogVersion, LOG_HEADER_SIZE));
// And since we write this manually we need to set the correct transaction id in the
// header of the log that we just wrote.
File currentLogFile = logFiles.getLogFileForVersion(currentLogVersion);
writeLogHeader(fs, currentLogFile, currentLogVersion, max(BASE_TX_ID, endTxId - 1));
if (!forensics) {
// since we just create new log and put checkpoint into it with offset equals to
// LOG_HEADER_SIZE we need to update last transaction offset to be equal to this newly defined max
// offset otherwise next checkpoint that use last transaction offset will be created for non
// existing offset that is in most of the cases bigger than new log size.
// Recovery will treat that as last checkpoint and will not try to recover store till new
// last closed transaction offset will not overcome old one. Till that happens it will be
// impossible for recovery process to restore the store
File neoStore = new File(tempStoreDir, MetaDataStore.DEFAULT_NAME);
MetaDataStore.setRecord(pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, LOG_HEADER_SIZE);
}
} finally {
life.shutdown();
}
}
use of org.neo4j.helpers.collection.Visitor in project neo4j by neo4j.
the class BatchInserterImpl method repopulateAllIndexes.
private void repopulateAllIndexes() throws IOException, IndexEntryConflictException {
if (!labelsTouched) {
return;
}
final IndexRule[] rules = getIndexesNeedingPopulation();
final IndexPopulator[] populators = new IndexPopulator[rules.length];
// the store is uncontended at this point, so creating a local LockService is safe.
final NewIndexDescriptor[] descriptors = new NewIndexDescriptor[rules.length];
for (int i = 0; i < rules.length; i++) {
IndexRule rule = rules[i];
descriptors[i] = rule.getIndexDescriptor();
populators[i] = schemaIndexProviders.apply(rule.getProviderDescriptor()).getPopulator(rule.getId(), descriptors[i], new IndexSamplingConfig(config));
populators[i].create();
}
Visitor<NodeUpdates, IOException> propertyUpdateVisitor = updates -> {
for (int i = 0; i < descriptors.length; i++) {
Optional<IndexEntryUpdate> update = updates.forIndex(descriptors[i].schema());
if (update.isPresent()) {
try {
populators[i].add(Collections.singletonList(update.get()));
} catch (IndexEntryConflictException conflict) {
throw conflict.notAllowed(descriptors[i]);
}
}
}
return true;
};
List<NewIndexDescriptor> descriptorList = Arrays.asList(descriptors);
int[] labelIds = descriptorList.stream().mapToInt(index -> index.schema().getLabelId()).toArray();
int[] propertyKeyIds = descriptorList.stream().flatMapToInt(d -> Arrays.stream(d.schema().getPropertyIds())).toArray();
InitialNodeLabelCreationVisitor labelUpdateVisitor = new InitialNodeLabelCreationVisitor();
StoreScan<IOException> storeScan = indexStoreView.visitNodes(labelIds, (propertyKeyId) -> PrimitiveIntCollections.contains(propertyKeyIds, propertyKeyId), propertyUpdateVisitor, labelUpdateVisitor, true);
storeScan.run();
for (IndexPopulator populator : populators) {
populator.verifyDeferredConstraints(indexStoreView);
populator.close(true);
}
labelUpdateVisitor.close();
}
use of org.neo4j.helpers.collection.Visitor in project neo4j by neo4j.
the class DefaultRecoverySPI method startRecovery.
@Override
public Visitor<CommittedTransactionRepresentation, Exception> startRecovery() {
// Calling this method means that recovery is required, tell storage engine about it
// This method will be called before recovery actually starts and so will ensure that
// each store is aware that recovery will be performed. At this point all the stores have
// already started btw.
// Go and read more at {@link CommonAbstractStore#deleteIdGenerator()}
storageEngine.prepareForRecoveryRequired();
transactionsToApply = new TransactionQueue(10_000, (first, last) -> storageEngine.apply(first, RECOVERY));
recoveryVisitor = new RecoveryVisitor(transactionsToApply);
return recoveryVisitor;
}
use of org.neo4j.helpers.collection.Visitor in project neo4j by neo4j.
the class RecoveryTest method shouldRecoverExistingData.
@Test
public void shouldRecoverExistingData() throws Exception {
final PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory(), "log", fileSystemRule.get());
File file = logFiles.getLogFileForVersion(logVersion);
writeSomeData(file, new Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException>() {
@Override
public boolean visit(Pair<LogEntryWriter, Consumer<LogPositionMarker>> pair) throws IOException {
LogEntryWriter writer = pair.first();
Consumer<LogPositionMarker> consumer = pair.other();
LogPositionMarker marker = new LogPositionMarker();
// last committed tx
consumer.accept(marker);
LogPosition lastCommittedTxPosition = marker.newPosition();
writer.writeStartEntry(0, 1, 2L, 3L, new byte[0]);
lastCommittedTxStartEntry = new LogEntryStart(0, 1, 2L, 3L, new byte[0], lastCommittedTxPosition);
writer.writeCommitEntry(4L, 5L);
lastCommittedTxCommitEntry = new OnePhaseCommit(4L, 5L);
// check point pointing to the previously committed transaction
writer.writeCheckPointEntry(lastCommittedTxPosition);
expectedCheckPointEntry = new CheckPoint(lastCommittedTxPosition);
// tx committed after checkpoint
consumer.accept(marker);
writer.writeStartEntry(0, 1, 6L, 4L, new byte[0]);
expectedStartEntry = new LogEntryStart(0, 1, 6L, 4L, new byte[0], marker.newPosition());
writer.writeCommitEntry(5L, 7L);
expectedCommitEntry = new OnePhaseCommit(5L, 7L);
return true;
}
});
LifeSupport life = new LifeSupport();
Recovery.Monitor monitor = mock(Recovery.Monitor.class);
final AtomicBoolean recoveryRequired = new AtomicBoolean();
try {
StorageEngine storageEngine = mock(StorageEngine.class);
final LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>();
LatestCheckPointFinder finder = new LatestCheckPointFinder(logFiles, fileSystemRule.get(), reader);
LogHeaderCache logHeaderCache = new LogHeaderCache(10);
TransactionMetadataCache metadataCache = new TransactionMetadataCache(100);
LogFile logFile = life.add(new PhysicalLogFile(fileSystemRule.get(), logFiles, 50, () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, mock(PhysicalLogFile.Monitor.class), logHeaderCache));
LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore(logFile, metadataCache, reader);
life.add(new Recovery(new DefaultRecoverySPI(storageEngine, logFiles, fileSystemRule.get(), logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR) {
private int nr = 0;
@Override
public Visitor<CommittedTransactionRepresentation, Exception> startRecovery() {
recoveryRequired.set(true);
final Visitor<CommittedTransactionRepresentation, Exception> actual = super.startRecovery();
return new Visitor<CommittedTransactionRepresentation, Exception>() {
@Override
public boolean visit(CommittedTransactionRepresentation tx) throws Exception {
actual.visit(tx);
switch(nr++) {
case 0:
assertEquals(lastCommittedTxStartEntry, tx.getStartEntry());
assertEquals(lastCommittedTxCommitEntry, tx.getCommitEntry());
break;
case 1:
assertEquals(expectedStartEntry, tx.getStartEntry());
assertEquals(expectedCommitEntry, tx.getCommitEntry());
break;
default:
fail("Too many recovered transactions");
}
return false;
}
};
}
}, monitor));
life.start();
InOrder order = inOrder(monitor);
order.verify(monitor, times(1)).recoveryRequired(any(LogPosition.class));
order.verify(monitor, times(1)).recoveryCompleted(2);
assertTrue(recoveryRequired.get());
} finally {
life.shutdown();
}
}
Aggregations