use of org.neo4j.logging.internal.SimpleLogService in project neo4j by neo4j.
the class StoreMigratorTest method shouldGenerateTransactionInformationWhenLogsAreEmpty.
@Test
void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Exception {
// given
long txId = 1;
Path neoStore = databaseLayout.metadataStore();
Files.createFile(neoStore);
Config config = mock(Config.class);
LogService logService = new SimpleLogService(NullLogProvider.getInstance(), NullLogProvider.getInstance());
// when
// ... transaction info not in neo store
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_ID, databaseLayout.getDatabaseName(), NULL));
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_CHECKSUM, databaseLayout.getDatabaseName(), NULL));
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP, databaseLayout.getDatabaseName(), NULL));
// ... and with migrator
RecordStorageMigrator migrator = new RecordStorageMigrator(fileSystem, pageCache, config, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
TransactionId actual = migrator.extractTransactionIdInformation(neoStore, txId, databaseLayout, NULL);
// then
assertEquals(txId, actual.transactionId());
assertEquals(TransactionIdStore.BASE_TX_CHECKSUM, actual.checksum());
assertEquals(TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP, actual.commitTimestamp());
}
use of org.neo4j.logging.internal.SimpleLogService in project neo4j by neo4j.
the class StoreMigratorTest method extractTransactionalInformationFromLogs.
private void extractTransactionalInformationFromLogs(Path customLogsLocation) throws IOException {
Config config = Config.defaults(transaction_logs_root_path, customLogsLocation);
LogService logService = new SimpleLogService(NullLogProvider.getInstance(), NullLogProvider.getInstance());
DatabaseManagementService managementService = new TestDatabaseManagementServiceBuilder(databaseLayout).setConfig(transaction_logs_root_path, customLogsLocation).build();
GraphDatabaseAPI database = (GraphDatabaseAPI) managementService.database(DEFAULT_DATABASE_NAME);
StorageEngineFactory storageEngineFactory = database.getDependencyResolver().resolveDependency(StorageEngineFactory.class);
for (int i = 0; i < 10; i++) {
try (Transaction transaction = database.beginTx()) {
transaction.createNode();
transaction.commit();
}
}
DatabaseLayout databaseLayout = database.databaseLayout();
Path neoStore = databaseLayout.metadataStore();
managementService.shutdown();
MetaDataStore.setRecord(pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, MetaDataRecordFormat.FIELD_NOT_PRESENT, databaseLayout.getDatabaseName(), NULL);
RecordStorageMigrator migrator = new RecordStorageMigrator(fileSystem, pageCache, config, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
LogPosition logPosition = migrator.extractTransactionLogPosition(neoStore, databaseLayout, 100, NULL);
LogFiles logFiles = LogFilesBuilder.activeFilesBuilder(databaseLayout, fileSystem, pageCache).withConfig(config).withCommandReaderFactory(storageEngineFactory.commandReaderFactory()).build();
assertEquals(0, logPosition.getLogVersion());
assertEquals(Files.size(logFiles.getLogFile().getHighestLogFile()), logPosition.getByteOffset());
}
use of org.neo4j.logging.internal.SimpleLogService in project neo4j by neo4j.
the class RecordStorageMigratorIT method shouldBeAbleToMigrateWithoutErrors.
@ParameterizedTest
@MethodSource("versions")
void shouldBeAbleToMigrateWithoutErrors(String version, LogPosition expectedLogPosition, Function<TransactionId, Boolean> txIdComparator) throws Exception {
// GIVEN a legacy database
Path prepare = testDirectory.directory("prepare");
var fs = testDirectory.getFileSystem();
MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, databaseLayout.databaseDirectory(), prepare);
AssertableLogProvider logProvider = new AssertableLogProvider(true);
LogService logService = new SimpleLogService(logProvider, logProvider);
RecordStoreVersionCheck check = getVersionCheck(pageCache, databaseLayout);
String versionToMigrateFrom = getVersionToMigrateFrom(check);
RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
// WHEN migrating
migrator.migrate(databaseLayout, migrationLayout, progressMonitor.startSection("section"), versionToMigrateFrom, getVersionToMigrateTo(check), EMPTY);
migrator.moveMigratedFiles(migrationLayout, databaseLayout, versionToMigrateFrom, getVersionToMigrateTo(check));
// THEN starting the new store should be successful
assertThat(testDirectory.getFileSystem().fileExists(databaseLayout.relationshipGroupDegreesStore())).isTrue();
GBPTreeRelationshipGroupDegreesStore.DegreesRebuilder noRebuildAssertion = new GBPTreeRelationshipGroupDegreesStore.DegreesRebuilder() {
@Override
public void rebuild(RelationshipGroupDegreesStore.Updater updater, CursorContext cursorContext, MemoryTracker memoryTracker) {
throw new IllegalStateException("Rebuild should not be required");
}
@Override
public long lastCommittedTxId() {
try {
return new RecordStorageEngineFactory().readOnlyTransactionIdStore(fs, databaseLayout, pageCache, NULL).getLastCommittedTransactionId();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
};
try (GBPTreeRelationshipGroupDegreesStore groupDegreesStore = new GBPTreeRelationshipGroupDegreesStore(pageCache, databaseLayout.relationshipGroupDegreesStore(), testDirectory.getFileSystem(), immediate(), noRebuildAssertion, writable(), PageCacheTracer.NULL, GBPTreeGenericCountsStore.NO_MONITOR, databaseLayout.getDatabaseName(), counts_store_max_cached_entries.defaultValue())) {
// The rebuild would happen here in start and will throw exception (above) if invoked
groupDegreesStore.start(NULL, INSTANCE);
// The store keeps track of committed transactions.
// It is essential that it starts with the transaction
// that is the last committed one at the upgrade time.
assertEquals(TX_ID, groupDegreesStore.txId());
}
StoreFactory storeFactory = new StoreFactory(databaseLayout, CONFIG, new ScanOnOpenOverwritingIdGeneratorFactory(fs, databaseLayout.getDatabaseName()), pageCache, fs, logService.getInternalLogProvider(), PageCacheTracer.NULL, writable());
storeFactory.openAllNeoStores().close();
assertThat(logProvider).forLevel(ERROR).doesNotHaveAnyLogs();
}
use of org.neo4j.logging.internal.SimpleLogService in project neo4j by neo4j.
the class StoreMigratorTest method shouldGenerateTransactionInformationWhenLogsNotPresent.
@Test
void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exception {
// given
long txId = 42;
Path neoStore = databaseLayout.metadataStore();
Files.createFile(neoStore);
Config config = mock(Config.class);
LogService logService = new SimpleLogService(NullLogProvider.getInstance(), NullLogProvider.getInstance());
// when
// ... transaction info not in neo store
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_ID, databaseLayout.getDatabaseName(), NULL));
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_CHECKSUM, databaseLayout.getDatabaseName(), NULL));
assertEquals(FIELD_NOT_PRESENT, getRecord(pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP, databaseLayout.getDatabaseName(), NULL));
// ... and with migrator
RecordStorageMigrator migrator = new RecordStorageMigrator(fileSystem, pageCache, config, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
TransactionId actual = migrator.extractTransactionIdInformation(neoStore, txId, databaseLayout, NULL);
// then
assertEquals(txId, actual.transactionId());
assertEquals(TransactionIdStore.UNKNOWN_TX_CHECKSUM, actual.checksum());
assertEquals(TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP, actual.commitTimestamp());
}
use of org.neo4j.logging.internal.SimpleLogService in project neo4j by neo4j.
the class QuickImport method main.
public static void main(String[] arguments) throws IOException {
Args args = Args.parse(arguments);
long nodeCount = parseLongWithUnit(args.get("nodes", null));
long relationshipCount = parseLongWithUnit(args.get("relationships", null));
int labelCount = args.getNumber("labels", 4).intValue();
int relationshipTypeCount = args.getNumber("relationship-types", 4).intValue();
Path dir = Path.of(args.get("into"));
long randomSeed = args.getNumber("random-seed", currentTimeMillis()).longValue();
Configuration config = Configuration.COMMAS;
Extractors extractors = new Extractors(config.arrayDelimiter());
IdType idType = IdType.valueOf(args.get("id-type", IdType.INTEGER.name()));
Groups groups = new Groups();
Header nodeHeader = parseNodeHeader(args, idType, extractors, groups);
Header relationshipHeader = parseRelationshipHeader(args, idType, extractors, groups);
Config dbConfig;
String dbConfigFileName = args.get("db-config", null);
if (dbConfigFileName != null) {
dbConfig = Config.newBuilder().fromFile(Path.of(dbConfigFileName)).build();
} else {
dbConfig = Config.defaults();
}
Boolean highIo = args.has("high-io") ? args.getBoolean("high-io") : null;
LogProvider logging = NullLogProvider.getInstance();
long pageCacheMemory = args.getNumber("pagecache-memory", org.neo4j.internal.batchimport.Configuration.MAX_PAGE_CACHE_MEMORY).longValue();
org.neo4j.internal.batchimport.Configuration importConfig = new org.neo4j.internal.batchimport.Configuration.Overridden(defaultConfiguration(dir)) {
@Override
public int maxNumberOfProcessors() {
return args.getNumber("processors", super.maxNumberOfProcessors()).intValue();
}
@Override
public boolean highIO() {
return highIo != null ? highIo : super.highIO();
}
@Override
public long pageCacheMemory() {
return pageCacheMemory;
}
@Override
public long maxMemoryUsage() {
String custom = args.get("max-memory", null);
return custom != null ? parseMaxMemory(custom) : super.maxMemoryUsage();
}
@Override
public IndexConfig indexConfig() {
return IndexConfig.create().withLabelIndex().withRelationshipTypeIndex();
}
};
float factorBadNodeData = args.getNumber("factor-bad-node-data", 0).floatValue();
float factorBadRelationshipData = args.getNumber("factor-bad-relationship-data", 0).floatValue();
Input input = new DataGeneratorInput(nodeCount, relationshipCount, idType, randomSeed, 0, nodeHeader, relationshipHeader, labelCount, relationshipTypeCount, factorBadNodeData, factorBadRelationshipData);
try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
Lifespan life = new Lifespan()) {
BatchImporter consumer;
if (args.getBoolean("to-csv")) {
consumer = new CsvOutput(dir, nodeHeader, relationshipHeader, config);
} else {
System.out.println("Seed " + randomSeed);
final JobScheduler jobScheduler = life.add(createScheduler());
boolean verbose = args.getBoolean("v");
ExecutionMonitor monitor = verbose ? new SpectrumExecutionMonitor(2, TimeUnit.SECONDS, System.out, 100) : defaultVisible();
consumer = BatchImporterFactory.withHighestPriority().instantiate(DatabaseLayout.ofFlat(dir), fileSystem, PageCacheTracer.NULL, importConfig, new SimpleLogService(logging, logging), monitor, EMPTY, dbConfig, RecordFormatSelector.selectForConfig(dbConfig, logging), NO_MONITOR, jobScheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), new IndexImporterFactoryImpl(dbConfig), INSTANCE);
}
consumer.doImport(input);
}
}
Aggregations