use of org.neo4j.unsafe.impl.batchimport.input.InputNode in project neo4j by neo4j.
the class CsvInputTest method shouldHandleMultipleInputGroups.
@Test
public void shouldHandleMultipleInputGroups() throws Exception {
// GIVEN multiple input groups, each with their own, specific, header
DataFactory<InputNode> group1 = data(":ID,name,kills:int,health:int\n" + "1,Jim,10,100\n" + "2,Abathur,0,200\n");
DataFactory<InputNode> group2 = data(":ID,type\n" + "3,zergling\n" + "4,csv\n");
Iterable<DataFactory<InputNode>> data = dataIterable(group1, group2);
Input input = new CsvInput(data, defaultFormatNodeFileHeader(), null, null, IdType.STRING, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
// WHEN iterating over them, THEN the expected data should come out
try (InputIterator<InputNode> nodes = input.nodes().iterator()) {
assertNode(nodes.next(), "1", properties("name", "Jim", "kills", 10, "health", 100), labels());
assertNode(nodes.next(), "2", properties("name", "Abathur", "kills", 0, "health", 200), labels());
assertNode(nodes.next(), "3", properties("type", "zergling"), labels());
assertNode(nodes.next(), "4", properties("type", "csv"), labels());
assertFalse(nodes.hasNext());
}
}
use of org.neo4j.unsafe.impl.batchimport.input.InputNode in project neo4j by neo4j.
the class CsvInputTest method shouldHaveIdSetAsPropertyIfIdHeaderEntryIsNamed.
@Test
public void shouldHaveIdSetAsPropertyIfIdHeaderEntryIsNamed() throws Exception {
// GIVEN
DataFactory<InputNode> data = data("myId:ID,name:string,level:int\n" + "abc,Mattias,1\n" + // this node is anonymous
"def,Johan,2\n");
Iterable<DataFactory<InputNode>> dataIterable = dataIterable(data);
Input input = new CsvInput(dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.STRING, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
// WHEN
try (ResourceIterator<InputNode> nodes = input.nodes().iterator()) {
// THEN
assertNode(nodes.next(), "abc", new Object[] { "myId", "abc", "name", "Mattias", "level", 1 }, labels());
assertNode(nodes.next(), "def", new Object[] { "myId", "def", "name", "Johan", "level", 2 }, labels());
assertFalse(nodes.hasNext());
}
}
use of org.neo4j.unsafe.impl.batchimport.input.InputNode in project neo4j by neo4j.
the class CsvInputTest method shouldTreatEmptyQuotedStringsAsNullIfConfiguredTo.
@Test
public void shouldTreatEmptyQuotedStringsAsNullIfConfiguredTo() throws Exception {
// GIVEN
Iterable<DataFactory<InputNode>> data = DataFactories.nodeData(CsvInputTest.<InputNode>data(":ID,one,two,three\n" + "1,\"\",,value"));
Configuration config = config(new Configuration.Overridden(COMMAS) {
@Override
public boolean emptyQuotedStringsAsNull() {
return true;
}
});
Input input = new CsvInput(data, defaultFormatNodeFileHeader(), null, null, IdType.INTEGER, config, silentBadCollector(0), getRuntime().availableProcessors());
// WHEN
try (InputIterator<InputNode> nodes = input.nodes().iterator()) {
InputNode node = nodes.next();
// THEN
assertNode(node, 1L, properties("three", "value"), labels());
assertFalse(nodes.hasNext());
}
}
use of org.neo4j.unsafe.impl.batchimport.input.InputNode in project neo4j by neo4j.
the class StoreMigrator method legacyNodesAsInput.
private InputIterable<InputNode> legacyNodesAsInput(NeoStores legacyStore, boolean requiresPropertyMigration, RecordCursors cursors) {
NodeStore store = legacyStore.getNodeStore();
final BiConsumer<InputNode, NodeRecord> propertyDecorator = propertyDecorator(requiresPropertyMigration, cursors);
return new StoreScanAsInputIterable<InputNode, NodeRecord>(store) {
@Override
protected InputNode inputEntityOf(NodeRecord record) {
InputNode node = new InputNode("legacy store", record.getId(), record.getId() * NodeRecordFormat.RECORD_SIZE, record.getId(), InputEntity.NO_PROPERTIES, record.getNextProp(), InputNode.NO_LABELS, record.getLabelField());
propertyDecorator.accept(node, record);
return node;
}
};
}
use of org.neo4j.unsafe.impl.batchimport.input.InputNode in project neo4j by neo4j.
the class StoreMigrator method migrateWithBatchImporter.
private void migrateWithBatchImporter(File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat) throws IOException {
prepareBatchImportMigration(storeDir, migrationDir, oldFormat, newFormat);
boolean requiresDynamicStoreMigration = !newFormat.dynamic().equals(oldFormat.dynamic());
boolean requiresPropertyMigration = !newFormat.property().equals(oldFormat.property()) || requiresDynamicStoreMigration;
File badFile = new File(storeDir, Configuration.BAD_FILE_NAME);
try (NeoStores legacyStore = instantiateLegacyStore(oldFormat, storeDir);
RecordCursors nodeInputCursors = new RecordCursors(legacyStore);
RecordCursors relationshipInputCursors = new RecordCursors(legacyStore);
OutputStream badOutput = new BufferedOutputStream(new FileOutputStream(badFile, false))) {
Configuration importConfig = new Configuration.Overridden(config);
AdditionalInitialIds additionalInitialIds = readAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
// We have to make sure to keep the token ids if we're migrating properties/labels
BatchImporter importer = new ParallelBatchImporter(migrationDir.getAbsoluteFile(), fileSystem, pageCache, importConfig, logService, withDynamicProcessorAssignment(migrationBatchImporterMonitor(legacyStore, progressMonitor, importConfig), importConfig), additionalInitialIds, config, newFormat);
InputIterable<InputNode> nodes = legacyNodesAsInput(legacyStore, requiresPropertyMigration, nodeInputCursors);
InputIterable<InputRelationship> relationships = legacyRelationshipsAsInput(legacyStore, requiresPropertyMigration, relationshipInputCursors);
importer.doImport(Inputs.input(nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), Collectors.badCollector(badOutput, 0)));
// During migration the batch importer doesn't necessarily writes all entities, depending on
// which stores needs migration. Node, relationship, relationship group stores are always written
// anyways and cannot be avoided with the importer, but delete the store files that weren't written
// (left empty) so that we don't overwrite those in the real store directory later.
Collection<StoreFile> storesToDeleteFromMigratedDirectory = new ArrayList<>();
storesToDeleteFromMigratedDirectory.add(StoreFile.NEO_STORE);
if (!requiresPropertyMigration) {
// We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.PROPERTY_STORE, StoreFile.PROPERTY_STRING_STORE, StoreFile.PROPERTY_ARRAY_STORE));
}
if (!requiresDynamicStoreMigration) {
// We didn't migrate labels (dynamic node labels) or any other dynamic store
storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.NODE_LABEL_STORE, StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, StoreFile.SCHEMA_STORE));
}
StoreFile.fileOperation(DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values());
// When migrating on a block device there might be some files only accessible via the page cache.
try {
Predicate<FileHandle> fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream().anyMatch(storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fileHandle.getFile().getName()));
pageCache.streamFilesRecursive(migrationDir).filter(fileHandlePredicate).forEach(FileHandle.HANDLE_DELETE);
} catch (NoSuchFileException e) {
// This means that we had no files only present in the page cache, this is fine.
}
}
}
Aggregations