Examples with DefaultFileSystemAbstraction org.neo4j.io.fs.DefaultFileSystemAbstraction used on opensource projects

Example 66 with DefaultFileSystemAbstraction

use of org.neo4j.io.fs.DefaultFileSystemAbstraction in project neo4j by neo4j.

the class QuickImport method main.

public static void main(String[] arguments) throws IOException {
    Args args = Args.parse(arguments);
    long nodeCount = parseLongWithUnit(args.get("nodes", null));
    long relationshipCount = parseLongWithUnit(args.get("relationships", null));
    int labelCount = args.getNumber("labels", 4).intValue();
    int relationshipTypeCount = args.getNumber("relationship-types", 4).intValue();
    Path dir = Path.of(args.get("into"));
    long randomSeed = args.getNumber("random-seed", currentTimeMillis()).longValue();
    Configuration config = Configuration.COMMAS;
    Extractors extractors = new Extractors(config.arrayDelimiter());
    IdType idType = IdType.valueOf(args.get("id-type", IdType.INTEGER.name()));
    Groups groups = new Groups();
    Header nodeHeader = parseNodeHeader(args, idType, extractors, groups);
    Header relationshipHeader = parseRelationshipHeader(args, idType, extractors, groups);
    Config dbConfig;
    String dbConfigFileName = args.get("db-config", null);
    if (dbConfigFileName != null) {
        dbConfig = Config.newBuilder().fromFile(Path.of(dbConfigFileName)).build();
    } else {
        dbConfig = Config.defaults();
    }
    Boolean highIo = args.has("high-io") ? args.getBoolean("high-io") : null;
    LogProvider logging = NullLogProvider.getInstance();
    long pageCacheMemory = args.getNumber("pagecache-memory", org.neo4j.internal.batchimport.Configuration.MAX_PAGE_CACHE_MEMORY).longValue();
    org.neo4j.internal.batchimport.Configuration importConfig = new org.neo4j.internal.batchimport.Configuration.Overridden(defaultConfiguration(dir)) {

        @Override
        public int maxNumberOfProcessors() {
            return args.getNumber("processors", super.maxNumberOfProcessors()).intValue();
        }

        @Override
        public boolean highIO() {
            return highIo != null ? highIo : super.highIO();
        }

        @Override
        public long pageCacheMemory() {
            return pageCacheMemory;
        }

        @Override
        public long maxMemoryUsage() {
            String custom = args.get("max-memory", null);
            return custom != null ? parseMaxMemory(custom) : super.maxMemoryUsage();
        }

        @Override
        public IndexConfig indexConfig() {
            return IndexConfig.create().withLabelIndex().withRelationshipTypeIndex();
        }
    };
    float factorBadNodeData = args.getNumber("factor-bad-node-data", 0).floatValue();
    float factorBadRelationshipData = args.getNumber("factor-bad-relationship-data", 0).floatValue();
    Input input = new DataGeneratorInput(nodeCount, relationshipCount, idType, randomSeed, 0, nodeHeader, relationshipHeader, labelCount, relationshipTypeCount, factorBadNodeData, factorBadRelationshipData);
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        Lifespan life = new Lifespan()) {
        BatchImporter consumer;
        if (args.getBoolean("to-csv")) {
            consumer = new CsvOutput(dir, nodeHeader, relationshipHeader, config);
        } else {
            System.out.println("Seed " + randomSeed);
            final JobScheduler jobScheduler = life.add(createScheduler());
            boolean verbose = args.getBoolean("v");
            ExecutionMonitor monitor = verbose ? new SpectrumExecutionMonitor(2, TimeUnit.SECONDS, System.out, 100) : defaultVisible();
            consumer = BatchImporterFactory.withHighestPriority().instantiate(DatabaseLayout.ofFlat(dir), fileSystem, PageCacheTracer.NULL, importConfig, new SimpleLogService(logging, logging), monitor, EMPTY, dbConfig, RecordFormatSelector.selectForConfig(dbConfig, logging), NO_MONITOR, jobScheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), new IndexImporterFactoryImpl(dbConfig), INSTANCE);
        }
        consumer.doImport(input);
    }
}

Example 67 with DefaultFileSystemAbstraction

use of org.neo4j.io.fs.DefaultFileSystemAbstraction in project neo4j by neo4j.

the class PartitionedIndexStorageTest method shouldListIndexPartitionsSorted.

@Test
void shouldListIndexPartitionsSorted() throws Exception {
    // GIVEN
    try (FileSystemAbstraction scramblingFs = new DefaultFileSystemAbstraction() {

        @Override
        public Path[] listFiles(Path directory) throws IOException {
            List<Path> files = asList(super.listFiles(directory));
            Collections.shuffle(files);
            return files.toArray(new Path[0]);
        }
    }) {
        PartitionedIndexStorage myStorage = new PartitionedIndexStorage(directoryFactory, scramblingFs, testDir.homePath());
        Path parent = myStorage.getIndexFolder();
        int directoryCount = 10;
        for (int i = 0; i < directoryCount; i++) {
            scramblingFs.mkdirs(parent.resolve(String.valueOf(i + 1)));
        }
        // WHEN
        Map<Path, Directory> directories = myStorage.openIndexDirectories();
        // THEN
        assertEquals(directoryCount, directories.size());
        int previous = 0;
        for (Map.Entry<Path, Directory> directory : directories.entrySet()) {
            int current = parseInt(directory.getKey().getFileName().toString());
            assertTrue(current > previous, "Wanted directory " + current + " to have higher id than previous " + previous);
            previous = current;
        }
    }
}

Example 68 with DefaultFileSystemAbstraction

use of org.neo4j.io.fs.DefaultFileSystemAbstraction in project neo4j by neo4j.

the class PageCacheSupportExtension method createField.

@Override
protected PageCache createField(ExtensionContext extensionContext) {
    Store fileSystemStore = getStore(extensionContext, FileSystemExtension.FILE_SYSTEM_NAMESPACE);
    FileSystemAbstraction contextFileSystem = fileSystemStore.get(FileSystemExtension.FILE_SYSTEM, FileSystemAbstraction.class);
    FileSystemAbstraction pageCacheFileSystem = Optional.ofNullable(contextFileSystem).orElseGet(DefaultFileSystemAbstraction::new);
    return new PageCacheRule().getPageCache(pageCacheFileSystem, config);
}

Example 69 with DefaultFileSystemAbstraction

use of org.neo4j.io.fs.DefaultFileSystemAbstraction in project neo4j by neo4j.

the class PageCacheStressTest method run.

public void run() throws Exception {
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
        JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCacheUnderTest = new MuninnPageCache(swapperFactory, jobScheduler, config(numberOfCachePages).pageCacheTracer(tracer))) {
            PageCacheStresser pageCacheStresser = new PageCacheStresser(numberOfPages, numberOfThreads, workingDirectory);
            pageCacheStresser.stress(pageCacheUnderTest, tracer, condition);
        }
    }
}

Example 70 with DefaultFileSystemAbstraction

use of org.neo4j.io.fs.DefaultFileSystemAbstraction in project neo4j by neo4j.

the class SingleFilePageSwapperTest method setUp.

@BeforeEach
void setUp() {
    path = Path.of("file").normalize();
    ephemeralFileSystem = new EphemeralFileSystemAbstraction();
    fileSystem = new DefaultFileSystemAbstraction();
    threadRegistryFactory = new ThreadRegistryFactory();
    operationExecutor = Executors.newSingleThreadExecutor(threadRegistryFactory);
}