Examples with FileSystemAbstraction org.neo4j.io.fs.FileSystemAbstraction used on opensource projects

Example 71 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class RandomPageCacheTestHarness method runIteration.

@SuppressWarnings("unchecked")
private void runIteration(long timeout, TimeUnit unit) throws Exception {
    assert filePageSize % recordFormat.getRecordSize() == 0 : "File page size must be a multiple of the record size";
    if (!fixedRandomSeed) {
        randomSeed = ThreadLocalRandom.current().nextLong();
    }
    FileSystemAbstraction fs = this.fs;
    File[] files = buildFileNames();
    RandomAdversary adversary = new RandomAdversary(mischiefRate, failureRate, errorRate);
    adversary.setProbabilityFactor(0.0);
    if (useAdversarialIO) {
        adversary.setSeed(randomSeed);
        fs = new AdversarialFileSystemAbstraction(adversary, fs);
    }
    PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory();
    swapperFactory.setFileSystemAbstraction(fs);
    MuninnPageCache cache = new MuninnPageCache(swapperFactory, cachePageCount, cachePageSize, tracer, cursorTracerSupplier);
    cache.setPrintExceptionsOnClose(false);
    Map<File, PagedFile> fileMap = new HashMap<>(files.length);
    for (int i = 0; i < Math.min(files.length, initialMappedFiles); i++) {
        File file = files[i];
        fileMap.put(file, cache.map(file, filePageSize));
    }
    plan = plan(cache, files, fileMap);
    Callable<Void> planRunner = new PlanRunner(plan);
    Future<Void>[] futures = new Future[concurrencyLevel];
    ExecutorService executor = Executors.newFixedThreadPool(concurrencyLevel);
    for (int i = 0; i < concurrencyLevel; i++) {
        futures[i] = executor.submit(planRunner);
    }
    if (preparation != null) {
        preparation.run(cache, this.fs, plan.getFilesTouched());
    }
    adversary.setProbabilityFactor(1.0);
    plan.start();
    long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout);
    long now;
    try {
        for (Future<Void> future : futures) {
            now = System.currentTimeMillis();
            if (deadlineMillis < now) {
                throw new TimeoutException();
            }
            future.get(deadlineMillis - now, TimeUnit.MILLISECONDS);
        }
        adversary.setProbabilityFactor(0.0);
        runVerificationPhase(cache);
    } finally {
        adversary.setProbabilityFactor(0.0);
        for (Future<Void> future : futures) {
            future.cancel(true);
        }
        executor.shutdown();
        now = System.currentTimeMillis();
        executor.awaitTermination(deadlineMillis - now, TimeUnit.MILLISECONDS);
        plan.close();
        cache.close();
        if (this.fs instanceof EphemeralFileSystemAbstraction) {
            this.fs.close();
            this.fs = new EphemeralFileSystemAbstraction();
        } else {
            for (File file : files) {
                file.delete();
            }
        }
    }
}

Example 72 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class PageCacheStressTest method run.

public void run() throws Exception {
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction()) {
        PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory();
        swapperFactory.setFileSystemAbstraction(fs);
        try (PageCache pageCacheUnderTest = new MuninnPageCache(swapperFactory, numberOfCachePages, cachePageSize, tracer, pageCursorTracerSupplier)) {
            PageCacheStresser pageCacheStresser = new PageCacheStresser(numberOfPages, numberOfThreads, workingDirectory);
            pageCacheStresser.stress(pageCacheUnderTest, condition);
        }
    }
}

Example 73 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class DumpCommandTest method shouldRespectTheStoreLock.

@Test
public void shouldRespectTheStoreLock() throws Exception {
    Path databaseDirectory = homeDir.resolve("data/databases/foo.db");
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        StoreLocker storeLocker = new StoreLocker(fileSystem)) {
        storeLocker.checkLock(databaseDirectory.toFile());
        execute("foo.db");
        fail("expected exception");
    } catch (CommandFailed e) {
        assertThat(e.getMessage(), equalTo("the database is in use -- stop Neo4j and try again"));
    }
}

Example 74 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class LoadCommandTest method shouldRespectTheStoreLock.

@Test
public void shouldRespectTheStoreLock() throws IOException, IncorrectUsage {
    Path databaseDirectory = homeDir.resolve("data/databases/foo.db");
    Files.createDirectories(databaseDirectory);
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        StoreLocker locker = new StoreLocker(fileSystem)) {
        locker.checkLock(databaseDirectory.toFile());
        execute("foo.db", "--force");
        fail("expected exception");
    } catch (CommandFailed e) {
        assertThat(e.getMessage(), equalTo("the database is in use -- stop Neo4j and try again"));
    }
}

Example 75 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class ImportTool method main.

/**
     * Runs the import tool given the supplied arguments.
     *
     * @param incomingArguments arguments for specifying input and configuration for the import.
     * @param defaultSettingsSuitableForTests default configuration geared towards unit/integration
     * test environments, for example lower default buffer sizes.
     */
public static void main(String[] incomingArguments, boolean defaultSettingsSuitableForTests) throws IOException {
    System.err.println("WARNING: neo4j-import is deprecated and support for it will be removed in a future\n" + "version of Neo4j; please use neo4j-admin import instead.\n");
    PrintStream out = System.out;
    PrintStream err = System.err;
    Args args = Args.parse(incomingArguments);
    if (ArrayUtil.isEmpty(incomingArguments) || asksForUsage(args)) {
        printUsage(out);
        return;
    }
    File storeDir;
    Collection<Option<File[]>> nodesFiles, relationshipsFiles;
    boolean enableStacktrace;
    Number processors = null;
    Input input = null;
    int badTolerance;
    Charset inputEncoding;
    boolean skipBadRelationships, skipDuplicateNodes, ignoreExtraColumns;
    Config dbConfig;
    OutputStream badOutput = null;
    IdType idType = null;
    int pageSize = UNSPECIFIED;
    Collector badCollector;
    org.neo4j.unsafe.impl.batchimport.Configuration configuration = null;
    File logsDir;
    File badFile;
    boolean success = false;
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction()) {
        storeDir = args.interpretOption(Options.STORE_DIR.key(), Converters.<File>mandatory(), Converters.toFile(), Validators.DIRECTORY_IS_WRITABLE, Validators.CONTAINS_NO_EXISTING_DATABASE);
        Config config = Config.defaults();
        config.augment(stringMap(GraphDatabaseSettings.neo4j_home.name(), storeDir.getAbsolutePath()));
        logsDir = config.get(GraphDatabaseSettings.logs_directory);
        fs.mkdirs(logsDir);
        badFile = new File(storeDir, BAD_FILE_NAME);
        badOutput = new BufferedOutputStream(fs.openAsOutputStream(badFile, false));
        nodesFiles = extractInputFiles(args, Options.NODE_DATA.key(), err);
        relationshipsFiles = extractInputFiles(args, Options.RELATIONSHIP_DATA.key(), err);
        validateInputFiles(nodesFiles, relationshipsFiles);
        enableStacktrace = args.getBoolean(Options.STACKTRACE.key(), Boolean.FALSE, Boolean.TRUE);
        processors = args.getNumber(Options.PROCESSORS.key(), null);
        idType = args.interpretOption(Options.ID_TYPE.key(), withDefault((IdType) Options.ID_TYPE.defaultValue()), TO_ID_TYPE);
        badTolerance = parseNumberOrUnlimited(args, Options.BAD_TOLERANCE);
        inputEncoding = Charset.forName(args.get(Options.INPUT_ENCODING.key(), defaultCharset().name()));
        skipBadRelationships = args.getBoolean(Options.SKIP_BAD_RELATIONSHIPS.key(), (Boolean) Options.SKIP_BAD_RELATIONSHIPS.defaultValue(), true);
        skipDuplicateNodes = args.getBoolean(Options.SKIP_DUPLICATE_NODES.key(), (Boolean) Options.SKIP_DUPLICATE_NODES.defaultValue(), true);
        ignoreExtraColumns = args.getBoolean(Options.IGNORE_EXTRA_COLUMNS.key(), (Boolean) Options.IGNORE_EXTRA_COLUMNS.defaultValue(), true);
        badCollector = badCollector(badOutput, badTolerance, collect(skipBadRelationships, skipDuplicateNodes, ignoreExtraColumns));
        dbConfig = loadDbConfig(args.interpretOption(Options.DATABASE_CONFIG.key(), Converters.<File>optional(), Converters.toFile(), Validators.REGEX_FILE_EXISTS));
        configuration = importConfiguration(processors, defaultSettingsSuitableForTests, dbConfig, pageSize);
        input = new CsvInput(nodeData(inputEncoding, nodesFiles), defaultFormatNodeFileHeader(), relationshipData(inputEncoding, relationshipsFiles), defaultFormatRelationshipFileHeader(), idType, csvConfiguration(args, defaultSettingsSuitableForTests), badCollector, configuration.maxNumberOfProcessors());
        doImport(out, err, storeDir, logsDir, badFile, fs, nodesFiles, relationshipsFiles, enableStacktrace, input, dbConfig, badOutput, configuration);
        success = true;
    } catch (IllegalArgumentException e) {
        throw andPrintError("Input error", e, false, err);
    } catch (IOException e) {
        throw andPrintError("File error", e, false, err);
    } finally {
        if (!success && badOutput != null) {
            badOutput.close();
        }
    }
}