Examples with ChimeraNameSpaceProvider org.dcache.chimera.namespace.ChimeraNameSpaceProvider used on opensource projects

Example 1 with ChimeraNameSpaceProvider

use of org.dcache.chimera.namespace.ChimeraNameSpaceProvider in project dcache by dCache.

the class PerformanceTest method main.

public static void main(String[] arguments) throws Exception {
    /* Parse arguments.
         */
    Args args = new Args(arguments);
    if (args.argc() < 5) {
        System.err.print("Usage: nsp-performance.sh [-threads=<n>] ");
        for (Operation aOp : Operation.values()) {
            System.err.print("[-" + aOp.getUserInput() + "] ");
        }
        System.err.println(" <file>");
        System.err.println("  where <file> contains a list of paths to load and the remaining ");
        System.err.println("  parameters are the Chimera connection details.");
        System.err.println("Options:");
        for (Operation aOp : Operation.values()) {
            System.err.println(aOp);
        }
        System.err.println("\t-threads\tSets number of concurrent reads");
        System.err.println("\t-delay\tSets delay in seconds between progress updates");
        System.err.println("");
        System.err.println("Remaining arguments are passed on to the provider factory.");
        System.exit(2);
    }
    String jdbc = args.argv(0);
    String user = args.argv(1);
    String password = args.argv(2);
    String fileName = args.argv(3);
    args.shift(5);
    ops = getOps(args);
    int concurrency = (args.hasOption("threads")) ? Integer.parseInt(args.getOpt("threads")) : 1;
    int delay = (args.hasOption("delay")) ? Integer.parseInt(args.getOpt("delay")) : 10;
    /* Instantiate provider.
         */
    System.out.println("Starting chimera... ");
    HikariDataSource dataSource = FsFactory.getDataSource(jdbc, user, password);
    PlatformTransactionManager txManager = new DataSourceTransactionManager(dataSource);
    FileSystemProvider fileSystem = new JdbcFs(dataSource, txManager);
    provider = new ChimeraNameSpaceProvider();
    provider.setAclEnabled(false);
    provider.setExtractor(new ChimeraOsmStorageInfoExtractor(StorageInfo.DEFAULT_ACCESS_LATENCY, StorageInfo.DEFAULT_RETENTION_POLICY));
    provider.setFileSystem(fileSystem);
    provider.setInheritFileOwnership(false);
    provider.setPermissionHandler(new PosixPermissionHandler());
    provider.setUploadDirectory("/upload");
    provider.setVerifyAllLookups(true);
    tx = new TransactionTemplate(txManager);
    /* Read paths.
         */
    System.out.println("Loading " + fileName);
    queue = new LinkedBlockingQueue<>();
    List<String> paths = getPaths(fileName);
    queue.addAll(paths);
    /* Run test.
         */
    System.out.println("Running test...");
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
    ExecutorService executor = Executors.newCachedThreadPool();
    ScheduledFuture<?> progressTask = scheduler.scheduleAtFixedRate(new ProgressTask(), delay, delay, TimeUnit.SECONDS);
    Stopwatch watch = Stopwatch.createStarted();
    for (int i = 0; i < concurrency; i++) {
        executor.execute(new PerformanceTest());
    }
    executor.shutdown();
    executor.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
    watch.stop();
    progressTask.cancel(false);
    scheduler.shutdownNow();
    /* Report result.
         */
    StringWriter info = new StringWriter();
    provider.getInfo(new PrintWriter(info));
    System.out.println();
    System.out.println(info);
    System.out.println("Number of files  : " + paths.size());
    System.out.println("Number of threads: " + concurrency);
    System.out.println("Operations       : " + ops.stream().map(Operation::getUserInput).collect(joining(",")));
    System.out.println("Total time       : " + watch);
    System.out.println("Average pr. op   : " + ((double) watch.elapsed(TimeUnit.MICROSECONDS)) / paths.size() + " µs");
    System.out.println("Frequency        : " + 1000 * paths.size() / watch.elapsed(TimeUnit.MILLISECONDS) + " Hz");
    fileSystem.close();
    dataSource.close();
}

Example 2 with ChimeraNameSpaceProvider

use of org.dcache.chimera.namespace.ChimeraNameSpaceProvider in project dcache by dCache.

the class PnfsManagerTest method setUp.

@Before
public void setUp() throws Exception {
    /*
         * init Chimera DB
         */
    Properties dbProperties = new Properties();
    try (InputStream input = Resources.asByteSource(DB_TEST_PROPERTIES).openStream()) {
        dbProperties.load(input);
    }
    _conn = DriverManager.getConnection(dbProperties.getProperty("chimera.db.url"), dbProperties.getProperty("chimera.db.user"), dbProperties.getProperty("chimera.db.password"));
    _conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
    Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(_conn));
    Liquibase liquibase = new Liquibase("org/dcache/chimera/changelog/changelog-master.xml", new ClassLoaderResourceAccessor(), database);
    // Uncomment the following line when testing with mysql database
    /*
         * Liquibase liquibase = new Liquibase(changeLogFile, new
         * ClassLoaderResourceAccessor(), new JdbcConnection(conn));
         */
    liquibase.update("");
    _fs = FsFactory.createFileSystem(dbProperties.getProperty("chimera.db.url"), dbProperties.getProperty("chimera.db.user"), dbProperties.getProperty("chimera.db.password"));
    ChimeraNameSpaceProvider chimera = new ChimeraNameSpaceProvider();
    chimera.setExtractor(new ChimeraOsmStorageInfoExtractor(StorageInfo.DEFAULT_ACCESS_LATENCY, StorageInfo.DEFAULT_RETENTION_POLICY));
    chimera.setInheritFileOwnership(true);
    chimera.setVerifyAllLookups(true);
    chimera.setAllowMoveToDirectoryWithDifferentStorageClass(true);
    chimera.setPermissionHandler(new PosixPermissionHandler());
    chimera.setAclEnabled(false);
    chimera.setFileSystem(_fs);
    chimera.setUploadDirectory("/upload");
    chimera.setUploadSubDirectory("%d");
    _pnfsManager = new PnfsManagerV3();
    _pnfsManager.setThreads(1);
    _pnfsManager.setListThreads(1);
    _pnfsManager.setCacheModificationRelay(null);
    _pnfsManager.setLogSlowThreshold(0);
    _pnfsManager.setNameSpaceProvider(chimera);
    _pnfsManager.setQueueMaxSize(0);
    _pnfsManager.setFolding(true);
    _pnfsManager.setDirectoryListLimit(100);
    _pnfsManager.init();
    _fs.mkdir("/pnfs");
    FsInode baseInode = _fs.mkdir("/pnfs/testRoot");
    byte[] sGroupTagData = "chimera".getBytes();
    byte[] osmTemplateTagData = "StoreName sql".getBytes();
    _fs.createTag(baseInode, "sGroup");
    _fs.createTag(baseInode, "OSMTemplate");
    _fs.setTag(baseInode, "sGroup", sGroupTagData, 0, sGroupTagData.length);
    _fs.setTag(baseInode, "OSMTemplate", osmTemplateTagData, 0, osmTemplateTagData.length);
}

Example 3 with ChimeraNameSpaceProvider

use of org.dcache.chimera.namespace.ChimeraNameSpaceProvider in project dcache by dCache.

the class PnfsManagerTest method testMoveEntry.

@Test
public void testMoveEntry() throws Exception {
    PnfsCreateEntryMessage message = new PnfsCreateEntryMessage("/pnfs/testRoot/testMoveEntry", FileAttributes.ofFileType(DIR));
    _pnfsManager.createEntry(message);
    assertTrue("failed to create a directory", message.getReturnCode() == 0);
    FsInode srcInode = _fs.mkdir("/pnfs/testRoot/testMoveEntry/sourceDirectory");
    byte[] srcTagData = "foo".getBytes();
    _fs.setTag(srcInode, "sGroup", srcTagData, 0, srcTagData.length);
    FsInode dstInode = _fs.mkdir("/pnfs/testRoot/testMoveEntry/destinationDirectory");
    byte[] dstTagData = "bar".getBytes();
    _fs.setTag(dstInode, "sGroup", dstTagData, 0, dstTagData.length);
    message = new PnfsCreateEntryMessage("/pnfs/testRoot/testMoveEntry/sourceDirectory/sourceFile", FileAttributes.ofFileType(REGULAR));
    _pnfsManager.createEntry(message);
    assertTrue("failed to create an entry", message.getReturnCode() == 0);
    PnfsRenameMessage pnfsRenameMessage = new PnfsRenameMessage("/pnfs/testRoot/testMoveEntry/sourceDirectory/sourceFile", "/pnfs/testRoot/testMoveEntry/destinationDirectory/destinationFile", false);
    _pnfsManager.rename(pnfsRenameMessage);
    assertTrue("failed to move file to directory", pnfsRenameMessage.getReturnCode() == 0);
    ChimeraNameSpaceProvider provider = (ChimeraNameSpaceProvider) _pnfsManager.getNameSpaceProvider();
    provider.setAllowMoveToDirectoryWithDifferentStorageClass(false);
    pnfsRenameMessage = new PnfsRenameMessage("/pnfs/testRoot/testMoveEntry/destinationDirectory/destinationFile", "/pnfs/testRoot/testMoveEntry/sourceDirectory/sourceFile", false);
    _pnfsManager.rename(pnfsRenameMessage);
    assertTrue("succeeded to move file to directory with different tag, a failure", pnfsRenameMessage.getReturnCode() != 0);
    pnfsRenameMessage = new PnfsRenameMessage("/pnfs/testRoot/testMoveEntry/destinationDirectory", "/pnfs/testRoot/testMoveEntry/sourceDirectory/destinationDirectory", false);
    _pnfsManager.rename(pnfsRenameMessage);
    assertTrue("succeeded to move directory to directory with different tag, a failure", pnfsRenameMessage.getReturnCode() != 0);
}