Examples with CrailConfiguration org.apache.crail.conf.CrailConfiguration used on opensource projects

Example 6 with CrailConfiguration

use of org.apache.crail.conf.CrailConfiguration in project incubator-crail by apache.

the class CrailHadoopFileSystem method initialize.

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);
    setConf(conf);
    try {
        CrailConfiguration crailConf = new CrailConfiguration();
        this.dfs = CrailStore.newInstance(crailConf);
        Path _workingDir = new Path("/user/" + CrailConstants.USER);
        this.workingDir = new Path("/user/" + CrailConstants.USER).makeQualified(uri, _workingDir);
        this.uri = URI.create(CrailConstants.NAMENODE_ADDRESS);
        LOG.info("CrailHadoopFileSystem fs initialization done..");
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 7 with CrailConfiguration

use of org.apache.crail.conf.CrailConfiguration in project YCSB by brianfrankcooper.

the class CrailClient method init.

@Override
public void init() throws DBException {
    super.init();
    try {
        CrailConfiguration crailConf = new CrailConfiguration();
        this.client = CrailStore.newInstance(crailConf);
        usertable = getProperties().getProperty("table", "usertable");
        enumerateKeys = Boolean.parseBoolean(getProperties().getProperty("crail.enumeratekeys", "false"));
        if (client.lookup(usertable).get() == null) {
            client.create(usertable, CrailNodeType.TABLE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir();
        }
        this.startTime = System.nanoTime();
    } catch (Exception e) {
        throw new DBException(e);
    }
}

Example 8 with CrailConfiguration

use of org.apache.crail.conf.CrailConfiguration in project incubator-crail by apache.

the class StorageServer method main.

public static void main(String[] args) throws Exception {
    Logger LOG = CrailUtils.getLogger();
    CrailConfiguration conf = new CrailConfiguration();
    CrailConstants.updateConstants(conf);
    CrailConstants.printConf();
    CrailConstants.verify();
    int splitIndex = 0;
    for (String param : args) {
        if (param.equalsIgnoreCase("--")) {
            break;
        }
        splitIndex++;
    }
    // default values
    StringTokenizer tokenizer = new StringTokenizer(CrailConstants.STORAGE_TYPES, ",");
    if (!tokenizer.hasMoreTokens()) {
        throw new Exception("No storage types defined!");
    }
    String storageName = tokenizer.nextToken();
    int storageType = 0;
    HashMap<String, Integer> storageTypes = new HashMap<String, Integer>();
    storageTypes.put(storageName, storageType);
    for (int type = 1; tokenizer.hasMoreElements(); type++) {
        String name = tokenizer.nextToken();
        storageTypes.put(name, type);
    }
    int storageClass = -1;
    // custom values
    if (args != null) {
        Option typeOption = Option.builder("t").desc("storage type to start").hasArg().build();
        Option classOption = Option.builder("c").desc("storage class the server will attach to").hasArg().build();
        Options options = new Options();
        options.addOption(typeOption);
        options.addOption(classOption);
        CommandLineParser parser = new DefaultParser();
        try {
            CommandLine line = parser.parse(options, Arrays.copyOfRange(args, 0, splitIndex));
            if (line.hasOption(typeOption.getOpt())) {
                storageName = line.getOptionValue(typeOption.getOpt());
                storageType = storageTypes.get(storageName).intValue();
            }
            if (line.hasOption(classOption.getOpt())) {
                storageClass = Integer.parseInt(line.getOptionValue(classOption.getOpt()));
            }
        } catch (ParseException e) {
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp("Storage tier", options);
            System.exit(-1);
        }
    }
    if (storageClass < 0) {
        storageClass = storageType;
    }
    StorageTier storageTier = StorageTier.createInstance(storageName);
    if (storageTier == null) {
        throw new Exception("Cannot instantiate datanode of type " + storageName);
    }
    StorageServer server = storageTier.launchServer();
    String[] extraParams = null;
    splitIndex++;
    if (args.length > splitIndex) {
        extraParams = new String[args.length - splitIndex];
        for (int i = splitIndex; i < args.length; i++) {
            extraParams[i - splitIndex] = args[i];
        }
    }
    server.init(conf, extraParams);
    server.printConf(LOG);
    Thread thread = new Thread(server);
    thread.start();
    RpcClient rpcClient = RpcClient.createInstance(CrailConstants.NAMENODE_RPC_TYPE);
    rpcClient.init(conf, args);
    rpcClient.printConf(LOG);
    ConcurrentLinkedQueue<InetSocketAddress> namenodeList = CrailUtils.getNameNodeList();
    ConcurrentLinkedQueue<RpcConnection> connectionList = new ConcurrentLinkedQueue<RpcConnection>();
    while (!namenodeList.isEmpty()) {
        InetSocketAddress address = namenodeList.poll();
        RpcConnection connection = rpcClient.connect(address);
        connectionList.add(connection);
    }
    RpcConnection rpcConnection = connectionList.peek();
    if (connectionList.size() > 1) {
        rpcConnection = new RpcDispatcher(connectionList);
    }
    LOG.info("connected to namenode(s) " + rpcConnection.toString());
    StorageRpcClient storageRpc = new StorageRpcClient(storageType, CrailStorageClass.get(storageClass), server.getAddress(), rpcConnection);
    HashMap<Long, Long> blockCount = new HashMap<Long, Long>();
    long sumCount = 0;
    long lba = 0;
    while (server.isAlive()) {
        StorageResource resource = server.allocateResource();
        if (resource == null) {
            break;
        } else {
            storageRpc.setBlock(lba, resource.getAddress(), resource.getLength(), resource.getKey());
            lba += (long) resource.getLength();
            DataNodeStatistics stats = storageRpc.getDataNode();
            long newCount = stats.getFreeBlockCount();
            long serviceId = stats.getServiceId();
            long oldCount = 0;
            if (blockCount.containsKey(serviceId)) {
                oldCount = blockCount.get(serviceId);
            }
            long diffCount = newCount - oldCount;
            blockCount.put(serviceId, newCount);
            sumCount += diffCount;
            LOG.info("datanode statistics, freeBlocks " + sumCount);
        }
    }
    while (server.isAlive()) {
        DataNodeStatistics stats = storageRpc.getDataNode();
        long newCount = stats.getFreeBlockCount();
        long serviceId = stats.getServiceId();
        long oldCount = 0;
        if (blockCount.containsKey(serviceId)) {
            oldCount = blockCount.get(serviceId);
        }
        long diffCount = newCount - oldCount;
        blockCount.put(serviceId, newCount);
        sumCount += diffCount;
        LOG.info("datanode statistics, freeBlocks " + sumCount);
        Thread.sleep(CrailConstants.STORAGE_KEEPALIVE * 1000);
    }
}

Example 9 with CrailConfiguration

use of org.apache.crail.conf.CrailConfiguration in project incubator-crail by apache.

the class CrailFsck method directoryDump.

public void directoryDump(String filename, boolean randomize) throws Exception {
    CrailConfiguration conf = new CrailConfiguration();
    CrailConstants.updateConstants(conf);
    CoreDataStore fs = new CoreDataStore(conf);
    DirectoryInputStream iter = fs._listEntries(filename, randomize);
    System.out.println("#hash   \t\tname\t\tfilecomponent");
    int i = 0;
    while (iter.hasRecord()) {
        DirectoryRecord record = iter.nextRecord();
        String path = CrailUtils.combinePath(record.getParent(), record.getFile());
        FileName hash = new FileName(path);
        System.out.format(i + ": " + "%08d\t\t%s\t%d\n", record.isValid() ? 1 : 0, padRight(record.getFile(), 8), hash.getFileComponent());
        i++;
    }
    iter.close();
    fs.closeFileSystem();
}

Example 10 with CrailConfiguration

use of org.apache.crail.conf.CrailConfiguration in project incubator-crail by apache.

the class ClientTest method init.

@Before
public void init() throws Exception {
    CrailConfiguration conf = new CrailConfiguration();
    fs = CrailStore.newInstance(conf);
    fs.create(basePath, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get();
}