Examples with FileName org.apache.crail.metadata.FileName used on opensource projects

Example 11 with FileName

use of org.apache.crail.metadata.FileName in project incubator-crail by apache.

the class CoreDataStore method getBlockLocations.

public CrailBlockLocation[] getBlockLocations(String path, long start, long len) throws Exception {
    if (CrailConstants.DEBUG) {
        LOG.info("location: path " + path + ", start " + start + ", len " + len);
    }
    if (path == null) {
        LOG.info("Path null");
        return null;
    }
    if (start < 0 || len < 0) {
        LOG.info("Start or len invalid");
        throw new IOException("Invalid start or len parameter");
    }
    FileName name = new FileName(path);
    long rangeStart = CrailUtils.blockStartAddress(start);
    long range = start + len - CrailUtils.blockStartAddress(start);
    long blockCount = range / CrailConstants.BLOCK_SIZE;
    if (range % CrailConstants.BLOCK_SIZE > 0) {
        blockCount++;
    }
    CoreBlockLocation[] blockLocations = new CoreBlockLocation[(int) blockCount];
    HashMap<Long, DataNodeInfo> dataNodeSet = new HashMap<Long, DataNodeInfo>();
    HashMap<Long, DataNodeInfo> offset2DataNode = new HashMap<Long, DataNodeInfo>();
    for (long current = CrailUtils.blockStartAddress(start); current < start + len; current += CrailConstants.BLOCK_SIZE) {
        RpcGetLocation getLocationRes = rpcConnection.getLocation(name, current).get(CrailConstants.RPC_TIMEOUT, TimeUnit.MILLISECONDS);
        if (getLocationRes.getError() != RpcErrors.ERR_OK) {
            LOG.info("location: " + RpcErrors.messages[getLocationRes.getError()]);
            throw new IOException(RpcErrors.messages[getLocationRes.getError()]);
        }
        DataNodeInfo dataNodeInfo = getLocationRes.getBlockInfo().getDnInfo();
        dataNodeSet.put(dataNodeInfo.key(), dataNodeInfo);
        CoreBlockLocation location = new CoreBlockLocation();
        location.setOffset(current);
        location.setLength(Math.min(start + len - current, CrailConstants.BLOCK_SIZE));
        long index = (current - rangeStart) / CrailConstants.BLOCK_SIZE;
        blockLocations[(int) index] = location;
        offset2DataNode.put(current, dataNodeInfo);
    }
    // asign an identifier to each data node
    ArrayList<DataNodeInfo> dataNodeArray = new ArrayList<DataNodeInfo>(dataNodeSet.size());
    int index = 0;
    for (DataNodeInfo dn : dataNodeSet.values()) {
        dataNodeArray.add(index, dn);
        index++;
    }
    int locationSize = Math.min(CrailConstants.SHADOW_REPLICATION, dataNodeSet.size());
    int blockIndex = 0;
    for (int i = 0; i < blockLocations.length; i++) {
        CoreBlockLocation location = blockLocations[i];
        String[] hosts = new String[locationSize];
        String[] names = new String[locationSize];
        String[] topology = new String[locationSize];
        int[] storageType = new int[locationSize];
        int[] storageClass = new int[locationSize];
        int[] locationTiers = new int[locationSize];
        DataNodeInfo dnInfo = offset2DataNode.get(location.getOffset());
        DataNodeInfo mainDataNode = dataNodeSet.get(dnInfo.key());
        InetSocketAddress address = CrailUtils.datanodeInfo2SocketAddr(mainDataNode);
        names[0] = getMappedLocation(address.getAddress().getCanonicalHostName()) + ":" + address.getPort();
        hosts[0] = getMappedLocation(address.getAddress().getCanonicalHostName());
        topology[0] = "/default-rack/" + names[0];
        storageType[0] = mainDataNode.getStorageType();
        storageClass[0] = mainDataNode.getStorageClass();
        locationTiers[0] = mainDataNode.getLocationClass();
        for (int j = 1; j < locationSize; j++) {
            DataNodeInfo replicaDataNode = dataNodeArray.get(blockIndex);
            address = CrailUtils.datanodeInfo2SocketAddr(replicaDataNode);
            names[j] = getMappedLocation(address.getAddress().getCanonicalHostName()) + ":" + address.getPort();
            hosts[j] = getMappedLocation(address.getAddress().getCanonicalHostName());
            topology[j] = "/default-rack/" + names[j];
            storageType[j] = replicaDataNode.getStorageType();
            storageClass[j] = replicaDataNode.getStorageClass();
            locationTiers[j] = replicaDataNode.getLocationClass();
            blockIndex = (blockIndex + 1) % dataNodeArray.size();
        }
        location.setNames(names);
        location.setHosts(hosts);
        location.setTopologyPaths(topology);
        location.setStorageTypes(storageType);
        location.setStorageClasses(storageClass);
        location.setLocationAffinities(locationTiers);
    }
    return blockLocations;
}

Example 12 with FileName

use of org.apache.crail.metadata.FileName in project incubator-crail by apache.

the class CoreDataStore method _listEntries.

public DirectoryInputStream _listEntries(String name, boolean randomize) throws Exception {
    FileName directory = new FileName(name);
    if (CrailConstants.DEBUG) {
        LOG.info("getDirectoryList: " + name);
    }
    RpcGetFile fileRes = rpcConnection.getFile(directory, false).get(CrailConstants.RPC_TIMEOUT, TimeUnit.MILLISECONDS);
    if (fileRes.getError() != RpcErrors.ERR_OK) {
        LOG.info("getDirectoryList: " + RpcErrors.messages[fileRes.getError()]);
        throw new FileNotFoundException(RpcErrors.messages[fileRes.getError()]);
    }
    FileInfo dirInfo = fileRes.getFile();
    if (!dirInfo.getType().isContainer()) {
        LOG.info("getDirectoryList: " + RpcErrors.messages[RpcErrors.ERR_FILE_IS_NOT_DIR]);
        throw new FileNotFoundException(RpcErrors.messages[RpcErrors.ERR_FILE_IS_NOT_DIR]);
    }
    CoreDirectory dirFile = new CoreDirectory(this, dirInfo, name);
    DirectoryInputStream inputStream = dirFile.getDirectoryInputStream(randomize);
    return inputStream;
}