Examples with RecordElement com.bonree.brfs.disknode.data.write.record.RecordElement used on opensource projects

Example 1 with RecordElement

use of com.bonree.brfs.disknode.data.write.record.RecordElement in project BRFS by zhangnianli.

the class FileWriterManager method validElements.

// 获取缺失或多余的日志记录信息
private List<RecordElement> validElements(String filepath, List<RecordElement> originElements) {
    byte[] bytes = DataFileReader.readFile(filepath, 0);
    List<String> offsets = FileDecoder.getDataFileOffsets(bytes);
    LOG.info("adjust get [{}] records from data file of [{}]", offsets.size(), filepath);
    List<RecordElement> validElmentList = new ArrayList<RecordElement>();
    RecordElement element = originElements.get(0);
    if (element.getOffset() != 0) {
        // 没有文件头的日志记录，不应该发生的
        throw new IllegalStateException("no header record in file[" + filepath + "]");
    }
    validElmentList.add(element);
    for (int index = 0; index < offsets.size(); index++) {
        List<String> parts = Splitter.on("|").splitToList(offsets.get(index));
        int offset = Integer.parseInt(parts.get(0));
        int size = Integer.parseInt(parts.get(1));
        long crc = ByteUtils.crc(bytes, offset, size);
        if (index + 1 >= originElements.size()) {
            // 数据文件还有数据，但日志文件没有记录
            validElmentList.add(new RecordElement(offset, size, crc));
            continue;
        }
        element = originElements.get(index + 1);
        if (element.getOffset() != offset) {
            LOG.warn("excepted offset[{}], but get offset[{}] for file[{}]", offset, element.getOffset(), filepath);
            break;
        }
        if (element.getSize() != size) {
            LOG.warn("excepted size[{}], but get size[{}] for file[{}]", size, element.getSize(), filepath);
            break;
        }
        if (element.getCrc() != crc) {
            LOG.warn("excepted crc[{}], but get crc[{}] for file[{}]", crc, element.getCrc(), filepath);
            break;
        }
        validElmentList.add(element);
    }
    return validElmentList;
}

Example 2 with RecordElement

use of com.bonree.brfs.disknode.data.write.record.RecordElement in project BRFS by zhangnianli.

the class FileLengthMessageHandler method handle.

@Override
public void handle(HttpMessage msg, HandleResultCallback callback) {
    LOG.info("GET sequences of file[{}]", msg.getPath());
    String filePath = context.getConcreteFilePath(msg.getPath());
    threadPool.submit(new Runnable() {

        @Override
        public void run() {
            List<RecordElement> recordInfo = getRecordElements(filePath);
            if (recordInfo == null) {
                LOG.info("can not get record elements of file[{}]", filePath);
                callback.completed(new HandleResult(false));
                return;
            }
            // 获取所有文件序列号
            RecordElement lastElement = null;
            if (recordInfo != null) {
                for (RecordElement element : recordInfo) {
                    if (lastElement == null) {
                        lastElement = element;
                        continue;
                    }
                    if (lastElement.getOffset() < element.getOffset()) {
                        lastElement = element;
                    }
                }
            }
            if (lastElement == null) {
                LOG.info("no available record element of file[{}]", filePath);
                callback.completed(new HandleResult(false));
                return;
            }
            HandleResult result = new HandleResult(true);
            long fileLength = fileFormater.relativeOffset(lastElement.getOffset()) + lastElement.getSize();
            LOG.info("get file length[{}] from file[{}]", fileLength, filePath);
            result.setData(Longs.toByteArray(fileLength));
            callback.completed(result);
        }
    });
}

Example 3 with RecordElement

use of com.bonree.brfs.disknode.data.write.record.RecordElement in project BRFS by zhangnianli.

the class RecordFileWriter method write.

@Override
public void write(byte[] bytes, int offset, int size) throws IOException {
    RecordElement element = new RecordElement(delegate.position(), size, ByteUtils.crc(bytes, offset, size));
    recorder.put(element);
    delegate.write(bytes, offset, size);
}

Example 4 with RecordElement

use of com.bonree.brfs.disknode.data.write.record.RecordElement in project BRFS by zhangnianli.

the class MetadataFetchMessageHandler method handleMessage.

@Override
public void handleMessage(BaseMessage baseMessage, ResponseWriter<BaseResponse> writer) {
    String path = BrStringUtils.fromUtf8Bytes(baseMessage.getBody());
    if (path == null) {
        writer.write(new BaseResponse(ResponseCode.ERROR_PROTOCOL));
        return;
    }
    String filePath = context.getConcreteFilePath(path);
    LOG.info("GET metadata of file[{}]", filePath);
    List<RecordElement> recordInfo = getRecordElements(filePath);
    if (recordInfo == null) {
        LOG.info("can not get record elements of file[{}]", filePath);
        writer.write(new BaseResponse(ResponseCode.ERROR));
        return;
    }
    // 获取所有文件序列号
    RecordElement lastElement = null;
    if (recordInfo != null) {
        for (RecordElement element : recordInfo) {
            if (lastElement == null) {
                lastElement = element;
                continue;
            }
            if (lastElement.getOffset() < element.getOffset()) {
                lastElement = element;
            }
        }
    }
    if (lastElement == null) {
        LOG.info("no available record element of file[{}]", filePath);
        writer.write(new BaseResponse(ResponseCode.ERROR));
        return;
    }
    long fileLength = fileFormater.relativeOffset(lastElement.getOffset()) + lastElement.getSize();
    LOG.info("get file length[{}] from file[{}]", fileLength, filePath);
    BaseResponse response = new BaseResponse(ResponseCode.OK);
    response.setBody(Longs.toByteArray(fileLength));
    writer.write(response);
}

Example 5 with RecordElement

use of com.bonree.brfs.disknode.data.write.record.RecordElement in project BRFS by zhangnianli.

the class MetadataFetchMessageHandler method getRecordElements.

private List<RecordElement> getRecordElements(String filePath) {
    List<RecordElement> recordInfo = null;
    Pair<RecordFileWriter, WriteWorker> binding = writerManager.getBinding(filePath, false);
    if (binding != null) {
        RecordElementReader recordReader = null;
        try {
            binding.first().flush();
            writerManager.adjustFileWriter(filePath);
            RecordCollection recordSet = binding.first().getRecordCollection();
            recordReader = recordSet.getRecordElementReader();
            recordInfo = new ArrayList<RecordElement>();
            for (RecordElement element : recordReader) {
                recordInfo.add(element);
            }
        } catch (Exception e) {
            LOG.error("getSequnceNumbers from file[{}] error", filePath, e);
        } finally {
            CloseUtils.closeQuietly(recordReader);
        }
    } else {
        // 到这有两种情况：
        // 1、文件打开操作未成功后进行同步；
        // 2、文件关闭成功后进行文件同步;
        recordInfo = new ArrayList<RecordElement>();
        File dataFile = new File(filePath);
        if (dataFile.exists()) {
            // 到这的唯一机会是，多副本文件关闭时只有部分关闭成功，当磁盘节点恢复正常
            // 后，需要再次进行同步流程让所有副本文件关闭，因为没有日志文件，所以只能
            // 通过解析数据文件生成序列号列表
            byte[] bytes = DataFileReader.readFile(dataFile);
            List<String> offsetInfos = FileDecoder.getDataFileOffsets(fileFormater.fileHeader().length(), bytes);
            for (String info : offsetInfos) {
                List<String> parts = Splitter.on('|').splitToList(info);
                int offset = Integer.parseInt(parts.get(0));
                int size = Integer.parseInt(parts.get(1));
                recordInfo.add(new RecordElement(offset, size, 0));
            }
        }
    }
    return recordInfo;
}