Examples with FileObjectSyncState com.bonree.brfs.common.filesync.FileObjectSyncState used on opensource projects

Example 1 with FileObjectSyncState

use of com.bonree.brfs.common.filesync.FileObjectSyncState in project BRFS by zhangnianli.

the class DefaultFileObjectSyncProcessor method process.

@Override
public boolean process(FileObjectSyncTask task) {
    FileObject file = task.file();
    LOG.info("start to synchronize file[{}]", file.node().getName());
    DuplicateNode[] nodeList = file.node().getDuplicateNodes();
    boolean syncAccomplished = true;
    List<FileObjectSyncState> fileStateList = getFileStateList(file.node());
    if (fileStateList.isEmpty()) {
        // 文件所在的所有磁盘节点都处于异常状态
        LOG.error("No available duplicate node is found to sync file[{}]", file.node().getName());
        if (task.isExpired()) {
            task.callback().timeout(file);
            return true;
        }
        return false;
    }
    if (fileStateList.size() != nodeList.length) {
        // 文件所在的所有磁盘节点中有部分不可用，这种情况先同步可用的磁盘节点信息
        LOG.warn("Not all duplicate nodes are available to sync file[{}]", file.node().getName());
        syncAccomplished = false;
    }
    long maxLength = -1;
    for (FileObjectSyncState state : fileStateList) {
        maxLength = Math.max(maxLength, state.getFileLength());
    }
    List<FileObjectSyncState> lack = new ArrayList<FileObjectSyncState>();
    List<FileObjectSyncState> full = new ArrayList<FileObjectSyncState>();
    for (FileObjectSyncState state : fileStateList) {
        if (state.getFileLength() != maxLength) {
            lack.add(state);
        } else {
            full.add(state);
        }
    }
    if (lack.isEmpty()) {
        if (syncAccomplished) {
            LOG.info("file[{}] is ok!", file.node().getName());
            task.callback().complete(file, maxLength);
            return true;
        } else {
            LOG.info("file[{}] is lack of some duplicate node!", file.node().getName());
            if (task.isExpired()) {
                LOG.info("file[{}] sync is expired!", file.node().getName());
                task.callback().timeout(file);
                return true;
            }
            return false;
        }
    } else {
        syncAccomplished &= doSynchronize(file.node(), maxLength, lack, full);
        if (syncAccomplished) {
            LOG.info("file[{}] sync is completed!", file.node().getName());
            task.callback().complete(file, maxLength);
            return true;
        } else {
            LOG.info("file[{}] sync is failed!", file.node().getName());
            if (task.isExpired()) {
                LOG.info("file[{}] sync is expired!", file.node().getName());
                task.callback().timeout(file);
                return true;
            }
            return false;
        }
    }
}

Example 2 with FileObjectSyncState

use of com.bonree.brfs.common.filesync.FileObjectSyncState in project BRFS by zhangnianli.

the class FileRecoveryMessageHandler method handleMessage.

@Override
public void handleMessage(BaseMessage baseMessage, ResponseWriter<BaseResponse> writer) {
    FileRecoveryMessage message = ProtoStuffUtils.deserialize(baseMessage.getBody(), FileRecoveryMessage.class);
    if (message == null) {
        LOG.error("decode recover message error");
        writer.write(new BaseResponse(ResponseCode.ERROR_PROTOCOL));
        return;
    }
    String filePath = null;
    try {
        filePath = context.getConcreteFilePath(message.getFilePath());
        LOG.info("starting recover file[{}]", filePath);
        Pair<RecordFileWriter, WriteWorker> binding = writerManager.getBinding(filePath, false);
        if (binding == null) {
            writer.write(new BaseResponse(ResponseCode.ERROR));
            return;
        }
        binding.first().position(fileFormater.absoluteOffset(message.getOffset()));
        byte[] bytes = null;
        for (String stateString : message.getSources()) {
            FileObjectSyncState state = SyncStateCodec.fromString(stateString);
            Service service = serviceManager.getServiceById(state.getServiceGroup(), state.getServiceId());
            if (service == null) {
                LOG.error("can not get service with[{}:{}]", state.getServiceGroup(), state.getServiceId());
                continue;
            }
            DiskNodeClient client = null;
            try {
                LOG.info("get data from{} to recover...", service);
                TcpClient<ReadObject, FileContentPart> readClient = clientGroup.createClient(new AsyncFileReaderCreateConfig() {

                    @Override
                    public SocketAddress remoteAddress() {
                        return new InetSocketAddress(service.getHost(), service.getExtraPort());
                    }

                    @Override
                    public int connectTimeoutMillis() {
                        return 3000;
                    }

                    @Override
                    public int maxPendingRead() {
                        return 0;
                    }
                }, ForkJoinPool.commonPool());
                client = new TcpDiskNodeClient(null, readClient);
                long lackBytes = state.getFileLength() - message.getOffset();
                CompletableFuture<byte[]> byteFuture = new CompletableFuture<byte[]>();
                ByteArrayOutputStream output = new ByteArrayOutputStream();
                client.readData(state.getFilePath(), message.getOffset(), (int) lackBytes, new ByteConsumer() {

                    @Override
                    public void error(Throwable e) {
                        byteFuture.completeExceptionally(e);
                    }

                    @Override
                    public void consume(byte[] bytes, boolean endOfConsume) {
                        try {
                            output.write(bytes);
                            if (endOfConsume) {
                                byteFuture.complete(output.toByteArray());
                                output.close();
                            }
                        } catch (Exception e) {
                            byteFuture.completeExceptionally(e);
                        }
                    }
                });
                bytes = byteFuture.get();
                if (bytes != null) {
                    LOG.info("read bytes length[{}], require[{}]", bytes.length, lackBytes);
                    break;
                }
            } catch (Exception e) {
                LOG.error("recover file[{}] error", filePath, e);
            } finally {
                CloseUtils.closeQuietly(client);
            }
        }
        if (bytes == null) {
            writer.write(new BaseResponse(ResponseCode.ERROR));
            return;
        }
        int offset = 0;
        int size = 0;
        while ((size = FileDecoder.getOffsets(offset, bytes)) > 0) {
            LOG.info("rewrite data[offset={}, size={}] to file[{}]", offset, size, filePath);
            binding.first().write(bytes, offset, size);
            offset += size;
            size = 0;
        }
        if (offset != bytes.length) {
            LOG.error("perhaps datas that being recoverd is not correct! get [{}], but recoverd[{}]", bytes.length, offset);
        }
        writer.write(new BaseResponse(ResponseCode.OK));
    } catch (Exception e) {
        LOG.error("recover file[{}] error", filePath, e);
        writer.write(new BaseResponse(ResponseCode.ERROR));
    }
}

Example 3 with FileObjectSyncState

use of com.bonree.brfs.common.filesync.FileObjectSyncState in project BRFS by zhangnianli.

the class RecoveryMessageHandler method handle.

@Override
public void handle(HttpMessage msg, HandleResultCallback callback) {
    HandleResult handleResult = new HandleResult();
    String filePath = null;
    try {
        filePath = context.getConcreteFilePath(msg.getPath());
        LOG.info("starting recover file[{}]", filePath);
        String lengthParam = msg.getParams().get("length");
        if (lengthParam == null) {
            handleResult.setSuccess(false);
            callback.completed(handleResult);
            return;
        }
        long fileLength = Long.parseLong(msg.getParams().get("length"));
        List<String> fullStates = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(msg.getParams().get("fulls"));
        Pair<RecordFileWriter, WriteWorker> binding = writerManager.getBinding(filePath, false);
        if (binding == null) {
            handleResult.setSuccess(false);
            callback.completed(handleResult);
            return;
        }
        binding.first().position(fileFormater.absoluteOffset(fileLength));
        byte[] bytes = null;
        for (String stateString : fullStates) {
            FileObjectSyncState state = SyncStateCodec.fromString(stateString);
            Service service = serviceManager.getServiceById(state.getServiceGroup(), state.getServiceId());
            if (service == null) {
                LOG.error("can not get service with[{}:{}]", state.getServiceGroup(), state.getServiceId());
                continue;
            }
            DiskNodeClient client = null;
            try {
                LOG.info("get data from{} to recover...", service);
                client = new HttpDiskNodeClient(service.getHost(), service.getPort());
                long lackBytes = state.getFileLength() - fileLength;
                CompletableFuture<byte[]> byteFuture = new CompletableFuture<byte[]>();
                ByteArrayOutputStream output = new ByteArrayOutputStream();
                client.readData(state.getFilePath(), fileLength, (int) lackBytes, new ByteConsumer() {

                    @Override
                    public void error(Throwable e) {
                        byteFuture.completeExceptionally(e);
                    }

                    @Override
                    public void consume(byte[] bytes, boolean endOfConsume) {
                        try {
                            output.write(bytes);
                            if (endOfConsume) {
                                byteFuture.complete(output.toByteArray());
                                output.close();
                            }
                        } catch (Exception e) {
                            byteFuture.completeExceptionally(e);
                        }
                    }
                });
                bytes = byteFuture.get();
                if (bytes != null) {
                    LOG.info("read bytes length[{}], require[{}]", bytes.length, lackBytes);
                    break;
                }
            } catch (Exception e) {
                LOG.error("recover file[{}] error", filePath, e);
            } finally {
                CloseUtils.closeQuietly(client);
            }
        }
        if (bytes == null) {
            handleResult.setSuccess(false);
            callback.completed(handleResult);
            return;
        }
        int offset = 0;
        int size = 0;
        while ((size = FileDecoder.getOffsets(offset, bytes)) > 0) {
            LOG.info("rewrite data[offset={}, size={}] to file[{}]", offset, size, filePath);
            binding.first().write(bytes, offset, size);
            offset += size;
            size = 0;
        }
        if (offset != bytes.length) {
            LOG.error("perhaps datas that being recoverd is not correct! get [{}], but recoverd[{}]", bytes.length, offset);
        }
        handleResult.setSuccess(true);
    } catch (Exception e) {
        LOG.error("recover file[{}] error", filePath, e);
        handleResult.setSuccess(false);
    } finally {
        callback.completed(handleResult);
    }
}

Example 4 with FileObjectSyncState

use of com.bonree.brfs.common.filesync.FileObjectSyncState in project BRFS by zhangnianli.

the class DefaultFileObjectSyncProcessor method getFileStateList.

private List<FileObjectSyncState> getFileStateList(FileNode fileNode) {
    List<FileObjectSyncState> fileStateList = new ArrayList<FileObjectSyncState>();
    for (DuplicateNode node : fileNode.getDuplicateNodes()) {
        DiskNodeConnection connection = connectionPool.getConnection(node.getGroup(), node.getId());
        if (connection == null || connection.getClient() == null) {
            LOG.error("duplication node[{}, {}] of [{}] is not available, that's maybe a trouble!", node.getGroup(), node.getId(), fileNode.getName());
            continue;
        }
        String filePath = pathMaker.buildPath(fileNode, node);
        LOG.info("checking---{}", filePath);
        long fileLength = connection.getClient().getFileLength(filePath);
        if (fileLength < 0) {
            LOG.error("duplication node[{}, {}] of [{}] can not get file length, that's maybe a trouble!", node.getGroup(), node.getId(), fileNode.getName());
            continue;
        }
        LOG.info("server{} -- {}", node.getId(), fileLength);
        fileStateList.add(new FileObjectSyncState(node.getGroup(), node.getId(), filePath, fileLength));
    }
    return fileStateList;
}

Example 5 with FileObjectSyncState

use of com.bonree.brfs.common.filesync.FileObjectSyncState in project BRFS by zhangnianli.

the class DefaultFileObjectSyncProcessor method doSynchronize.

private boolean doSynchronize(FileNode fileNode, long correctLength, List<FileObjectSyncState> lacks, List<FileObjectSyncState> fulls) {
    List<String> fullStates = new ArrayList<String>();
    for (FileObjectSyncState state : fulls) {
        fullStates.add(SyncStateCodec.toString(state));
    }
    boolean allSynced = true;
    for (FileObjectSyncState state : lacks) {
        DiskNodeConnection connection = connectionPool.getConnection(state.getServiceGroup(), state.getServiceId());
        if (connection == null) {
            LOG.error("can not recover file[{}], because of lack of connection to service[{}, {}]", fileNode.getName(), state.getServiceGroup(), state.getServiceId());
            allSynced = false;
            continue;
        }
        DiskNodeClient client = connection.getClient();
        if (client == null) {
            allSynced = false;
            continue;
        }
        LOG.info("start synchronize file[{}] at data node[{}, {}]", fileNode.getName(), state.getServiceGroup(), state.getServiceId());
        if (!client.recover(state.getFilePath(), state.getFileLength(), fullStates)) {
            LOG.error("can not synchronize file[{}] at data node[{}, {}]", fileNode.getName(), state.getServiceGroup(), state.getServiceId());
            allSynced = false;
        }
    }
    return allSynced;
}