Examples with FileFormater com.bonree.brfs.disknode.fileformat.FileFormater used on opensource projects

Example 1 with FileFormater

use of com.bonree.brfs.disknode.fileformat.FileFormater in project BRFS by zhangnianli.

the class EmptyMain method start.

@Override
public void start() throws Exception {
    LOG.info("Empty Main--port[{}]", httpConfig.getPort());
    createRootDirIfNeeded(diskContext.getRootDir());
    RecordCollectionManager recorderManager = new RecordCollectionManager();
    writerManager = new FileWriterManager(recorderManager);
    writerManager.start();
    writerManager.rebuildFileWriterbyDir(diskContext.getRootDir());
    server = new NettyHttpServer(httpConfig);
    requestHandlerExecutor = Executors.newFixedThreadPool(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_REQUEST_HANDLER_NUM), new PooledThreadFactory("request_handler"));
    FileFormater fileFormater = new SimpleFileFormater(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_FILE_MAX_CAPACITY));
    NettyHttpRequestHandler requestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    requestHandler.addMessageHandler("PUT", new OpenMessageHandler(diskContext, writerManager));
    requestHandler.addMessageHandler("POST", new WriteMessageHandler(diskContext, writerManager, fileFormater));
    requestHandler.addMessageHandler("GET", new ReadMessageHandler(diskContext, fileFormater));
    requestHandler.addMessageHandler("CLOSE", new CloseMessageHandler(diskContext, writerManager, fileFormater));
    requestHandler.addMessageHandler("DELETE", new DeleteMessageHandler(diskContext, writerManager));
    server.addContextHandler(DiskContext.URI_DISK_NODE_ROOT, requestHandler);
    NettyHttpRequestHandler flushRequestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    flushRequestHandler.addMessageHandler("POST", new FlushMessageHandler(diskContext, writerManager));
    server.addContextHandler(DiskContext.URI_FLUSH_NODE_ROOT, flushRequestHandler);
    NettyHttpRequestHandler sequenceRequestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    sequenceRequestHandler.addMessageHandler("GET", new FileLengthMessageHandler(diskContext, writerManager, fileFormater));
    server.addContextHandler(DiskContext.URI_LENGTH_NODE_ROOT, sequenceRequestHandler);
    NettyHttpRequestHandler listRequestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    listRequestHandler.addMessageHandler("GET", new ListMessageHandler(diskContext));
    server.addContextHandler(DiskContext.URI_LIST_NODE_ROOT, listRequestHandler);
    NettyHttpRequestHandler recoverRequestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    recoverRequestHandler.addMessageHandler("POST", new RecoveryMessageHandler(diskContext, serviceManager, writerManager, fileFormater));
    server.addContextHandler(DiskContext.URI_RECOVER_NODE_ROOT, recoverRequestHandler);
    NettyHttpRequestHandler pingRequestHandler = new NettyHttpRequestHandler(requestHandlerExecutor);
    pingRequestHandler.addMessageHandler("GET", new PingPongRequestHandler());
    server.addContextHandler(DiskContext.URI_PING_PONG_ROOT, pingRequestHandler);
    server.start();
}

Example 2 with FileFormater

use of com.bonree.brfs.disknode.fileformat.FileFormater in project BRFS by zhangnianli.

the class DataNodeBootStrap method start.

@Override
public void start() throws Exception {
    createRootDirIfNeeded(diskContext.getRootDir());
    RecordCollectionManager recorderManager = new RecordCollectionManager();
    writerManager = new FileWriterManager(recorderManager);
    writerManager.start();
    writerManager.rebuildFileWriterbyDir(diskContext.getRootDir());
    LOG.info("####################################come here###################################");
    FileFormater fileFormater = new SimpleFileFormater(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_FILE_MAX_CAPACITY));
    threadPool = Executors.newFixedThreadPool(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_REQUEST_HANDLER_NUM), new PooledThreadFactory("message_handler"));
    MessageChannelInitializer initializer = new MessageChannelInitializer(threadPool);
    initializer.addMessageHandler(TYPE_OPEN_FILE, new OpenFileMessageHandler(diskContext, writerManager));
    initializer.addMessageHandler(TYPE_WRITE_FILE, new WriteFileMessageHandler(diskContext, writerManager, fileFormater));
    initializer.addMessageHandler(TYPE_CLOSE_FILE, new CloseFileMessageHandler(diskContext, writerManager, fileFormater));
    initializer.addMessageHandler(TYPE_DELETE_FILE, new DeleteFileMessageHandler(diskContext, writerManager));
    initializer.addMessageHandler(TYPE_PING_PONG, new PingPongMessageHandler());
    initializer.addMessageHandler(TYPE_FLUSH_FILE, new FlushFileMessageHandler(diskContext, writerManager));
    initializer.addMessageHandler(TYPE_METADATA, new MetadataFetchMessageHandler(diskContext, writerManager, fileFormater));
    initializer.addMessageHandler(TYPE_LIST_FILE, new ListFileMessageHandler(diskContext));
    initializer.addMessageHandler(TYPE_RECOVER_FILE, new FileRecoveryMessageHandler(diskContext, serviceManager, writerManager, fileFormater, readerGroup));
    ServerConfig config = new ServerConfig();
    config.setBacklog(Integer.parseInt(System.getProperty(SystemProperties.PROP_NET_BACKLOG, "2048")));
    config.setBossThreadNums(1);
    config.setWorkerThreadNums(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_SERVER_IO_NUM));
    config.setPort(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_PORT));
    config.setHost(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_HOST));
    server = new TcpServer(config, initializer);
    server.start();
    ServerConfig fileServerConfig = new ServerConfig();
    fileServerConfig.setBacklog(Integer.parseInt(System.getProperty(SystemProperties.PROP_NET_BACKLOG, "2048")));
    fileServerConfig.setBossThreadNums(1);
    fileServerConfig.setWorkerThreadNums(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_FILE_READER_NUM));
    fileServerConfig.setPort(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_FILE_PORT));
    fileServerConfig.setHost(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_HOST));
    FileChannelInitializer fileInitializer = new FileChannelInitializer(new ReadObjectTranslator() {

        @Override
        public long offset(long offset) {
            return fileFormater.absoluteOffset(offset);
        }

        @Override
        public int length(int length) {
            return length;
        }

        @Override
        public String filePath(String path) {
            return diskContext.getConcreteFilePath(path);
        }
    });
    fileServer = new TcpServer(fileServerConfig, fileInitializer);
    fileServer.start();
}

Example 3 with FileFormater

use of com.bonree.brfs.disknode.fileformat.FileFormater in project BRFS by zhangnianli.

the class OpenFileMessageHandler method handleMessage.

@Override
public void handleMessage(BaseMessage baseMessage, ResponseWriter<BaseResponse> writer) {
    OpenFileMessage message = ProtoStuffUtils.deserialize(baseMessage.getBody(), OpenFileMessage.class);
    if (message == null) {
        writer.write(new BaseResponse(ResponseCode.ERROR_PROTOCOL));
        return;
    }
    FileFormater fileFormater = new SimpleFileFormater(Math.min(message.getCapacity(), MAX_CAPACITY));
    String realPath = diskContext.getConcreteFilePath(message.getFilePath());
    LOG.info("open file [{}]", realPath);
    Pair<RecordFileWriter, WriteWorker> binding = writerManager.getBinding(realPath, true);
    if (binding == null) {
        LOG.error("get file writer for file[{}] error!", realPath);
        writer.write(new BaseResponse(ResponseCode.ERROR));
        return;
    }
    try {
        binding.first().write(fileFormater.fileHeader().getBytes());
        binding.first().flush();
        BaseResponse response = new BaseResponse(ResponseCode.OK);
        response.setBody(Longs.toByteArray(fileFormater.maxBodyLength()));
        writer.write(response);
    } catch (Exception e) {
        LOG.error("write header to file[{}] error!", realPath);
        writer.write(new BaseResponse(ResponseCode.ERROR));
    }
}

Example 4 with FileFormater

use of com.bonree.brfs.disknode.fileformat.FileFormater in project BRFS by zhangnianli.

the class OpenMessageHandler method handle.

@Override
public void handle(HttpMessage msg, HandleResultCallback callback) {
    HandleResult result = new HandleResult();
    String realPath = null;
    try {
        String capacityParam = msg.getParams().get("capacity");
        if (capacityParam == null) {
            result.setSuccess(false);
            return;
        }
        long capacity = Long.parseLong(capacityParam);
        FileFormater fileFormater = new SimpleFileFormater(Math.min(capacity, MAX_CAPACITY));
        realPath = diskContext.getConcreteFilePath(msg.getPath());
        LOG.info("open file [{}]", realPath);
        Pair<RecordFileWriter, WriteWorker> binding = writerManager.getBinding(realPath, true);
        if (binding == null) {
            LOG.error("get file writer for file[{}] error!", realPath);
            result.setSuccess(false);
            return;
        }
        binding.first().write(fileFormater.fileHeader().getBytes());
        binding.first().flush();
        result.setData(Longs.toByteArray(fileFormater.maxBodyLength()));
        result.setSuccess(true);
    } catch (Exception e) {
        LOG.error("write header to file[{}] error!", realPath);
        result.setSuccess(false);
    } finally {
        callback.completed(result);
    }
}