Examples with DumpAllTask com.alibaba.nacos.config.server.service.dump.task.DumpAllTask used on opensource projects

Example 1 with DumpAllTask

use of com.alibaba.nacos.config.server.service.dump.task.DumpAllTask in project nacos by alibaba.

the class DumpService method dumpOperate.

protected void dumpOperate(DumpProcessor processor, DumpAllProcessor dumpAllProcessor, DumpAllBetaProcessor dumpAllBetaProcessor, DumpAllTagProcessor dumpAllTagProcessor) throws NacosException {
    String dumpFileContext = "CONFIG_DUMP_TO_FILE";
    TimerContext.start(dumpFileContext);
    try {
        LogUtil.DEFAULT_LOG.warn("DumpService start");
        Runnable dumpAll = () -> dumpAllTaskMgr.addTask(DumpAllTask.TASK_ID, new DumpAllTask());
        Runnable dumpAllBeta = () -> dumpAllTaskMgr.addTask(DumpAllBetaTask.TASK_ID, new DumpAllBetaTask());
        Runnable dumpAllTag = () -> dumpAllTaskMgr.addTask(DumpAllTagTask.TASK_ID, new DumpAllTagTask());
        Runnable clearConfigHistory = () -> {
            LOGGER.warn("clearConfigHistory start");
            if (canExecute()) {
                try {
                    Timestamp startTime = getBeforeStamp(TimeUtils.getCurrentTime(), 24 * getRetentionDays());
                    int pageSize = 1000;
                    LOGGER.warn("clearConfigHistory, getBeforeStamp:{}, pageSize:{}", startTime, pageSize);
                    persistService.removeConfigHistory(startTime, pageSize);
                } catch (Throwable e) {
                    LOGGER.error("clearConfigHistory error : {}", e.toString());
                }
            }
        };
        try {
            dumpConfigInfo(dumpAllProcessor);
            // update Beta cache
            LogUtil.DEFAULT_LOG.info("start clear all config-info-beta.");
            DiskUtil.clearAllBeta();
            if (persistService.isExistTable(BETA_TABLE_NAME)) {
                dumpAllBetaProcessor.process(new DumpAllBetaTask());
            }
            // update Tag cache
            LogUtil.DEFAULT_LOG.info("start clear all config-info-tag.");
            DiskUtil.clearAllTag();
            if (persistService.isExistTable(TAG_TABLE_NAME)) {
                dumpAllTagProcessor.process(new DumpAllTagTask());
            }
            // add to dump aggr
            List<ConfigInfoChanged> configList = persistService.findAllAggrGroup();
            if (configList != null && !configList.isEmpty()) {
                total = configList.size();
                List<List<ConfigInfoChanged>> splitList = splitList(configList, INIT_THREAD_COUNT);
                for (List<ConfigInfoChanged> list : splitList) {
                    MergeAllDataWorker work = new MergeAllDataWorker(list);
                    work.start();
                }
                LOGGER.info("server start, schedule merge end.");
            }
        } catch (Exception e) {
            LogUtil.FATAL_LOG.error("Nacos Server did not start because dumpservice bean construction failure :\n" + e);
            throw new NacosException(NacosException.SERVER_ERROR, "Nacos Server did not start because dumpservice bean construction failure :\n" + e.getMessage(), e);
        }
        if (!EnvUtil.getStandaloneMode()) {
            Runnable heartbeat = () -> {
                String heartBeatTime = TimeUtils.getCurrentTime().toString();
                // write disk
                try {
                    DiskUtil.saveHeartBeatToDisk(heartBeatTime);
                } catch (IOException e) {
                    LogUtil.FATAL_LOG.error("save heartbeat fail" + e.getMessage());
                }
            };
            ConfigExecutor.scheduleConfigTask(heartbeat, 0, 10, TimeUnit.SECONDS);
            long initialDelay = new Random().nextInt(INITIAL_DELAY_IN_MINUTE) + 10;
            LogUtil.DEFAULT_LOG.warn("initialDelay:{}", initialDelay);
            ConfigExecutor.scheduleConfigTask(dumpAll, initialDelay, DUMP_ALL_INTERVAL_IN_MINUTE, TimeUnit.MINUTES);
            ConfigExecutor.scheduleConfigTask(dumpAllBeta, initialDelay, DUMP_ALL_INTERVAL_IN_MINUTE, TimeUnit.MINUTES);
            ConfigExecutor.scheduleConfigTask(dumpAllTag, initialDelay, DUMP_ALL_INTERVAL_IN_MINUTE, TimeUnit.MINUTES);
        }
        ConfigExecutor.scheduleConfigTask(clearConfigHistory, 10, 10, TimeUnit.MINUTES);
    } finally {
        TimerContext.end(dumpFileContext, LogUtil.DUMP_LOG);
    }
}

Example 2 with DumpAllTask

use of com.alibaba.nacos.config.server.service.dump.task.DumpAllTask in project nacos by alibaba.

the class DumpService method dumpConfigInfo.

private void dumpConfigInfo(DumpAllProcessor dumpAllProcessor) throws IOException {
    int timeStep = 6;
    boolean isAllDump = true;
    // initial dump all
    FileInputStream fis = null;
    Timestamp heartheatLastStamp = null;
    try {
        if (isQuickStart()) {
            File heartbeatFile = DiskUtil.heartBeatFile();
            if (heartbeatFile.exists()) {
                fis = new FileInputStream(heartbeatFile);
                String heartheatTempLast = IoUtils.toString(fis, Constants.ENCODE);
                heartheatLastStamp = Timestamp.valueOf(heartheatTempLast);
                if (TimeUtils.getCurrentTime().getTime() - heartheatLastStamp.getTime() < timeStep * 60 * 60 * 1000) {
                    isAllDump = false;
                }
            }
        }
        if (isAllDump) {
            LogUtil.DEFAULT_LOG.info("start clear all config-info.");
            DiskUtil.clearAll();
            dumpAllProcessor.process(new DumpAllTask());
        } else {
            Timestamp beforeTimeStamp = getBeforeStamp(heartheatLastStamp, timeStep);
            DumpChangeProcessor dumpChangeProcessor = new DumpChangeProcessor(this, beforeTimeStamp, TimeUtils.getCurrentTime());
            dumpChangeProcessor.process(new DumpChangeTask());
            Runnable checkMd5Task = () -> {
                LogUtil.DEFAULT_LOG.error("start checkMd5Task");
                List<String> diffList = ConfigCacheService.checkMd5();
                for (String groupKey : diffList) {
                    String[] dg = GroupKey.parseKey(groupKey);
                    String dataId = dg[0];
                    String group = dg[1];
                    String tenant = dg[2];
                    ConfigInfoWrapper configInfo = persistService.queryConfigInfo(dataId, group, tenant);
                    ConfigCacheService.dumpChange(dataId, group, tenant, configInfo.getContent(), configInfo.getLastModified(), configInfo.getEncryptedDataKey());
                }
                LogUtil.DEFAULT_LOG.error("end checkMd5Task");
            };
            ConfigExecutor.scheduleConfigTask(checkMd5Task, 0, 12, TimeUnit.HOURS);
        }
    } catch (IOException e) {
        LogUtil.FATAL_LOG.error("dump config fail" + e.getMessage());
        throw e;
    } finally {
        if (null != fis) {
            try {
                fis.close();
            } catch (IOException e) {
                LogUtil.DEFAULT_LOG.warn("close file failed");
            }
        }
    }
}