Examples with FileMapping com.baidu.hugegraph.entity.load.FileMapping used on opensource projects

Example 6 with FileMapping

use of com.baidu.hugegraph.entity.load.FileMapping in project incubator-hugegraph-toolchain by apache.

the class FileMappingService method deleteUnfinishedFile.

@Async
@Scheduled(fixedRate = 10 * 60 * 1000)
public void deleteUnfinishedFile() {
    QueryWrapper<FileMapping> query = Wrappers.query();
    query.in("file_status", FileMappingStatus.UPLOADING.getValue());
    List<FileMapping> mappings = this.mapper.selectList(query);
    long threshold = this.config.get(HubbleOptions.UPLOAD_FILE_MAX_TIME_CONSUMING) * 1000;
    Date now = HubbleUtil.nowDate();
    for (FileMapping mapping : mappings) {
        Date updateTime = mapping.getUpdateTime();
        long duration = now.getTime() - updateTime.getTime();
        if (duration > threshold) {
            String filePath = mapping.getPath();
            try {
                FileUtils.forceDelete(new File(filePath));
            } catch (IOException e) {
                log.warn("Failed to delete expired uploading file {}", filePath, e);
            }
            this.remove(mapping.getId());
            // Delete corresponding uploading tokens
            Iterator<Map.Entry<String, ReadWriteLock>> iter;
            iter = this.uploadingTokenLocks.entrySet().iterator();
            iter.forEachRemaining(entry -> {
                String token = entry.getKey();
                if (token.startsWith(mapping.getName())) {
                    iter.remove();
                }
            });
        }
    }
}

Example 7 with FileMapping

use of com.baidu.hugegraph.entity.load.FileMapping in project incubator-hugegraph-toolchain by apache.

the class FileUploadController method delete.

@DeleteMapping
public Boolean delete(@PathVariable("connId") int connId, @PathVariable("jobId") int jobId, @RequestParam("name") String fileName, @RequestParam("token") String token) {
    JobManager jobEntity = this.jobService.get(jobId);
    Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
    Ex.check(jobEntity.getJobStatus() == JobStatus.UPLOADING || jobEntity.getJobStatus() == JobStatus.MAPPING || jobEntity.getJobStatus() == JobStatus.SETTING, "deleted.file.no-permission");
    FileMapping mapping = this.service.get(connId, jobId, fileName);
    Ex.check(mapping != null, "load.file-mapping.not-exist.name", fileName);
    ReadWriteLock lock = this.uploadingTokenLocks().get(token);
    if (lock != null) {
        lock.writeLock().lock();
    }
    try {
        this.service.deleteDiskFile(mapping);
        log.info("Prepare to remove file mapping {}", mapping.getId());
        this.service.remove(mapping.getId());
        long jobSize = jobEntity.getJobSize() - mapping.getTotalSize();
        jobEntity.setJobSize(jobSize);
        this.jobService.update(jobEntity);
        if (lock != null) {
            this.uploadingTokenLocks().remove(token);
        }
        return true;
    } finally {
        if (lock != null) {
            lock.writeLock().unlock();
        }
    }
}

Example 8 with FileMapping

use of com.baidu.hugegraph.entity.load.FileMapping in project incubator-hugegraph-toolchain by apache.

the class FileUploadController method checkFileValid.

private void checkFileValid(int connId, int jobId, JobManager jobEntity, MultipartFile file, String fileName) {
    Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
    Ex.check(jobEntity.getJobStatus() == JobStatus.DEFAULT || jobEntity.getJobStatus() == JobStatus.UPLOADING || jobEntity.getJobStatus() == JobStatus.MAPPING || jobEntity.getJobStatus() == JobStatus.SETTING, "load.upload.file.no-permission");
    // Now allowed to upload empty file
    Ex.check(!file.isEmpty(), "load.upload.file.cannot-be-empty");
    // Difficult: how to determine whether the file is csv or text
    log.debug("File content type: {}", file.getContentType());
    String format = FilenameUtils.getExtension(fileName);
    List<String> formatWhiteList = this.config.get(HubbleOptions.UPLOAD_FILE_FORMAT_LIST);
    Ex.check(formatWhiteList.contains(format), "load.upload.file.format.unsupported");
    long fileSize = file.getSize();
    long singleFileSizeLimit = this.config.get(HubbleOptions.UPLOAD_SINGLE_FILE_SIZE_LIMIT);
    Ex.check(fileSize <= singleFileSizeLimit, "load.upload.file.exceed-single-size", FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
    // Check is there a file with the same name
    FileMapping oldMapping = this.service.get(connId, jobId, fileName);
    Ex.check(oldMapping == null || oldMapping.getFileStatus() == FileMappingStatus.UPLOADING, "load.upload.file.existed", fileName);
    long totalFileSizeLimit = this.config.get(HubbleOptions.UPLOAD_TOTAL_FILE_SIZE_LIMIT);
    List<FileMapping> fileMappings = this.service.listAll();
    long currentTotalSize = fileMappings.stream().map(FileMapping::getTotalSize).reduce(0L, (Long::sum));
    Ex.check(fileSize + currentTotalSize <= totalFileSizeLimit, "load.upload.file.exceed-single-size", FileUtils.byteCountToDisplaySize(totalFileSizeLimit));
}

Example 9 with FileMapping

use of com.baidu.hugegraph.entity.load.FileMapping in project incubator-hugegraph-toolchain by apache.

the class LoadTaskController method reason.

@GetMapping("{id}/reason")
public Response reason(@PathVariable("connId") int connId, @PathVariable("jobId") int jobId, @PathVariable("id") int id) {
    LoadTask task = this.service.get(id);
    if (task == null) {
        throw new ExternalException("load.task.not-exist.id", id);
    }
    JobManager jobEntity = this.jobService.get(jobId);
    Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
    Integer fileId = task.getFileId();
    FileMapping mapping = this.fmService.get(fileId);
    String reason = this.service.readLoadFailedReason(mapping);
    return Response.builder().status(Constant.STATUS_OK).data(reason).build();
}

Example 10 with FileMapping

use of com.baidu.hugegraph.entity.load.FileMapping in project incubator-hugegraph-toolchain by apache.

the class LoadTaskController method start.

@PostMapping("start")
public List<LoadTask> start(@PathVariable("connId") int connId, @PathVariable("jobId") int jobId, @RequestParam("file_mapping_ids") List<Integer> fileIds) {
    GraphConnection connection = this.connService.get(connId);
    if (connection == null) {
        throw new ExternalException("graph-connection.not-exist.id", connId);
    }
    JobManager jobEntity = this.jobService.get(jobId);
    Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId);
    Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, "load.task.start.no-permission");
    boolean existError = false;
    try {
        List<LoadTask> tasks = new ArrayList<>();
        for (Integer fileId : fileIds) {
            FileMapping fileMapping = this.fmService.get(fileId);
            if (fileMapping == null) {
                throw new ExternalException("file-mapping.not-exist.id", fileId);
            }
            tasks.add(this.service.start(connection, fileMapping));
        }
        return tasks;
    } catch (Exception e) {
        existError = true;
        throw e;
    } finally {
        if (existError) {
            jobEntity.setJobStatus(JobStatus.FAILED);
        } else {
            jobEntity.setJobStatus(JobStatus.LOADING);
        }
        jobEntity.setUpdateTime(HubbleUtil.nowDate());
        this.jobService.update(jobEntity);
    }
}