Examples with ResourceInfo org.apache.dolphinscheduler.common.process.ResourceInfo used on opensource projects

Example 1 with ResourceInfo

use of org.apache.dolphinscheduler.common.process.ResourceInfo in project dolphinscheduler by apache.

the class FlinkParametersTest method getResourceFilesList.

@Test
public void getResourceFilesList() {
    FlinkParameters flinkParameters = new FlinkParameters();
    Assert.assertTrue(CollectionUtils.isEmpty(flinkParameters.getResourceFilesList()));
    ResourceInfo mainResource = new ResourceInfo();
    mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar");
    flinkParameters.setMainJar(mainResource);
    List<ResourceInfo> resourceInfos = new LinkedList<>();
    ResourceInfo resourceInfo1 = new ResourceInfo();
    resourceInfo1.setRes("testFlinkParameters1.jar");
    resourceInfos.add(resourceInfo1);
    flinkParameters.setResourceList(resourceInfos);
    List<ResourceInfo> resourceFilesList = flinkParameters.getResourceFilesList();
    Assert.assertNotNull(resourceFilesList);
    Assert.assertEquals(2, resourceFilesList.size());
    ResourceInfo resourceInfo2 = new ResourceInfo();
    resourceInfo2.setRes("testFlinkParameters2.jar");
    resourceInfos.add(resourceInfo2);
    flinkParameters.setResourceList(resourceInfos);
    resourceFilesList = flinkParameters.getResourceFilesList();
    Assert.assertNotNull(resourceFilesList);
    Assert.assertEquals(3, resourceFilesList.size());
}

Example 2 with ResourceInfo

use of org.apache.dolphinscheduler.common.process.ResourceInfo in project dolphinscheduler by apache.

the class TaskPriorityQueueConsumer method getResourceFullNames.

/**
 * get resource map key is full name and value is tenantCode
 */
protected Map<String, String> getResourceFullNames(TaskNode taskNode) {
    Map<String, String> resourceMap = new HashMap<>();
    AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams());
    if (baseParam != null) {
        List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
        if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
            // filter the resources that the resource id equals 0
            Set<ResourceInfo> oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet());
            if (CollectionUtils.isNotEmpty(oldVersionResources)) {
                oldVersionResources.forEach((t) -> resourceMap.put(t.getRes(), processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE)));
            }
            // get the resource id in order to get the resource names in batch
            Stream<Integer> resourceIdStream = projectResourceFiles.stream().map(resourceInfo -> resourceInfo.getId());
            Set<Integer> resourceIdsSet = resourceIdStream.filter(resId -> resId != 0).collect(Collectors.toSet());
            if (CollectionUtils.isNotEmpty(resourceIdsSet)) {
                Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]);
                List<Resource> resources = processService.listResourceByIds(resourceIds);
                resources.forEach((t) -> resourceMap.put(t.getFullName(), processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE)));
            }
        }
    }
    return resourceMap;
}

Example 3 with ResourceInfo

use of org.apache.dolphinscheduler.common.process.ResourceInfo in project dolphinscheduler by apache.

the class FlinkArgsUtilsTest method setUp.

@Before
public void setUp() throws Exception {
    ResourceInfo main = new ResourceInfo();
    main.setRes("testflink-1.0.0-SNAPSHOT.jar");
    mainJar = main;
}

Example 4 with ResourceInfo

use of org.apache.dolphinscheduler.common.process.ResourceInfo in project dolphinscheduler by apache.

the class UpgradeDao method updateProcessDefinitionJsonResourceList.

/**
 * updateProcessDefinitionJsonResourceList
 */
protected void updateProcessDefinitionJsonResourceList() {
    ResourceDao resourceDao = new ResourceDao();
    ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao();
    Map<Integer, String> replaceProcessDefinitionMap = new HashMap<>();
    try {
        Map<String, Integer> resourcesMap = resourceDao.listAllResources(dataSource.getConnection());
        Map<Integer, String> processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection());
        for (Map.Entry<Integer, String> entry : processDefinitionJsonMap.entrySet()) {
            JSONObject jsonObject = JSONObject.parseObject(entry.getValue());
            JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks"));
            for (int i = 0; i < tasks.size(); i++) {
                JSONObject task = tasks.getJSONObject(i);
                JSONObject param = (JSONObject) task.get("params");
                if (param != null) {
                    List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
                    ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
                    if (mainJar != null && mainJar.getId() == 0) {
                        String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s", mainJar.getRes());
                        if (resourcesMap.containsKey(fullName)) {
                            mainJar.setId(resourcesMap.get(fullName));
                            param.put("mainJar", JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
                        }
                    }
                    if (CollectionUtils.isNotEmpty(resourceList)) {
                        List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> {
                            String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s", resInfo.getRes());
                            if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) {
                                resInfo.setId(resourcesMap.get(fullName));
                            }
                            return resInfo;
                        }).collect(Collectors.toList());
                        param.put("resourceList", JSONArray.parse(JSONObject.toJSONString(newResourceList)));
                    }
                }
                task.put("params", param);
            }
            jsonObject.remove(jsonObject.getString("tasks"));
            jsonObject.put("tasks", tasks);
            replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toJSONString());
        }
        if (replaceProcessDefinitionMap.size() > 0) {
            processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(), replaceProcessDefinitionMap);
        }
    } catch (Exception e) {
        logger.error("update process definition json resource list error", e);
    }
}

Example 5 with ResourceInfo

use of org.apache.dolphinscheduler.common.process.ResourceInfo in project dolphinscheduler by apache.

the class SparkArgsUtils method buildArgs.

/**
 * build args
 *
 * @param param param
 * @return argument list
 */
public static List<String> buildArgs(SparkParameters param) {
    List<String> args = new ArrayList<>();
    String deployMode = "cluster";
    args.add(Constants.MASTER);
    if (StringUtils.isNotEmpty(param.getDeployMode())) {
        deployMode = param.getDeployMode();
    }
    if (!"local".equals(deployMode)) {
        args.add("yarn");
        args.add(Constants.DEPLOY_MODE);
    }
    args.add(param.getDeployMode());
    ProgramType type = param.getProgramType();
    String mainClass = param.getMainClass();
    if (type != null && type != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) {
        args.add(Constants.MAIN_CLASS);
        args.add(mainClass);
    }
    int driverCores = param.getDriverCores();
    if (driverCores != 0) {
        args.add(Constants.DRIVER_CORES);
        args.add(String.format("%d", driverCores));
    }
    String driverMemory = param.getDriverMemory();
    if (StringUtils.isNotEmpty(driverMemory)) {
        args.add(Constants.DRIVER_MEMORY);
        args.add(driverMemory);
    }
    int numExecutors = param.getNumExecutors();
    if (numExecutors != 0) {
        args.add(Constants.NUM_EXECUTORS);
        args.add(String.format("%d", numExecutors));
    }
    int executorCores = param.getExecutorCores();
    if (executorCores != 0) {
        args.add(Constants.EXECUTOR_CORES);
        args.add(String.format("%d", executorCores));
    }
    String executorMemory = param.getExecutorMemory();
    if (StringUtils.isNotEmpty(executorMemory)) {
        args.add(Constants.EXECUTOR_MEMORY);
        args.add(executorMemory);
    }
    // --files --conf --libjar ...
    String others = param.getOthers();
    String queue = param.getQueue();
    if (StringUtils.isNotEmpty(others)) {
        if (!others.contains(Constants.SPARK_QUEUE) && StringUtils.isNotEmpty(queue)) {
            args.add(Constants.SPARK_QUEUE);
            args.add(queue);
        }
        args.add(others);
    } else if (StringUtils.isNotEmpty(queue)) {
        args.add(Constants.SPARK_QUEUE);
        args.add(queue);
    }
    ResourceInfo mainJar = param.getMainJar();
    if (mainJar != null) {
        args.add(mainJar.getRes());
    }
    String mainArgs = param.getMainArgs();
    if (StringUtils.isNotEmpty(mainArgs)) {
        args.add(mainArgs);
    }
    return args;
}