Examples with StringMap com.linkedin.data.template.StringMap used on opensource projects

Example 31 with StringMap

use of com.linkedin.data.template.StringMap in project incubator-gobblin by apache.

the class FlowConfigTest method testUpdate.

@Test(dependsOnMethods = "testGet")
public void testUpdate() throws Exception {
    FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME);
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1b");
    flowProperties.put("param2", "value2b");
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)).setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)).setProperties(new StringMap(flowProperties));
    _client.updateFlowConfig(flowConfig);
    FlowConfig retrievedFlowConfig = _client.getFlowConfig(flowId);
    Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
    Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME);
    Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE);
    Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
    // Add this asssert when getFlowSpec() is changed to return the raw flow spec
    // Assert.assertEquals(flowConfig.getProperties().size(), 2);
    Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b");
    Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b");
}

Example 32 with StringMap

use of com.linkedin.data.template.StringMap in project incubator-gobblin by apache.

the class FlowConfigTest method testCreate.

@Test
public void testCreate() throws Exception {
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1");
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)).setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    _client.createFlowConfig(flowConfig);
}

Example 33 with StringMap

use of com.linkedin.data.template.StringMap in project incubator-gobblin by apache.

the class FlowConfigsResource method get.

/**
 * Retrieve the flow configuration with the given key
 * @param key flow config id key containing group name and flow name
 * @return {@link FlowConfig} with flow configuration
 */
@Override
public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) {
    String flowGroup = key.getKey().getFlowGroup();
    String flowName = key.getKey().getFlowName();
    LOG.info("Get called with flowGroup " + flowGroup + " flowName " + flowName);
    try {
        URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null);
        URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null);
        FlowSpec spec = (FlowSpec) getFlowCatalog().getSpec(flowUri);
        FlowConfig flowConfig = new FlowConfig();
        Properties flowProps = spec.getConfigAsProperties();
        Schedule schedule = null;
        if (flowProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
            schedule = new Schedule();
            schedule.setCronSchedule(flowProps.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY));
        }
        if (flowProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
            flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH));
        } else if (spec.getTemplateURIs().isPresent()) {
            flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ","));
        } else {
            flowConfig.setTemplateUris("NA");
        }
        if (schedule != null) {
            if (flowProps.containsKey(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)) {
                schedule.setRunImmediately(Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)));
            }
            flowConfig.setSchedule(schedule);
        }
        // remove keys that were injected as part of flowSpec creation
        flowProps.remove(ConfigurationKeys.JOB_SCHEDULE_KEY);
        flowProps.remove(ConfigurationKeys.JOB_TEMPLATE_PATH);
        StringMap flowPropsAsStringMap = new StringMap();
        flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps));
        return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)).setProperties(flowPropsAsStringMap);
    } catch (URISyntaxException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowName, e);
    } catch (SpecNotFoundException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow requested does not exist: " + flowName, null);
    }
    return null;
}

Example 34 with StringMap

use of com.linkedin.data.template.StringMap in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV100 method processQueryById.

private JobExecutionInfo processQueryById(Connection connection, String jobId, JobExecutionQuery query, Filter tableFilter) throws SQLException {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(jobId));
    // Query job execution information
    try (PreparedStatement jobExecutionQueryStatement = connection.prepareStatement(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE)) {
        jobExecutionQueryStatement.setString(1, jobId);
        try (ResultSet jobRs = jobExecutionQueryStatement.executeQuery()) {
            if (!jobRs.next()) {
                return null;
            }
            JobExecutionInfo jobExecutionInfo = resultSetToJobExecutionInfo(jobRs);
            // Query job metrics
            if (query.isIncludeJobMetrics()) {
                try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(JOB_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                    jobMetricQueryStatement.setString(1, jobRs.getString(2));
                    try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) {
                        MetricArray jobMetrics = new MetricArray();
                        while (jobMetricRs.next()) {
                            jobMetrics.add(resultSetToMetric(jobMetricRs));
                        }
                        // Add job metrics
                        jobExecutionInfo.setMetrics(jobMetrics);
                    }
                }
            }
            // Query job properties
            Set<String> requestedJobPropertyKeys = null;
            if (query.hasJobProperties()) {
                requestedJobPropertyKeys = new HashSet<>(Arrays.asList(query.getJobProperties().split(",")));
            }
            try (PreparedStatement jobPropertiesQueryStatement = connection.prepareStatement(JOB_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                jobPropertiesQueryStatement.setString(1, jobExecutionInfo.getJobId());
                try (ResultSet jobPropertiesRs = jobPropertiesQueryStatement.executeQuery()) {
                    Map<String, String> jobProperties = Maps.newHashMap();
                    while (jobPropertiesRs.next()) {
                        Map.Entry<String, String> property = resultSetToProperty(jobPropertiesRs);
                        if (requestedJobPropertyKeys == null || requestedJobPropertyKeys.contains(property.getKey())) {
                            jobProperties.put(property.getKey(), property.getValue());
                        }
                    }
                    // Add job properties
                    jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
                }
            }
            // Query task execution information
            if (query.isIncludeTaskExecutions()) {
                TaskExecutionInfoArray taskExecutionInfos = new TaskExecutionInfoArray();
                String taskExecutionQuery = TASK_EXECUTION_QUERY_STATEMENT_TEMPLATE;
                // Add table filter if applicable
                if (tableFilter.isPresent()) {
                    taskExecutionQuery += " AND " + tableFilter;
                }
                try (PreparedStatement taskExecutionQueryStatement = connection.prepareStatement(taskExecutionQuery)) {
                    taskExecutionQueryStatement.setString(1, jobId);
                    if (tableFilter.isPresent()) {
                        tableFilter.addParameters(taskExecutionQueryStatement, 2);
                    }
                    try (ResultSet taskRs = taskExecutionQueryStatement.executeQuery()) {
                        while (taskRs.next()) {
                            TaskExecutionInfo taskExecutionInfo = resultSetToTaskExecutionInfo(taskRs);
                            // Query task metrics for each task execution record
                            if (query.isIncludeTaskMetrics()) {
                                try (PreparedStatement taskMetricQueryStatement = connection.prepareStatement(TASK_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                                    taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                    try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) {
                                        MetricArray taskMetrics = new MetricArray();
                                        while (taskMetricRs.next()) {
                                            taskMetrics.add(resultSetToMetric(taskMetricRs));
                                        }
                                        // Add task metrics
                                        taskExecutionInfo.setMetrics(taskMetrics);
                                    }
                                }
                            }
                            taskExecutionInfos.add(taskExecutionInfo);
                            // Query task properties
                            Set<String> queryTaskPropertyKeys = null;
                            if (query.hasTaskProperties()) {
                                queryTaskPropertyKeys = new HashSet<>(Arrays.asList(query.getTaskProperties().split(",")));
                            }
                            try (PreparedStatement taskPropertiesQueryStatement = connection.prepareStatement(TASK_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                                taskPropertiesQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                try (ResultSet taskPropertiesRs = taskPropertiesQueryStatement.executeQuery()) {
                                    Map<String, String> taskProperties = Maps.newHashMap();
                                    while (taskPropertiesRs.next()) {
                                        Map.Entry<String, String> property = resultSetToProperty(taskPropertiesRs);
                                        if (queryTaskPropertyKeys == null || queryTaskPropertyKeys.contains(property.getKey())) {
                                            taskProperties.put(property.getKey(), property.getValue());
                                        }
                                    }
                                    // Add job properties
                                    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
                                }
                            }
                        // Add task properties
                        }
                        // Add task execution information
                        jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
                    }
                }
            }
            return jobExecutionInfo;
        }
    }
}

Example 35 with StringMap

use of com.linkedin.data.template.StringMap in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV101 method upsertTaskProperties.

private void upsertTaskProperties(Connection connection, Optional<StringMap> jobProperties, TaskExecutionInfoArray taskExecutions) throws SQLException {
    Optional<PreparedStatement> upsertStatement = Optional.absent();
    int batchSize = 0;
    for (TaskExecutionInfo taskExecution : taskExecutions) {
        if (taskExecution.hasTaskProperties()) {
            for (Map.Entry<String, String> property : taskExecution.getTaskProperties().entrySet()) {
                if (!jobProperties.isPresent() || !jobProperties.get().containsKey(property.getKey()) || !jobProperties.get().get(property.getKey()).equals(property.getValue())) {
                    if (!upsertStatement.isPresent()) {
                        upsertStatement = Optional.of(connection.prepareStatement(TASK_PROPERTY_UPSERT_STATEMENT_TEMPLATE));
                    }
                    addPropertyToBatch(upsertStatement.get(), property.getKey(), property.getValue(), taskExecution.getTaskId());
                    if (batchSize++ > 1000) {
                        executeBatches(upsertStatement);
                        upsertStatement = Optional.absent();
                        batchSize = 0;
                    }
                }
            }
        }
    }
    executeBatches(upsertStatement);
}