Examples with ComponentCommon org.apache.storm.generated.ComponentCommon used on opensource projects

Example 16 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class TopologySpoutLag method getLagResultForKafka.

private static Map<String, Object> getLagResultForKafka(String spoutId, SpoutSpec spoutSpec, Map topologyConf, boolean old) throws IOException {
    ComponentCommon componentCommon = spoutSpec.get_common();
    String json = componentCommon.get_json_conf();
    Map<String, Object> result = null;
    String errorMsg = "Offset lags for kafka not supported for older versions. Please update kafka spout to latest version.";
    if (json != null && !json.isEmpty()) {
        List<String> commands = new ArrayList<>();
        String stormHomeDir = System.getenv("STORM_BASE_DIR");
        if (stormHomeDir != null && !stormHomeDir.endsWith("/")) {
            stormHomeDir += File.separator;
        }
        commands.add(stormHomeDir != null ? stormHomeDir + "bin" + File.separator + "storm-kafka-monitor" : "storm-kafka-monitor");
        Map<String, Object> jsonMap = null;
        try {
            jsonMap = (Map<String, Object>) JSONValue.parseWithException(json);
        } catch (ParseException e) {
            throw new IOException(e);
        }
        commands.addAll(old ? getCommandLineOptionsForOldKafkaSpout(jsonMap, topologyConf) : getCommandLineOptionsForNewKafkaSpout(jsonMap));
        logger.debug("Command to run: {}", commands);
        // if commands contains one or more null value, spout is compiled with lower version of storm-kafka / storm-kafka-client
        if (!commands.contains(null)) {
            String resultFromMonitor = ShellUtils.execCommand(commands.toArray(new String[0]));
            try {
                result = (Map<String, Object>) JSONValue.parseWithException(resultFromMonitor);
            } catch (ParseException e) {
                logger.debug("JSON parsing failed, assuming message as error message: {}", resultFromMonitor);
                // json parsing fail -> error received
                errorMsg = resultFromMonitor;
            }
        }
    }
    Map<String, Object> kafkaSpoutLagInfo = new HashMap<>();
    kafkaSpoutLagInfo.put(SPOUT_ID, spoutId);
    kafkaSpoutLagInfo.put(SPOUT_TYPE, "KAFKA");
    if (result != null) {
        kafkaSpoutLagInfo.put(SPOUT_LAG_RESULT, result);
    } else {
        kafkaSpoutLagInfo.put(ERROR_INFO, errorMsg);
    }
    return kafkaSpoutLagInfo;
}

Example 17 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class TopologySpoutLag method addLagResultForKafkaSpout.

private static void addLagResultForKafkaSpout(Map<String, Map<String, Object>> finalResult, String spoutId, SpoutSpec spoutSpec) throws IOException {
    ComponentCommon componentCommon = spoutSpec.get_common();
    String json = componentCommon.get_json_conf();
    if (!Strings.isNullOrEmpty(json)) {
        Map<String, Object> jsonMap = null;
        try {
            jsonMap = (Map<String, Object>) JSONValue.parseWithException(json);
        } catch (ParseException e) {
            throw new IOException(e);
        }
        if (jsonMap.containsKey(TOPICS_CONFIG) && jsonMap.containsKey(GROUPID_CONFIG) && jsonMap.containsKey(BOOTSTRAP_CONFIG)) {
            finalResult.put(spoutId, getLagResultForNewKafkaSpout(spoutId, spoutSpec));
        }
    }
}

Example 18 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class TopologySpoutLag method getLagResultForKafka.

private static Map<String, Object> getLagResultForKafka(String spoutId, SpoutSpec spoutSpec) throws IOException {
    ComponentCommon componentCommon = spoutSpec.get_common();
    String json = componentCommon.get_json_conf();
    Map<String, Object> result = null;
    String errorMsg = new StringBuilder("Make sure Kafka spout version is latest and ").append(TOPICS_CONFIG).append(", ").append(GROUPID_CONFIG).append(" & ").append(BOOTSTRAP_CONFIG).append(" are not null for newer versions of Kafka spout.").toString();
    if (!Strings.isNullOrEmpty(json)) {
        List<String> commands = new ArrayList<>();
        String stormHomeDir = System.getenv("STORM_BASE_DIR");
        if (stormHomeDir != null && !stormHomeDir.endsWith("/")) {
            stormHomeDir += File.separator;
        }
        commands.add(stormHomeDir != null ? stormHomeDir + "bin" + File.separator + "storm-kafka-monitor" : "storm-kafka-monitor");
        Map<String, Object> jsonMap = null;
        try {
            jsonMap = (Map<String, Object>) JSONValue.parseWithException(json);
        } catch (ParseException e) {
            throw new IOException(e);
        }
        commands.addAll(getCommandLineOptionsForNewKafkaSpout(jsonMap));
        File extraPropertiesFile = createExtraPropertiesFile(jsonMap);
        if (extraPropertiesFile != null) {
            commands.add("-c");
            commands.add(extraPropertiesFile.getAbsolutePath());
        }
        LOGGER.debug("Command to run: {}", commands);
        // if commands contains one or more null value, spout is compiled with lower version of storm-kafka-client
        if (!commands.contains(null)) {
            try {
                String resultFromMonitor = new ShellCommandRunnerImpl().execCommand(commands.toArray(new String[0]));
                try {
                    result = (Map<String, Object>) JSONValue.parseWithException(resultFromMonitor);
                } catch (ParseException e) {
                    LOGGER.debug("JSON parsing failed, assuming message as error message: {}", resultFromMonitor);
                    // json parsing fail -> error received
                    errorMsg = resultFromMonitor;
                }
            } finally {
                if (extraPropertiesFile != null) {
                    extraPropertiesFile.delete();
                }
            }
        }
    }
    Map<String, Object> kafkaSpoutLagInfo = new HashMap<>();
    kafkaSpoutLagInfo.put(SPOUT_ID, spoutId);
    kafkaSpoutLagInfo.put(SPOUT_TYPE, "KAFKA");
    if (result != null) {
        kafkaSpoutLagInfo.put(SPOUT_LAG_RESULT, result);
    } else {
        kafkaSpoutLagInfo.put(ERROR_INFO, errorMsg);
    }
    return kafkaSpoutLagInfo;
}

Example 19 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class ThriftTopologyUtilsTest method testGetComponentCommonWithWorkerHook.

@Test
public void testGetComponentCommonWithWorkerHook() {
    StormTopology stormTopology = genereateStormTopology(true);
    ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, "bolt-1");
    Assert.assertEquals("We expect to get bolt-1's common", new Bolt().get_common(), componentCommon);
}

Example 20 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method addMetricStreams.

public static void addMetricStreams(StormTopology topology) {
    for (Object component : allComponents(topology).values()) {
        ComponentCommon common = getComponentCommon(component);
        StreamInfo streamInfo = Thrift.outputFields(Arrays.asList("task-info", "data-points"));
        common.put_to_streams(Constants.METRICS_STREAM_ID, streamInfo);
    }
}