use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.
the class Nimbus method uploadNewCredentials.
@Override
public void uploadNewCredentials(String topoName, Credentials credentials) throws NotAliveException, InvalidTopologyException, AuthorizationException, TException {
try {
uploadNewCredentialsCalls.mark();
IStormClusterState state = stormClusterState;
String topoId = toTopoId(topoName);
if (topoId == null) {
throw new NotAliveException(topoName + " is not alive");
}
Map<String, Object> topoConf = tryReadTopoConf(topoId, blobStore);
if (credentials == null) {
credentials = new Credentials(Collections.emptyMap());
}
checkAuthorization(topoName, topoConf, "uploadNewCredentials");
synchronized (credUpdateLock) {
state.setCredentials(topoId, credentials, topoConf);
}
} catch (Exception e) {
LOG.warn("Upload Creds topology exception. (topology name='{}')", topoName, e);
if (e instanceof TException) {
throw (TException) e;
}
throw new RuntimeException(e);
}
}
use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.
the class Nimbus method submitTopologyWithOpts.
@Override
public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, String jsonConf, StormTopology topology, SubmitOptions options) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
try {
submitTopologyWithOptsCalls.mark();
assertIsLeader();
assert (options != null);
validateTopologyName(topoName);
checkAuthorization(topoName, null, "submitTopology");
assertTopoActive(topoName, false);
@SuppressWarnings("unchecked") Map<String, Object> topoConf = (Map<String, Object>) JSONValue.parse(jsonConf);
try {
ConfigValidation.validateFields(topoConf);
} catch (IllegalArgumentException ex) {
throw new InvalidTopologyException(ex.getMessage());
}
validator.validate(topoName, topoConf, topology);
Utils.validateTopologyBlobStoreMap(topoConf, Sets.newHashSet(blobStore.listKeys()));
long uniqueNum = submittedCount.incrementAndGet();
String topoId = topoName + "-" + uniqueNum + "-" + Time.currentTimeSecs();
Map<String, String> creds = null;
if (options.is_set_creds()) {
creds = options.get_creds().get_creds();
}
topoConf.put(Config.STORM_ID, topoId);
topoConf.put(Config.TOPOLOGY_NAME, topoName);
topoConf = normalizeConf(conf, topoConf, topology);
ReqContext req = ReqContext.context();
Principal principal = req.principal();
String submitterPrincipal = principal == null ? null : principal.toString();
String submitterUser = principalToLocal.toLocal(principal);
String systemUser = System.getProperty("user.name");
@SuppressWarnings("unchecked") Set<String> topoAcl = new HashSet<>((List<String>) topoConf.getOrDefault(Config.TOPOLOGY_USERS, Collections.emptyList()));
topoAcl.add(submitterPrincipal);
topoAcl.add(submitterUser);
topoConf.put(Config.TOPOLOGY_SUBMITTER_PRINCIPAL, OR(submitterPrincipal, ""));
//Don't let the user set who we launch as
topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, OR(submitterUser, systemUser));
topoConf.put(Config.TOPOLOGY_USERS, new ArrayList<>(topoAcl));
topoConf.put(Config.STORM_ZOOKEEPER_SUPERACL, conf.get(Config.STORM_ZOOKEEPER_SUPERACL));
if (!Utils.isZkAuthenticationConfiguredStormServer(conf)) {
topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME);
topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
}
if (!(Boolean) conf.getOrDefault(Config.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, false)) {
topoConf.remove(Config.TOPOLOGY_CLASSPATH_BEGINNING);
}
Map<String, Object> totalConf = merge(conf, topoConf);
topology = normalizeTopology(totalConf, topology);
IStormClusterState state = stormClusterState;
if (creds != null) {
Map<String, Object> finalConf = Collections.unmodifiableMap(topoConf);
for (INimbusCredentialPlugin autocred : nimbusAutocredPlugins) {
autocred.populateCredentials(creds, finalConf);
}
}
if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false) && (submitterUser == null || submitterUser.isEmpty())) {
throw new AuthorizationException("Could not determine the user to run this topology as.");
}
//this validates the structure of the topology
StormCommon.systemTopology(totalConf, topology);
validateTopologySize(topoConf, conf, topology);
if (Utils.isZkAuthenticationConfiguredStormServer(conf) && !Utils.isZkAuthenticationConfiguredTopology(topoConf)) {
throw new IllegalArgumentException("The cluster is configured for zookeeper authentication, but no payload was provided.");
}
LOG.info("Received topology submission for {} with conf {}", topoName, Utils.redactValue(topoConf, Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD));
// cleanup thread killing topology in b/w assignment and starting the topology
synchronized (submitLock) {
assertTopoActive(topoName, false);
//cred-update-lock is not needed here because creds are being added for the first time.
if (creds != null) {
state.setCredentials(topoId, new Credentials(creds), topoConf);
}
LOG.info("uploadedJar {}", uploadedJarLocation);
setupStormCode(conf, topoId, uploadedJarLocation, totalConf, topology);
waitForDesiredCodeReplication(totalConf, topoId);
state.setupHeatbeats(topoId);
if (Utils.getBoolean(totalConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE), false)) {
state.setupBackpressure(topoId);
}
notifyTopologyActionListener(topoName, "submitTopology");
TopologyStatus status = null;
switch(options.get_initial_status()) {
case INACTIVE:
status = TopologyStatus.INACTIVE;
break;
case ACTIVE:
status = TopologyStatus.ACTIVE;
break;
default:
throw new IllegalArgumentException("Inital Status of " + options.get_initial_status() + " is not allowed.");
}
startTopology(topoName, topoId, status);
}
} catch (Exception e) {
LOG.warn("Topology submission exception. (topology name='{}')", topoName, e);
if (e instanceof TException) {
throw (TException) e;
}
throw new RuntimeException(e);
}
}
use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.
the class StormCommon method validateIds.
@SuppressWarnings("unchecked")
private static void validateIds(StormTopology topology) throws InvalidTopologyException {
List<String> componentIds = new ArrayList<>();
for (StormTopology._Fields field : Thrift.getTopologyFields()) {
if (!ThriftTopologyUtils.isWorkerHook(field) && !ThriftTopologyUtils.isDependencies(field)) {
Object value = topology.getFieldValue(field);
Map<String, Object> componentMap = (Map<String, Object>) value;
componentIds.addAll(componentMap.keySet());
for (String id : componentMap.keySet()) {
if (Utils.isSystemId(id)) {
throw new InvalidTopologyException(id + " is not a valid component id.");
}
}
for (Object componentObj : componentMap.values()) {
ComponentCommon common = getComponentCommon(componentObj);
Set<String> streamIds = common.get_streams().keySet();
for (String id : streamIds) {
if (Utils.isSystemId(id)) {
throw new InvalidTopologyException(id + " is not a valid stream id.");
}
}
}
}
}
List<String> offending = Utils.getRepeat(componentIds);
if (!offending.isEmpty()) {
throw new InvalidTopologyException("Duplicate component ids: " + offending);
}
}
use of org.apache.storm.generated.InvalidTopologyException in project flink by apache.
the class FlinkClient method submitTopologyWithOpts.
/**
* Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink does not support
* uploading a jar file before hand. Jar files are always uploaded directly when a program is submitted.
*/
public void submitTopologyWithOpts(final String name, final String uploadedJarLocation, final FlinkTopology topology) throws AlreadyAliveException, InvalidTopologyException {
if (this.getTopologyJobId(name) != null) {
throw new AlreadyAliveException();
}
final URI uploadedJarUri;
final URL uploadedJarUrl;
try {
uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI();
uploadedJarUrl = uploadedJarUri.toURL();
JobWithJars.checkJarFile(uploadedJarUrl);
} catch (final IOException e) {
throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e);
}
try {
FlinkClient.addStormConfigToTopology(topology, conf);
} catch (ClassNotFoundException e) {
LOG.error("Could not register class for Kryo serialization.", e);
throw new InvalidTopologyException("Could not register class for Kryo serialization.");
}
final StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph();
streamGraph.setJobName(name);
final JobGraph jobGraph = streamGraph.getJobGraph();
jobGraph.addJar(new Path(uploadedJarUri));
final Configuration configuration = jobGraph.getJobConfiguration();
configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerHost);
configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);
final ClusterClient client;
try {
client = new StandaloneClusterClient(configuration);
} catch (final IOException e) {
throw new RuntimeException("Could not establish a connection to the job manager", e);
}
try {
ClassLoader classLoader = JobWithJars.buildUserCodeClassLoader(Collections.<URL>singletonList(uploadedJarUrl), Collections.<URL>emptyList(), this.getClass().getClassLoader());
client.runDetached(jobGraph, classLoader);
} catch (final ProgramInvocationException e) {
throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e);
}
}
use of org.apache.storm.generated.InvalidTopologyException in project flink by apache.
the class FlinkSubmitter method submitTopology.
/**
* Submits a topology to run on the cluster. A topology runs forever or until explicitly killed. The given {@link
* FlinkProgressListener} is ignored because progress bars are not supported by Flink.
*
* @param name
* the name of the storm.
* @param stormConf
* the topology-specific configuration. See {@link Config}.
* @param topology
* the processing to execute.
* @throws AlreadyAliveException
* if a topology with this name is already running
* @throws InvalidTopologyException
* if an invalid topology was submitted
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void submitTopology(final String name, final Map stormConf, final FlinkTopology topology) throws AlreadyAliveException, InvalidTopologyException {
if (!Utils.isValidConf(stormConf)) {
throw new IllegalArgumentException("Storm conf is not valid. Must be json-serializable");
}
final Configuration flinkConfig = GlobalConfiguration.loadConfiguration();
if (!stormConf.containsKey(Config.NIMBUS_HOST)) {
stormConf.put(Config.NIMBUS_HOST, flinkConfig.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost"));
}
if (!stormConf.containsKey(Config.NIMBUS_THRIFT_PORT)) {
stormConf.put(Config.NIMBUS_THRIFT_PORT, new Integer(flinkConfig.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 6123)));
}
final String serConf = JSONValue.toJSONString(stormConf);
final FlinkClient client = FlinkClient.getConfiguredClient(stormConf);
try {
if (client.getTopologyJobId(name) != null) {
throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
}
String localJar = System.getProperty("storm.jar");
if (localJar == null) {
try {
for (final URL url : ((ContextEnvironment) ExecutionEnvironment.getExecutionEnvironment()).getJars()) {
// TODO verify that there is only one jar
localJar = new File(url.toURI()).getAbsolutePath();
}
} catch (final URISyntaxException e) {
// ignore
} catch (final ClassCastException e) {
// ignore
}
}
logger.info("Submitting topology " + name + " in distributed mode with conf " + serConf);
client.submitTopologyWithOpts(name, localJar, topology);
} catch (final InvalidTopologyException e) {
logger.warn("Topology submission exception: " + e.get_msg());
throw e;
} catch (final AlreadyAliveException e) {
logger.warn("Topology already alive exception", e);
throw e;
}
logger.info("Finished submitting topology: " + name);
}
Aggregations