Examples with Credentials org.apache.storm.generated.Credentials used on opensource projects

Example 1 with Credentials

use of org.apache.storm.generated.Credentials in project storm by apache.

the class Nimbus method uploadNewCredentials.

@Override
public void uploadNewCredentials(String topoName, Credentials credentials) throws NotAliveException, InvalidTopologyException, AuthorizationException, TException {
    try {
        uploadNewCredentialsCalls.mark();
        IStormClusterState state = stormClusterState;
        String topoId = toTopoId(topoName);
        if (topoId == null) {
            throw new NotAliveException(topoName + " is not alive");
        }
        Map<String, Object> topoConf = tryReadTopoConf(topoId, blobStore);
        if (credentials == null) {
            credentials = new Credentials(Collections.emptyMap());
        }
        checkAuthorization(topoName, topoConf, "uploadNewCredentials");
        synchronized (credUpdateLock) {
            state.setCredentials(topoId, credentials, topoConf);
        }
    } catch (Exception e) {
        LOG.warn("Upload Creds topology exception. (topology name='{}')", topoName, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 2 with Credentials

use of org.apache.storm.generated.Credentials in project storm by apache.

the class Nimbus method submitTopologyWithOpts.

@Override
public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, String jsonConf, StormTopology topology, SubmitOptions options) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
    try {
        submitTopologyWithOptsCalls.mark();
        assertIsLeader();
        assert (options != null);
        validateTopologyName(topoName);
        checkAuthorization(topoName, null, "submitTopology");
        assertTopoActive(topoName, false);
        @SuppressWarnings("unchecked") Map<String, Object> topoConf = (Map<String, Object>) JSONValue.parse(jsonConf);
        try {
            ConfigValidation.validateFields(topoConf);
        } catch (IllegalArgumentException ex) {
            throw new InvalidTopologyException(ex.getMessage());
        }
        validator.validate(topoName, topoConf, topology);
        Utils.validateTopologyBlobStoreMap(topoConf, Sets.newHashSet(blobStore.listKeys()));
        long uniqueNum = submittedCount.incrementAndGet();
        String topoId = topoName + "-" + uniqueNum + "-" + Time.currentTimeSecs();
        Map<String, String> creds = null;
        if (options.is_set_creds()) {
            creds = options.get_creds().get_creds();
        }
        topoConf.put(Config.STORM_ID, topoId);
        topoConf.put(Config.TOPOLOGY_NAME, topoName);
        topoConf = normalizeConf(conf, topoConf, topology);
        ReqContext req = ReqContext.context();
        Principal principal = req.principal();
        String submitterPrincipal = principal == null ? null : principal.toString();
        String submitterUser = principalToLocal.toLocal(principal);
        String systemUser = System.getProperty("user.name");
        @SuppressWarnings("unchecked") Set<String> topoAcl = new HashSet<>((List<String>) topoConf.getOrDefault(Config.TOPOLOGY_USERS, Collections.emptyList()));
        topoAcl.add(submitterPrincipal);
        topoAcl.add(submitterUser);
        topoConf.put(Config.TOPOLOGY_SUBMITTER_PRINCIPAL, OR(submitterPrincipal, ""));
        //Don't let the user set who we launch as
        topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, OR(submitterUser, systemUser));
        topoConf.put(Config.TOPOLOGY_USERS, new ArrayList<>(topoAcl));
        topoConf.put(Config.STORM_ZOOKEEPER_SUPERACL, conf.get(Config.STORM_ZOOKEEPER_SUPERACL));
        if (!Utils.isZkAuthenticationConfiguredStormServer(conf)) {
            topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME);
            topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
        }
        if (!(Boolean) conf.getOrDefault(Config.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, false)) {
            topoConf.remove(Config.TOPOLOGY_CLASSPATH_BEGINNING);
        }
        Map<String, Object> totalConf = merge(conf, topoConf);
        topology = normalizeTopology(totalConf, topology);
        IStormClusterState state = stormClusterState;
        if (creds != null) {
            Map<String, Object> finalConf = Collections.unmodifiableMap(topoConf);
            for (INimbusCredentialPlugin autocred : nimbusAutocredPlugins) {
                autocred.populateCredentials(creds, finalConf);
            }
        }
        if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false) && (submitterUser == null || submitterUser.isEmpty())) {
            throw new AuthorizationException("Could not determine the user to run this topology as.");
        }
        //this validates the structure of the topology
        StormCommon.systemTopology(totalConf, topology);
        validateTopologySize(topoConf, conf, topology);
        if (Utils.isZkAuthenticationConfiguredStormServer(conf) && !Utils.isZkAuthenticationConfiguredTopology(topoConf)) {
            throw new IllegalArgumentException("The cluster is configured for zookeeper authentication, but no payload was provided.");
        }
        LOG.info("Received topology submission for {} with conf {}", topoName, Utils.redactValue(topoConf, Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD));
        // cleanup thread killing topology in b/w assignment and starting the topology
        synchronized (submitLock) {
            assertTopoActive(topoName, false);
            //cred-update-lock is not needed here because creds are being added for the first time.
            if (creds != null) {
                state.setCredentials(topoId, new Credentials(creds), topoConf);
            }
            LOG.info("uploadedJar {}", uploadedJarLocation);
            setupStormCode(conf, topoId, uploadedJarLocation, totalConf, topology);
            waitForDesiredCodeReplication(totalConf, topoId);
            state.setupHeatbeats(topoId);
            if (Utils.getBoolean(totalConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE), false)) {
                state.setupBackpressure(topoId);
            }
            notifyTopologyActionListener(topoName, "submitTopology");
            TopologyStatus status = null;
            switch(options.get_initial_status()) {
                case INACTIVE:
                    status = TopologyStatus.INACTIVE;
                    break;
                case ACTIVE:
                    status = TopologyStatus.ACTIVE;
                    break;
                default:
                    throw new IllegalArgumentException("Inital Status of " + options.get_initial_status() + " is not allowed.");
            }
            startTopology(topoName, topoId, status);
        }
    } catch (Exception e) {
        LOG.warn("Topology submission exception. (topology name='{}')", topoName, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 3 with Credentials

use of org.apache.storm.generated.Credentials in project storm by apache.

the class Worker method checkCredentialsChanged.

public void checkCredentialsChanged() {
    Credentials newCreds = workerState.stormClusterState.credentials(topologyId, null);
    if (!ObjectUtils.equals(newCreds, credentialsAtom.get())) {
        // This does not have to be atomic, worst case we update when one is not needed
        AuthUtils.updateSubject(subject, autoCreds, (null == newCreds) ? null : newCreds.get_creds());
        for (IRunningExecutor executor : executorsAtom.get()) {
            executor.credentialsChanged(newCreds);
        }
        credentialsAtom.set(newCreds);
    }
}

Example 4 with Credentials

use of org.apache.storm.generated.Credentials in project storm by apache.

the class Nimbus method renewCredentials.

private void renewCredentials() throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping credential renewal.");
        return;
    }
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Collection<ICredentialsRenewer> renewers = credRenewers;
    Object lock = credUpdateLock;
    List<String> assignedIds = state.activeStorms();
    if (assignedIds != null) {
        for (String id : assignedIds) {
            Map<String, Object> topoConf = Collections.unmodifiableMap(tryReadTopoConf(id, store));
            synchronized (lock) {
                Credentials origCreds = state.credentials(id, null);
                if (origCreds != null) {
                    Map<String, String> orig = origCreds.get_creds();
                    Map<String, String> newCreds = new HashMap<>(orig);
                    for (ICredentialsRenewer renewer : renewers) {
                        LOG.info("Renewing Creds For {} with {}", id, renewer);
                        renewer.renew(newCreds, topoConf);
                    }
                    if (!newCreds.equals(origCreds)) {
                        state.setCredentials(id, new Credentials(newCreds), topoConf);
                    }
                }
            }
        }
    }
}

Example 5 with Credentials

use of org.apache.storm.generated.Credentials in project storm by apache.

the class Worker method start.

public void start() throws Exception {
    LOG.info("Launching worker for {} on {}:{} with id {} and conf {}", topologyId, assignmentId, port, workerId, conf);
    // if ConfigUtils.isLocalMode(conf) returns false then it is in distributed mode.
    if (!ConfigUtils.isLocalMode(conf)) {
        // Distributed mode
        SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
        String pid = Utils.processPid();
        FileUtils.touch(new File(ConfigUtils.workerPidPath(conf, workerId, pid)));
        FileUtils.writeStringToFile(new File(ConfigUtils.workerArtifactsPidPath(conf, topologyId, port)), pid, Charset.forName("UTF-8"));
    }
    final Map topologyConf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils.readSupervisorStormConf(conf, topologyId));
    List<ACL> acls = Utils.getWorkerACL(topologyConf);
    IStateStorage stateStorage = ClusterUtils.mkStateStorage(conf, topologyConf, acls, new ClusterStateContext(DaemonType.WORKER));
    IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(stateStorage, acls, new ClusterStateContext());
    Credentials initialCredentials = stormClusterState.credentials(topologyId, null);
    Map<String, String> initCreds = new HashMap<>();
    if (initialCredentials != null) {
        initCreds.putAll(initialCredentials.get_creds());
    }
    autoCreds = AuthUtils.GetAutoCredentials(topologyConf);
    subject = AuthUtils.populateSubject(null, autoCreds, initCreds);
    Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws Exception {
            workerState = new WorkerState(conf, context, topologyId, assignmentId, port, workerId, topologyConf, stateStorage, stormClusterState);
            // Heartbeat here so that worker process dies if this fails
            // it's important that worker heartbeat to supervisor ASAP so that supervisor knows
            // that worker is running and moves on
            doHeartBeat();
            executorsAtom = new AtomicReference<>(null);
            // launch heartbeat threads immediately so that slow-loading tasks don't cause the worker to timeout
            // to the supervisor
            workerState.heartbeatTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), () -> {
                try {
                    doHeartBeat();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });
            workerState.executorHeartbeatTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), Worker.this::doExecutorHeartbeats);
            workerState.registerCallbacks();
            workerState.refreshConnections(null);
            workerState.activateWorkerWhenAllConnectionsReady();
            workerState.refreshStormActive(null);
            workerState.runWorkerStartHooks();
            List<IRunningExecutor> newExecutors = new ArrayList<IRunningExecutor>();
            for (List<Long> e : workerState.getExecutors()) {
                if (ConfigUtils.isLocalMode(topologyConf)) {
                    newExecutors.add(LocalExecutor.mkExecutor(workerState, e, initCreds).execute());
                } else {
                    newExecutors.add(Executor.mkExecutor(workerState, e, initCreds).execute());
                }
            }
            executorsAtom.set(newExecutors);
            EventHandler<Object> tupleHandler = (packets, seqId, batchEnd) -> workerState.sendTuplesToRemoteWorker((HashMap<Integer, ArrayList<TaskMessage>>) packets, seqId, batchEnd);
            // This thread will publish the messages destined for remote tasks to remote connections
            transferThread = Utils.asyncLoop(() -> {
                workerState.transferQueue.consumeBatchWhenAvailable(tupleHandler);
                return 0L;
            });
            DisruptorBackpressureCallback disruptorBackpressureHandler = mkDisruptorBackpressureHandler(workerState);
            workerState.transferQueue.registerBackpressureCallback(disruptorBackpressureHandler);
            workerState.transferQueue.setEnableBackpressure((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE));
            workerState.transferQueue.setHighWaterMark(Utils.getDouble(topologyConf.get(Config.BACKPRESSURE_DISRUPTOR_HIGH_WATERMARK)));
            workerState.transferQueue.setLowWaterMark(Utils.getDouble(topologyConf.get(Config.BACKPRESSURE_DISRUPTOR_LOW_WATERMARK)));
            WorkerBackpressureCallback backpressureCallback = mkBackpressureHandler();
            backpressureThread = new WorkerBackpressureThread(workerState.backpressureTrigger, workerState, backpressureCallback);
            if ((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE)) {
                backpressureThread.start();
                stormClusterState.topologyBackpressure(topologyId, workerState::refreshThrottle);
                int pollingSecs = Utils.getInt(topologyConf.get(Config.TASK_BACKPRESSURE_POLL_SECS));
                workerState.refreshBackpressureTimer.scheduleRecurring(0, pollingSecs, workerState::refreshThrottle);
            }
            credentialsAtom = new AtomicReference<Credentials>(initialCredentials);
            establishLogSettingCallback();
            workerState.stormClusterState.credentials(topologyId, Worker.this::checkCredentialsChanged);
            workerState.refreshCredentialsTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_CREDENTIALS_POLL_SECS), new Runnable() {

                @Override
                public void run() {
                    checkCredentialsChanged();
                    if ((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE)) {
                        checkThrottleChanged();
                    }
                }
            });
            // The jitter allows the clients to get the data at different times, and avoids thundering herd
            if (!(Boolean) topologyConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING)) {
                workerState.refreshLoadTimer.scheduleRecurringWithJitter(0, 1, 500, workerState::refreshLoad);
            }
            workerState.refreshConnectionsTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshConnections);
            workerState.resetLogLevelsTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_LOG_LEVEL_RESET_POLL_SECS), logConfigManager::resetLogLevels);
            workerState.refreshActiveTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshStormActive);
            LOG.info("Worker has topology config {}", Utils.redactValue(topologyConf, Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD));
            LOG.info("Worker {} for storm {} on {}:{}  has finished loading", workerId, topologyId, assignmentId, port);
            return this;
        }

        ;
    });
}