Search in sources :

Example 16 with BugzillaAPIException

use of com.redhat.qe.auto.bugzilla.BugzillaAPIException in project rhsm-qe by RedHatQE.

the class SubscriptionManagerTasks method logRuntimeErrors.

/**
 * Check the SSHCommandResult for errors.  If an error is detected, log a warning with information to help troubleshoot it.
 * @param result
 */
public void logRuntimeErrors(SSHCommandResult result) {
    String issue;
    // TEMPORARY WORKAROUND FOR BUG
    // 201704041152:20.587 - FINE: ssh root@hp-moonshot-03-c36.lab.eng.rdu.redhat.com subscription-manager identity
    // 201704041152:21.594 - FINE: Stdout:
    // 201704041152:21.594 - FINE: Stderr:
    // This system is not yet registered. Try 'subscription-manager register --help' for more information.
    // 
    // ** COLLECTED WARNINGS **
    // /sys/firmware/efi/systab: SMBIOS entry point missing
    // No SMBIOS nor DMI entry point found, sorry.
    // ** END OF WARNINGS **
    // 
    // 201704041152:21.595 - FINE: ExitCode: 1
    issue = "** COLLECTED WARNINGS **";
    if (result.getStderr().contains(issue)) {
        // Bug 1438869 - No SMBIOS nor DMI entry point found, sorry.
        String bugId = "1438869";
        // Bug 1438869 - No SMBIOS nor DMI entry point found, sorry.
        boolean invokeWorkaroundWhileBugIsOpen = true;
        try {
            if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
            } else {
                invokeWorkaroundWhileBugIsOpen = false;
            }
        } catch (BugzillaAPIException be) {
        /* ignore exception */
        } catch (RuntimeException re) {
        /* ignore exception */
        }
        if (invokeWorkaroundWhileBugIsOpen) {
            throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
        }
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // 201611111427:53.146 - FINE: ssh root@jsefler-rhel6server.usersys.redhat.com subscription-manager register --username=testuser1 --password=password --org=admin
    // 201611111427:53.954 - FINE: Stdout: Registering to: jsefler-candlepin.usersys.redhat.com:8443/candlepin
    // 201611111427:53.954 - FINE: Stderr: 'module' object has no attribute 'PROXY_AUTHENTICATION_REQUIRED'
    // 201611111427:53.954 - FINE: ExitCode: 70
    issue = "'module' object has no attribute 'PROXY_AUTHENTICATION_REQUIRED'";
    if (result.getStderr().contains(issue)) {
        // Bug 1394351 - 'module' object has no attribute 'PROXY_AUTHENTICATION_REQUIRED'
        String bugId = "1394351";
        // Bug 1394351 - 'module' object has no attribute 'PROXY_AUTHENTICATION_REQUIRED'
        boolean invokeWorkaroundWhileBugIsOpen = true;
        try {
            if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
            } else {
                invokeWorkaroundWhileBugIsOpen = false;
            }
        } catch (BugzillaAPIException be) {
        /* ignore exception */
        } catch (RuntimeException re) {
        /* ignore exception */
        }
        if (invokeWorkaroundWhileBugIsOpen) {
            throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
        }
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // 201611111443:59.584 - FINE: ssh root@jsefler-rhel6server.usersys.redhat.com subscription-manager register --username=testuser1 --password=password --org=admin
    // 201611111444:00.277 - FINE: Stdout: Registering to: jsefler-candlepin.usersys.redhat.com:8443/candlepin
    // 201611111444:00.277 - FINE: Stderr: global name 'socket' is not defined
    // 201611111444:00.277 - FINE: ExitCode: 70
    issue = "global name 'socket' is not defined";
    if (result.getStderr().contains(issue)) {
        // Bug 1390688 - global name 'socket' is not defined
        String bugId = "1390688";
        // Bug 1390688 - global name 'socket' is not defined
        boolean invokeWorkaroundWhileBugIsOpen = true;
        try {
            if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
            } else {
                invokeWorkaroundWhileBugIsOpen = false;
            }
        } catch (BugzillaAPIException be) {
        /* ignore exception */
        } catch (RuntimeException re) {
        /* ignore exception */
        }
        if (invokeWorkaroundWhileBugIsOpen) {
            throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
        }
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // 201503290125:44.627 - FINE: ssh root@jsefler-os6server.usersys.redhat.com subscription-manager unsubscribe --all
    // 201503290125:46.273 - FINE: Stdout:
    // 3 subscriptions removed at the server.
    // 3 local certificates have been deleted.
    // 201503290125:46.274 - FINE: Stderr:
    // Traceback (most recent call last):
    // File "/usr/share/rhsm/subscription_manager/dbus_interface.py", line 59, in emit_status
    // self.validity_iface.emit_status()
    // File "/usr/lib/python2.6/site-packages/dbus/proxies.py", line 68, in __call__
    // return self._proxy_method(*args, **keywords)
    // File "/usr/lib/python2.6/site-packages/dbus/proxies.py", line 140, in __call__
    // **keywords)
    // File "/usr/lib/python2.6/site-packages/dbus/connection.py", line 630, in call_blocking
    // message, timeout)
    // dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
    // 201503290125:46.274 - FINE: ExitCode: 0
    issue = "Message did not receive a reply (timeout by message bus)";
    // dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Did not receive a reply. Possible causes include: the remote application did not send a reply, the message bus security policy blocked the reply, the reply timeout expired, or the network connection was broken.
    issue = "dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply";
    if (result.getStderr().contains(issue)) {
        // Bug 1207306 - dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        String bugId = "1207306";
        // Bug 1207306 - dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        boolean invokeWorkaroundWhileBugIsOpen = true;
        try {
            if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
            } else {
                invokeWorkaroundWhileBugIsOpen = false;
            }
        } catch (BugzillaAPIException be) {
        /* ignore exception */
        } catch (RuntimeException re) {
        /* ignore exception */
        }
        if (invokeWorkaroundWhileBugIsOpen) {
            throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
        }
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // 201503281401:54.767 - FINE: Stdout: Successfully attached a subscription for: Red Hat Enterprise Linux High Touch Beta
    // 201503281401:54.767 - FINE: Stderr:
    // Traceback (most recent call last):
    // File "/usr/share/rhsm/subscription_manager/dbus_interface.py", line 59, in emit_status
    // self.validity_iface.emit_status()
    // File "/usr/lib/python2.6/site-packages/dbus/proxies.py", line 68, in __call__
    // return self._proxy_method(*args, **keywords)
    // File "/usr/lib/python2.6/site-packages/dbus/proxies.py", line 140, in __call__
    // **keywords)
    // File "/usr/lib/python2.6/site-packages/dbus/connection.py", line 630, in call_blocking
    // message, timeout)
    // dbus.exceptions.DBusException: org.freedesktop.DBus.Python.UnboundLocalError: Traceback (most recent call last):
    // File "/usr/lib/python2.6/site-packages/dbus/service.py", line 702, in _message_cb
    // retval = candidate_method(self, *args, **keywords)
    // File "/usr/libexec/rhsmd", line 202, in emit_status
    // self._dbus_properties = refresh_compliance_status(self._dbus_properties)
    // File "/usr/share/rhsm/subscription_manager/managerlib.py", line 920, in refresh_compliance_status
    // entitlements[label] = (name, state, message)
    // UnboundLocalError: local variable 'state' referenced before assignment
    // 201503281401:54.767 - FINE: ExitCode: 0
    issue = "UnboundLocalError: local variable 'state' referenced before assignment";
    if (result.getStderr().contains(issue)) {
        // Bug 1198369 - UnboundLocalError: local variable 'state' referenced before assignment
        String bugId = "1198369";
        // Bug 1198369 - UnboundLocalError: local variable 'state' referenced before assignment
        boolean invokeWorkaroundWhileBugIsOpen = true;
        try {
            if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
            } else {
                invokeWorkaroundWhileBugIsOpen = false;
            }
        } catch (BugzillaAPIException be) {
        /* ignore exception */
        } catch (RuntimeException re) {
        /* ignore exception */
        }
        if (invokeWorkaroundWhileBugIsOpen) {
            throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
        }
    }
    // Let's tweak SSHCommandResult result for results that come from an expect script...
    if (result.getStdout().trim().startsWith("spawn")) {
        if (result.getStderr().trim().isEmpty()) {
            // create a new SSHCommandResult result with a fake exitCode and the actual stdout stuffed into stderr
            result = new SSHCommandResult(new Integer(-1), result.getStdout(), result.getStdout());
        }
    }
    // no reason to suspect an error when ExitCode is 0
    if (result.getExitCode() != null) {
        if (result.getExitCode().equals(0)) {
            return;
        }
    }
    if ((result.getStdout() + result.getStderr()).toLowerCase().contains("Runtime Error".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Error updating system data on the server".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("undefined method".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("The proxy server received an invalid response from an upstream server".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Problem encountered".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Remote server error".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Unable to verify server's identity".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Unable to reach the server".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Connection reset by peer".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Request failed due to concurrent modification".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("The system is unable to complete the requested transaction".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("object is not iterable".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("Unable to serialize objects to JSON.".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("timeout by message bus".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("timed out".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("'idCert'".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains(("See " + rhsmLogFile).toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().contains("''".toLowerCase()) || (result.getStdout() + result.getStderr()).toLowerCase().trim().isEmpty() || (result.getExitCode() == null) || (result.getExitCode() > (Integer.valueOf(1)))) {
        // some commands legitimately return 1
        // [root@jsefler-7 ~]# LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        String getTracebackCommand = "LINE_NUMBER=$(grep --line-number 'Making request:' " + rhsmLogFile + " | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n \"$LINE_NUMBER\" ]; then tail -n +$LINE_NUMBER " + rhsmLogFile + "; fi;";
        SSHCommandResult getTracebackCommandResult = sshCommandRunner.runCommandAndWaitWithoutLogging(getTracebackCommand);
        if (!getTracebackCommandResult.getStdout().isEmpty())
            log.warning("Last request from " + rhsmLogFile + ":\n" + getTracebackCommandResult.getStdout());
        // /*debugTestSSL*/ if ((result.getStdout()+result.getStderr()).toLowerCase().contains("Unable to verify server's identity".toLowerCase())) log.warning("Current RHSM Configuration:\n"); sshCommandRunner.runCommandAndWait("subscription-manager config --list");
        // TEMPORARY WORKAROUND FOR BUG
        // 201411121258:42.413 - FINE: ssh root@ibm-p8-kvm-04-guest-01.rhts.eng.bos.redhat.com subscription-manager register --username=stage_auto_testuser --password=**** --baseurl=myhost.example.com/ --force (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201411121258:53.631 - FINE: Stdout:
        // The system with UUID 44d314cd-23e4-43e3-a256-a74828d6377f has been unregistered
        // The system has been registered with ID: 2017fa81-1b82-46da-84db-4b789118c6a9
        // Remote server error. Please check the connection details, or see /var/log/rhsm/rhsm.log for more information.
        // (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411121258:53.632 - FINE: Stderr:  (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411121258:53.632 - FINE: ExitCode: 70 (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411121258:53.632 - FINE: ssh root@ibm-p8-kvm-04-guest-01.rhts.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi; (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201411121258:53.837 - WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2014-11-12 12:58:53,331 [DEBUG] rhsmd @connection.py:466 - Making request: GET /subscription/consumers/2017fa81-1b82-46da-84db-4b789118c6a9/compliance
        // 2014-11-12 12:58:53,802 [DEBUG] rhsmd @connection.py:489 - Response: status=401
        // 2014-11-12 12:58:54,770 [DEBUG] subscription-manager @connection.py:489 - Response: status=502
        // 2014-11-12 12:58:54,770 [ERROR] subscription-manager @connection.py:518 - Response: 502
        // 2014-11-12 12:58:54,771 [ERROR] subscription-manager @connection.py:519 - JSON parsing error: No JSON object could be decoded
        // 2014-11-12 12:58:54,771 [ERROR] subscription-manager @managercli.py:157 - exception caught in subscription-manager
        // 2014-11-12 12:58:54,771 [ERROR] subscription-manager @managercli.py:158 - Server error attempting a GET to /subscription/ returned status 502
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 82, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 73, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 2530, in main
        // return CLI.main(self)
        // File "/usr/share/rhsm/subscription_manager/cli.py", line 160, in main
        // return cmd.main()
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 478, in main
        // return_code = self._do_command()
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1092, in _do_command
        // profile_mgr.update_check(self.cp, consumer['uuid'], True)
        // File "/usr/share/rhsm/subscription_manager/cache.py", line 348, in update_check
        // if not uep.supports_resource(PACKAGES_RESOURCE):
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 716, in supports_resource
        // self._load_supported_resources()
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 703, in _load_supported_resources
        // resources_list = self.conn.request_get("/")
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 570, in request_get
        // return self._request("GET", method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 498, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 546, in validateResponse
        // handler=handler)
        // RemoteServerException: Server error attempting a GET to /subscription/ returned status 502
        // 2014-11-12 19:10:09,319 [DEBUG] subscription-manager @connection.py:466 - Making request: GET /subscription/consumers/2031a746-d558-4dbe-9edb-4bded14b6a92/certificates/serials
        // 2014-11-12 19:10:09,820 [DEBUG] subscription-manager @connection.py:489 - Response: status=502
        // 2014-11-12 19:10:09,820 [ERROR] subscription-manager @connection.py:518 - Response: 502
        // 2014-11-12 19:10:09,820 [ERROR] subscription-manager @connection.py:519 - JSON parsing error: No JSON object could be decoded
        // 2014-11-12 19:10:09,821 [ERROR] subscription-manager @managercli.py:157 - Unable to attach: Server error attempting a GET to /subscription/consumers/2031a746-d558-4dbe-9edb-4bded14b6a92/certificates/serials returned status 502
        // 2014-11-12 19:10:09,821 [ERROR] subscription-manager @managercli.py:158 - Server error attempting a GET to /subscription/consumers/2031a746-d558-4dbe-9edb-4bded14b6a92/certificates/serials returned status 502
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1509, in _do_command
        // report = self.entcertlib.update()
        // File "/usr/share/rhsm/subscription_manager/certlib.py", line 31, in update
        // self.report = self.locker.run(self._do_update)
        // File "/usr/share/rhsm/subscription_manager/certlib.py", line 17, in run
        // return action()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 43, in _do_update
        // return action.perform()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 119, in perform
        // expected = self._get_expected_serials()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 254, in _get_expected_serials
        // exp = self.get_certificate_serials_list()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 234, in get_certificate_serials_list
        // reply = self.uep.getCertificateSerials(identity.uuid)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 965, in getCertificateSerials
        // return self.conn.request_get(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 570, in request_get
        // return self._request("GET", method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 498, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 546, in validateResponse
        // handler=handler)
        // RemoteServerException: Server error attempting a GET to /subscription/consumers/2031a746-d558-4dbe-9edb-4bded14b6a92/certificates/serials returned status 502
        // 2014-11-26 10:03:18,366 [DEBUG] subscription-manager @connection.py:466 - Making request: GET /subscription/consumers/69dfafe4-04b4-44aa-8173-06c215632710
        // 2014-11-26 10:03:25,766 [DEBUG] subscription-manager @connection.py:489 - Response: status=502
        // 2014-11-26 10:03:25,767 [ERROR] subscription-manager @managercli.py:874 - The proxy server received an invalid response from an upstream server
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 868, in _do_command
        // self.show_service_level()
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 892, in show_service_level
        // consumer = self.cp.getConsumer(self.identity.uuid)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 869, in getConsumer
        // return self.conn.request_get(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 570, in request_get
        // return self._request("GET", method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 498, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 540, in validateResponse
        // raise RestlibException(response['status'], error_msg)
        // RestlibException: The proxy server received an invalid response from an upstream server
        issue = "Response: status=502";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1105173 - subscription-manager encounters frequent 502 responses from stage IT-Candlepin
            String bugId = "1105173";
            // Bug 1105173 - subscription-manager encounters frequent 502 responses from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            // duplicate of Bug 1113741 - RHEL 7 (and 6?): subscription-manager fails with "JSON parsing error: No JSON object could be decoded" error
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2014-11-16 05:22:14,215 [DEBUG] subscription-manager @connection.py:466 - Making request: POST /subscription/consumers/a21f1acf-ddd7-420a-8903-d75e9ba45e1f/entitlements?pool=8a99f981498757d40149a5a9b04f4b00
        // 2014-11-16 05:23:14,628 [ERROR] subscription-manager @managercli.py:157 - Unable to attach: timed out
        // 2014-11-16 05:23:14,628 [ERROR] subscription-manager @managercli.py:158 - timed out
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1462, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 974, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 573, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 480, in _request
        // response = conn.getresponse()
        // File "/usr/lib64/python2.7/httplib.py", line 1045, in getresponse
        // response.begin()
        // File "/usr/lib64/python2.7/httplib.py", line 409, in begin
        // version, status, reason = self._read_status()
        // File "/usr/lib64/python2.7/httplib.py", line 365, in _read_status
        // line = self.fp.readline(_MAXLINE + 1)
        // File "/usr/lib64/python2.7/socket.py", line 476, in readline
        // data = self._sock.recv(self._rbufsize)
        // File "/usr/lib64/python2.7/site-packages/M2Crypto/SSL/Connection.py", line 228, in read
        // return self._read_bio(size)
        // File "/usr/lib64/python2.7/site-packages/M2Crypto/SSL/Connection.py", line 213, in _read_bio
        // return m2.ssl_read(self.ssl, size, self._timeout)
        // SSLTimeoutError: timed out
        issue = "SSLTimeoutError: timed out";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1165239 - subscription-manager encounters frequent SSLTimeoutErrors from stage IT-Candlepin
            String bugId = "1165239";
            // Bug 1165239 - subscription-manager encounters frequent SSLTimeoutErrors from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-07-20 16:59:05,912 [DEBUG] subscription-manager:31469 @connection.py:573 - Making request: PUT /subscription/consumers/a508bc4a-0986-4795-be1f-8a058a2b44a4/packages
        // 2016-07-20 16:59:36,992 [DEBUG] subscription-manager:31469 @connection.py:602 - Response: status=500
        // 2016-07-20 16:59:36,993 [ERROR] subscription-manager:31469 @connection.py:631 - Response: 500
        // 2016-07-20 16:59:36,993 [ERROR] subscription-manager:31469 @connection.py:632 - JSON parsing error: No JSON object could be decoded
        // 2016-07-20 16:59:36,993 [ERROR] subscription-manager:31469 @cache.py:166 - Error updating system data on the server
        // 2016-07-20 16:59:36,993 [ERROR] subscription-manager:31469 @cache.py:167 - Server error attempting a PUT to /subscription/consumers/a508bc4a-0986-4795-be1f-8a058a2b44a4/packages returned status 500
        // Traceback (most recent call last):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cache.py", line 158, in update_check
        // self._sync_with_server(uep, consumer_uuid)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cache.py", line 417, in _sync_with_server
        // self.current_profile.collect())
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1055, in updatePackageProfile
        // ret = self.conn.request_put(method, pkg_dicts)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 703, in request_put
        // return self._request("PUT", method, params)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 611, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 667, in validateResponse
        // handler=handler)
        // RemoteServerException: Server error attempting a PUT to /subscription/consumers/a508bc4a-0986-4795-be1f-8a058a2b44a4/packages returned status 500
        // 2016-07-20 16:59:36,995 [ERROR] subscription-manager:31469 @managercli.py:174 - exception caught in subscription-manager
        // 2016-07-20 16:59:36,995 [ERROR] subscription-manager:31469 @managercli.py:175 - Error updating system data on the server, see /var/log/rhsm/rhsm.log for more details.
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 81, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 72, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 2732, in main
        // return CLI.main(self)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cli.py", line 160, in main
        // return cmd.main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 526, in main
        // return_code = self._do_command()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 1161, in _do_command
        // profile_mgr.update_check(self.cp, consumer['uuid'], True)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cache.py", line 405, in update_check
        // return CacheManager.update_check(self, uep, consumer_uuid, force)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cache.py", line 168, in update_check
        // raise Exception(_("Error updating system data on the server, see /var/log/rhsm/rhsm.log "
        // Exception: Error updating system data on the server, see /var/log/rhsm/rhsm.log for more details.
        issue = "Error updating system data on the server";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1358508 - Error updating system data on the server
            String bugId = "1358508";
            // Bug 1358508 - Error updating system data on the server
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2014-11-18 13:31:35,122 [DEBUG] subscription-manager @connection.py:466 - Making request: DELETE /subscription/consumers/b7b49c97-f25a-43b7-9820-e7cc76bccbc3/entitlements
        // 2014-11-18 13:32:34,127 [DEBUG] subscription-manager @connection.py:489 - Response: status=500
        // 2014-11-18 13:32:34,127 [ERROR] subscription-manager @managercli.py:1625 - Runtime Error Lock wait timeout exceeded; try restarting transaction at com.mysql.jdbc.SQLError.createSQLException:1,078
        issue = "Runtime Error Lock wait timeout exceeded";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1084782 - Runtime Error Lock wait timeout exceeded; try restarting transaction at com.mysql.jdbc.SQLError.createSQLException:1,078
            String bugId = "1084782";
            // Bug 1084782 - Runtime Error Lock wait timeout exceeded; try restarting transaction at com.mysql.jdbc.SQLError.createSQLException:1,078
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1165295 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded" from stage IT-Candlepin
            String bugId = "1165295";
            // Bug 1165295 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded" from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1161736 - subscription-manager doesn't behave in a consistent way
            String bugId = "1161736";
            // Bug 1161736 - subscription-manager doesn't behave in a consistent way
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1231308 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded"/"Unable to verify server's identity" from stage IT-Candlepin
            String bugId = "1231308";
            // Bug 1231308 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded"/"Unable to verify server's identity" from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1357117 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded" from stage IT-Candlepin
            String bugId = "1357117";
            // Bug 1357117 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded" from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2015-03-31 03:19:17,873 [DEBUG] rhsmd:22939 @connection.py:494 - Making request: GET /candlepin/consumers/589555ff-42bf-45e4-9799-1419bc945006/compliance
        // 2015-03-31 03:19:18,270 [DEBUG] rhsmd:22939 @connection.py:521 - Response: status=200, requestUuid=eabfdf97-edf7-44ac-91ee-815e82c350be
        // 2015-03-31 03:19:18,339 [DEBUG] rhsmd:22939 @cache.py:272 - Started thread to write cache: /var/lib/rhsm/cache/entitlement_status.json
        // 2015-03-31 03:19:19,130 [DEBUG] subscription-manager:22933 @connection.py:521 - Response: status=500
        // 2015-03-31 03:19:19,130 [ERROR] subscription-manager:22933 @managercli.py:161 - Unregister failed
        // 2015-03-31 03:19:19,130 [ERROR] subscription-manager:22933 @managercli.py:162 - Runtime Error could not execute statement at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse:2,102
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1240, in _do_command
        // managerlib.unregister(self.cp, self.identity.uuid)
        // File "/usr/share/rhsm/subscription_manager/managerlib.py", line 788, in unregister
        // uep.unregisterConsumer(consumer_uuid)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 990, in unregisterConsumer
        // return self.conn.request_delete(method)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 614, in request_delete
        // return self._request("DELETE", method, params)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 530, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 572, in validateResponse
        // raise RestlibException(response['status'], error_msg)
        // RestlibException: Runtime Error could not execute statement at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse:2,102
        issue = "Runtime Error could not execute statement at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse:2,102";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1207721 - Runtime Error could not execute statement at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse:2,102
            String bugId = "1207721";
            // Bug 1207721 - Runtime Error could not execute statement at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse:2,102
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 201411291639:35.361 - FINE: ssh root@cloud-qe-22.idmqe.lab.eng.bos.redhat.com subscription-manager unsubscribe --serial=8305861300287544370 (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201411291639:41.734 - FINE: Stdout:
        // Serial numbers unsuccessfully removed at the server:
        // The proxy server received an invalid response from an upstream server
        // (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411291639:41.735 - FINE: Stderr:  (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411291639:41.735 - FINE: ExitCode: 1 (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201411291639:41.735 - FINE: ssh root@cloud-qe-22.idmqe.lab.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi; (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201411291639:41.924 - WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2014-11-29 16:39:42,297 [DEBUG] subscription-manager @connection.py:466 - Making request: GET /subscription/consumers/5de32c6a-7a46-4769-a69f-ea7c4f7a8db2/compliance
        // 2014-11-29 16:39:42,799 [DEBUG] subscription-manager @connection.py:489 - Response: status=200
        // 2014-11-29 16:39:42,799 [DEBUG] subscription-manager @cache.py:249 - Started thread to write cache: /var/lib/rhsm/cache/entitlement_status.json
        // 2014-11-29 16:39:42,800 [DEBUG] subscription-manager @cert_sorter.py:193 - valid entitled products: []
        // 2014-11-29 16:39:42,800 [DEBUG] subscription-manager @cert_sorter.py:194 - expired entitled products: []
        // 2014-11-29 16:39:42,800 [DEBUG] subscription-manager @cert_sorter.py:195 - partially entitled products: []
        // 2014-11-29 16:39:42,800 [DEBUG] subscription-manager @cert_sorter.py:196 - unentitled products: ['71']
        // 2014-11-29 16:39:42,800 [DEBUG] subscription-manager @cert_sorter.py:197 - future products: []
        // 2014-11-29 16:39:42,801 [DEBUG] subscription-manager @cert_sorter.py:198 - partial stacks: []
        // 2014-11-29 16:39:42,801 [DEBUG] subscription-manager @cert_sorter.py:199 - entitlements valid until: None
        // 2014-11-29 16:39:42,913 [INFO] rhsmd @rhsmd:273 - rhsmd started
        // 2014-11-29 16:39:42,915 [INFO] rhsmd @rhsmd:182 - D-Bus interface com.redhat.SubscriptionManager.EntitlementStatus.update_status called with status = 1
        // 2014-11-29 16:39:42,951 [DEBUG] rhsmd @identity.py:131 - Loading consumer info from identity certificates.
        // 2014-11-29 16:39:42,954 [INFO] rhsmd @rhsmd:149 - D-Bus signal com.redhat.SubscriptionManager.EntitlementStatus.entitlement_status_changed emitted
        // 2014-11-29 16:39:42,969 [DEBUG] subscription-manager @dbus_interface.py:60 - Failed to update rhsmd
        // 2014-11-29 16:39:42,969 [ERROR] subscription-manager @dbus_interface.py:61 - org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/dbus_interface.py", line 57, in _update
        // self.validity_iface.emit_status(ignore_reply=self.has_main_loop)
        // File "/usr/lib64/python2.7/site-packages/dbus/proxies.py", line 145, in __call__
        // **keywords)
        // File "/usr/lib64/python2.7/site-packages/dbus/connection.py", line 651, in call_blocking
        // message, timeout)
        // DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        issue = "Message did not receive a reply (timeout by message bus)";
        issue = "DBusException: org.freedesktop.DBus.Error.NoReply";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1207306 - dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
            String bugId = "1207306";
            // Bug 1207306 - dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2017-03-30 15:32:21.794  FINE: ssh root@ibm-x3550m3-07.lab.eng.brq.redhat.com subscription-manager list --available
        // 2017-03-30 15:32:22.569  FINE: Stdout:
        // 2017-03-30 15:32:22.569  FINE: Stderr: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        // 2017-03-30 15:32:22.569  FINE: ExitCode: 70
        // 2017-03-30 15:32:22.570  FINE: ssh root@ibm-x3550m3-07.lab.eng.brq.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2017-03-30 15:32:23.124  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2017-03-30 21:32:16,929 [DEBUG] subscription-manager:12742:MainThread @connection.py:473 - Making request: GET /subscription/owners/7964055/pools?consumer=0d8975a8-8400-4e6a-a009-3f425f0ca843
        // 2017-03-30 21:32:18,436 [INFO] subscription-manager:12742:MainThread @connection.py:509 - Response: status=200, requestUuid=268b7e91-fed9-4f55-ac59-e200be523559, request="GET /subscription/owners/7964055/pools?consumer=0d8975a8-8400-4e6a-a009-3f425f0ca843"
        // 2017-03-30 21:32:18,441 [DEBUG] subscription-manager:12742:MainThread @managerlib.py:550 - Filtering 0 total pools
        // 2017-03-30 21:32:18,441 [DEBUG] subscription-manager:12742:MainThread @managerlib.py:556 -      Removed 0 incompatible pools
        // 2017-03-30 21:32:18,441 [DEBUG] subscription-manager:12742:MainThread @managerlib.py:589 -      13 pools to display, -13 filtered out
        // 2017-03-30 21:32:21,558 [DEBUG] subscription-manager:12802:MainThread @https.py:54 - Using standard libs to provide httplib and ssl
        // 2017-03-30 21:32:21,659 [DEBUG] subscription-manager:12802:MainThread @dbus_interface.py:35 - self.has_main_loop=False
        // 2017-03-30 21:32:21,721 [DEBUG] subscription-manager:12802:MainThread @ga_loader.py:89 - ga_loader GaImporterGtk3
        // 2017-03-30 21:32:21,729 [DEBUG] subscription-manager:12802:MainThread @plugins.py:569 - loaded plugin modules: [<module 'container_content' from '/usr/share/rhsm-plugins/container_content.pyc'>, <module 'ostree_content' from '/usr/share/rhsm-plugins/ostree_content.pyc'>]
        // 2017-03-30 21:32:21,729 [DEBUG] subscription-manager:12802:MainThread @plugins.py:570 - loaded plugins: {'container_content.ContainerContentPlugin': <container_content.ContainerContentPlugin object at 0x1221590>, 'ostree_content.OstreeContentPlugin': <ostree_content.OstreeContentPlugin object at 0x1221b50>}
        // 2017-03-30 21:32:21,729 [DEBUG] subscription-manager:12802:MainThread @identity.py:132 - Loading consumer info from identity certificates.
        // 2017-03-30 21:32:21,787 [INFO] subscription-manager:12802:MainThread @managercli.py:316 - X-Correlation-ID: 47a6ded7ad2b4611935850319cf8d19b
        // 2017-03-30 21:32:21,788 [INFO] subscription-manager:12802:MainThread @managercli.py:394 - Client Versions: {'python-rhsm': '1.19.2-1.el7', 'subscription-manager': '1.19.4-1.el7'}
        // 2017-03-30 21:32:21,789 [INFO] subscription-manager:12802:MainThread @connection.py:763 - Connection built: host=subscription.rhsm.stage.redhat.com port=443 handler=/subscription auth=identity_cert ca_dir=/etc/rhsm/ca/ insecure=False
        // 2017-03-30 21:32:21,789 [INFO] subscription-manager:12802:MainThread @connection.py:763 - Connection built: host=subscription.rhsm.stage.redhat.com port=443 handler=/subscription auth=none
        // 2017-03-30 21:32:21,790 [INFO] subscription-manager:12802:MainThread @managercli.py:369 - Consumer Identity name=ibm-x3550m3-07.lab.eng.brq.redhat.com uuid=0d8975a8-8400-4e6a-a009-3f425f0ca843
        // 2017-03-30 21:32:21,790 [DEBUG] subscription-manager:12802:MainThread @cache.py:157 - Checking current system info against cache: /var/lib/rhsm/facts/facts.json
        // 2017-03-30 21:32:21,915 [ERROR] subscription-manager:12802:MainThread @managercli.py:179 - exception caught in subscription-manager
        // 2017-03-30 21:32:21,915 [ERROR] subscription-manager:12802:MainThread @managercli.py:180 - org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 89, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 80, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 2792, in main
        // return CLI.main(self)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cli.py", line 160, in main
        // return cmd.main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 544, in main
        // return_code = self._do_command()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 2372, in _do_command
        // filter_string=self.options.filter_string)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 325, in get_available_entitlements
        // overlapping, uninstalled, text, filter_string)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 529, in get_filtered_pools_list
        // self.identity.uuid, active_on=active_on, filter_string=filter_string):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 283, in list_pools
        // require(FACTS).update_check(uep, consumer_uuid)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cache.py", line 158, in update_check
        // if self.has_changed() or force:
        // File "/usr/lib/python2.7/site-packages/subscription_manager/facts.py", line 69, in has_changed
        // self.facts = self.get_facts(True)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/facts.py", line 79, in get_facts
        // facts = facts_dbus_client.GetFacts()
        // File "/usr/lib/python2.7/site-packages/rhsmlib/dbus/facts/client.py", line 57, in GetFacts
        // return self.interface.GetFacts(*args, **kwargs)
        // File "/usr/lib64/python2.7/site-packages/dbus/proxies.py", line 70, in __call__
        // return self._proxy_method(*args, **keywords)
        // File "/usr/lib64/python2.7/site-packages/dbus/proxies.py", line 145, in __call__
        // **keywords)
        // File "/usr/lib64/python2.7/site-packages/dbus/connection.py", line 651, in call_blocking
        // message, timeout)
        // DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
        issue = "Message did not receive a reply (timeout by message bus)";
        issue = "DBusException: org.freedesktop.DBus.Error.NoReply";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1438561 - DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
            String bugId = "1438561";
            // Bug 1438561 - DBusException: org.freedesktop.DBus.Error.NoReply: Message did not receive a reply (timeout by message bus)
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2015-05-11 20:56:36,029 [DEBUG] subscription-manager:70001 @connection.py:494 - Making request: POST /subscription/consumers/4abab952-0b4b-4daa-bb34-5a0938c99672/entitlements?pool=8a99f9864d0ba396014d10a2642a1537
        // 2015-05-11 20:56:48,012 [DEBUG] subscription-manager:70001 @connection.py:521 - Response: status=500
        // 2015-05-11 20:56:48,012 [ERROR] subscription-manager:70001 @managercli.py:1496 - Runtime Error com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1486, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 1017, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 605, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 530, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 572, in validateResponse
        // raise RestlibException(response['status'], error_msg)
        // RestlibException: Runtime Error com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
        issue = "Runtime Error com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock";
        if (getTracebackCommandResult.getStdout().contains(issue)) {
            // Bug 1220830 - subscription-manager encounters occasional Deadlock from stage IT-Candlepin
            String bugId = "1220830";
            // Bug 1220830 - subscription-manager encounters occasional Deadlock from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 201506121027:12.829 - FINE: ssh root@jsefler-os6.usersys.redhat.com subscription-manager subscribe --pool=8a99f98a4d730ed9014d96e90c9d275a --pool=8a99f9874d730eea014d96e682e1121e --pool=8a99f9874d730eea014d96e6849f124b --pool=8a99f9894d730ec6014d96e73e4064d9 --pool=8a99f9874d730eea014d96e687b4129e --pool=8a99f9874d730eea014d96e680ae11b9 --pool=8a99f9874d730eea014d96e6886312c7 --pool=8a99f9874d730eea014d96e68651126b --pool=8a99f9874d730eea014d96e67fd41189 --pool=8a99f9874d730eea014d96e682c71206 --pool=8a99f9874d730eea014d96e67fbb1174 --pool=8a99f9874d730eea014d96e6809811a2 --pool=8a99f9874d730eea014d96e683d1123a --pool=8a99f9874d730eea014d96e681b611eb --pool=8a99f9874d730eea014d96e6819e11d4 --pool=8a99f9874d730eea014d96e6890c12eb (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201506121036:51.780 - FINE: Stdout:
        // Successfully attached a subscription for: Red Hat JBoss Enterprise Application Platform with Management, 16 Core Standard, L3 Support Partner
        // Successfully attached a subscription for: Red Hat Enterprise Linux Workstation, Standard
        // Successfully attached a subscription for: Red Hat Enterprise Linux Server, Standard (Physical or Virtual Nodes)
        // Successfully attached a subscription for: Red Hat Enterprise Linux for IBM POWER, Standard (4 sockets) (Up to 30 LPARs) with Smart Management
        // Successfully attached a subscription for: Red Hat Enterprise Linux Server, Premium (Physical or Virtual Nodes)
        // Successfully attached a subscription for: Red Hat Enterprise Linux Server for HPC Compute Node, Self-support (8 sockets) (Up to 1 guest)
        // Successfully attached a subscription for: Red Hat Enterprise Linux High Touch Beta
        // Successfully attached a subscription for: Red Hat Enterprise Linux for Virtual Datacenters, Premium
        // Successfully attached a subscription for: Red Hat Enterprise Linux Desktop, Self-support
        // Successfully attached a subscription for: Red Hat Enterprise Linux Workstation, Standard
        // (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201506121036:51.796 - FINE: Stderr: Unable to verify server's identity: (104, 'Connection reset by peer')
        // (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201506121036:51.798 - FINE: ExitCode: 70 (com.redhat.qe.tools.SSHCommandRunner.runCommandAndWait)
        // 201506121036:51.814 - FINE: ssh root@jsefler-os6.usersys.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi; (com.redhat.qe.tools.SSHCommandRunner.run)
        // 201506121036:51.868 - WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2015-06-12 10:36:02,337 [DEBUG] rhsmd:1998 @connection.py:494 - Making request: GET /subscription/consumers/e5c74163-3bdb-47c6-ba26-f4d392d8bd92/compliance
        // 2015-06-12 10:36:51,737 [ERROR] subscription-manager:1721 @managercli.py:160 - Unable to attach: (104, 'Connection reset by peer')
        // 2015-06-12 10:36:51,737 [ERROR] subscription-manager:1721 @managercli.py:161 - (104, 'Connection reset by peer')
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1486, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 1017, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 605, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 512, in _request
        // response = conn.getresponse()
        // File "/usr/lib64/python2.6/httplib.py", line 1012, in getresponse
        // response.begin()
        // File "/usr/lib64/python2.6/httplib.py", line 404, in begin
        // version, status, reason = self._read_status()
        // File "/usr/lib64/python2.6/httplib.py", line 360, in _read_status
        // line = self.fp.readline(_MAXLINE + 1)
        // File "/usr/lib64/python2.6/socket.py", line 479, in readline
        // data = self._sock.recv(self._rbufsize)
        // File "/usr/lib64/python2.6/site-packages/M2Crypto/SSL/Connection.py", line 228, in read
        // return self._read_bio(size)
        // File "/usr/lib64/python2.6/site-packages/M2Crypto/SSL/Connection.py", line 213, in _read_bio
        // return m2.ssl_read(self.ssl, size, self._timeout)
        // SSLError: (104, 'Connection reset by peer')
        issue = "(104, 'Connection reset by peer')";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1231308 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded"/"Unable to verify server's identity" from stage IT-Candlepin
            String bugId = "1231308";
            // Bug 1231308 - subscription-manager encounters frequent "Runtime Error Lock wait timeout exceeded"/"Unable to verify server's identity" from stage IT-Candlepin
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1302364 - Unable to verify server's identity: (104, 'Connection reset by peer')
            String bugId = "1302364";
            // Bug 1302364 - Unable to verify server's identity: (104, 'Connection reset by peer')
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2018-01-15 14:39:26,359 [DEBUG] subscription-manager:34555:MainThread  @connection.py:543 - Making request: DELETE  /subscription/consumers/c3b6eb11-da15-47ea-893d-7d3896af5952/entitlements
        // 2018-01-15  14:39:55,131 [ERROR] subscription-manager:34555:MainThread  @managercli.py:181 - Unable to perform remove due to the following  exception: [Errno 104] Connection reset by peer
        // 2018-01-15 14:39:55,131 [ERROR] subscription-manager:34555:MainThread @managercli.py:182 - [Errno 104] Connection reset by peer
        // Traceback (most recent call last):
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/managercli.py", line 1707, in _do_command
        // total = ent_service.remove_all_entitlements()
        // File "/usr/lib64/python2.7/site-packages/rhsmlib/services/entitlement.py", line 303, in remove_all_entitlements
        // response = self.cp.unbindAll(self.identity.uuid)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1273, in unbindAll
        // return self.conn.request_delete(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 702, in request_delete
        // return self._request("DELETE", method, params, headers=headers)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 716, in _request
        // info=info, headers=headers)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 573, in _request
        // response = conn.getresponse()
        // File "/usr/lib64/python2.7/httplib.py", line 1089, in getresponse
        // response.begin()
        // File "/usr/lib64/python2.7/httplib.py", line 444, in begin
        // version, status, reason = self._read_status()
        // File "/usr/lib64/python2.7/httplib.py", line 400, in _read_status
        // line = self.fp.readline(_MAXLINE + 1)
        // File "/usr/lib64/python2.7/socket.py", line 476, in readline
        // data = self._sock.recv(self._rbufsize)
        // File "/usr/lib64/python2.7/ssl.py", line 759, in recv
        // return self.read(buflen)
        // File "/usr/lib64/python2.7/ssl.py", line 653, in read
        // v = self._sslobj.read(len or 1024)
        // error: [Errno 104] Connection reset by peer
        issue = "[Errno 104] Connection reset by peer";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1535150 - [Errno 104] Connection reset by peer
            String bugId = "1535150";
            // Bug 1535150 - [Errno 104] Connection reset by peer
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-01-27 16:24:22.520  FINE: ssh root@ibm-x3550m3-09.lab.eng.brq.redhat.com subscription-manager unsubscribe --all
        // 2016-01-27 16:24:50.438  FINE: Stdout: 1 subscription removed at the server.
        // 
        // 2016-01-27 16:24:50.439  FINE: Stderr: Network error, unable to connect to server. Please see /var/log/rhsm/rhsm.log for more information.
        // 
        // 2016-01-27 16:24:50.439  FINE: ExitCode: 70
        // 2016-01-27 16:24:50.439  FINE: ssh root@ibm-x3550m3-09.lab.eng.brq.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2016-01-27 16:24:50.896  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2016-01-27 22:24:33,609 [DEBUG] subscription-manager:23027 @connection.py:557 - Making request: GET /subscription/consumers/162ac9ed-d2aa-45d9-921a-ce3aeaae180d/certificates/serials
        // 2016-01-27 22:24:51,334 [ERROR] subscription-manager:23027 @entcertlib.py:121 - [Errno -3] Temporary failure in name resolution
        // Traceback (most recent call last):
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 119, in perform
        // expected = self._get_expected_serials()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 254, in _get_expected_serials
        // exp = self.get_certificate_serials_list()
        // File "/usr/share/rhsm/subscription_manager/entcertlib.py", line 234, in get_certificate_serials_list
        // reply = self.uep.getCertificateSerials(identity.uuid)
        // File "/usr/lib/python2.6/site-packages/rhsm/connection.py", line 1145, in getCertificateSerials
        // return self.conn.request_get(method)
        // File "/usr/lib/python2.6/site-packages/rhsm/connection.py", line 681, in request_get
        // return self._request("GET", method)
        // File "/usr/lib/python2.6/site-packages/rhsm/connection.py", line 571, in _request
        // conn.request(request_type, handler, body=body, headers=headers)
        // File "/usr/lib/python2.6/httplib.py", line 936, in request
        // self._send_request(method, url, body, headers)
        // File "/usr/lib/python2.6/httplib.py", line 973, in _send_request
        // self.endheaders()
        // File "/usr/lib/python2.6/httplib.py", line 930, in endheaders
        // self._send_output()
        // File "/usr/lib/python2.6/httplib.py", line 802, in _send_output
        // self.send(msg)
        // File "/usr/lib/python2.6/httplib.py", line 761, in send
        // self.connect()
        // File "/usr/lib/python2.6/site-packages/M2Crypto/httpslib.py", line 51, in connect
        // socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
        // gaierror: [Errno -3] Temporary failure in name resolution
        issue = "[Errno -3] Temporary failure in name resolution";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1302798 - gaierror: [Errno -3] Temporary failure in name resolution
            String bugId = "1302798";
            // Bug 1302798 - gaierror: [Errno -3] Temporary failure in name resolution
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-08-01 20:00:32.143  FINE: ssh root@ibm-z10-77.rhts.eng.bos.redhat.com subscription-manager list --available
        // 2016-08-01 20:00:51.651  FINE: Stdout:
        // 2016-08-01 20:00:51.651  FINE: Stderr: Unable to serialize objects to JSON.
        // 
        // 2016-08-01 20:00:51.651  FINE: ExitCode: 70
        // 2016-08-01 20:00:51.651  FINE: ssh root@ibm-z10-77.rhts.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2016-08-01 20:00:51.815  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2016-08-01 20:00:58,186 [DEBUG] subscription-manager:30729 @connection.py:573 - Making request: GET /subscription/owners/7964055/pools?consumer=4b7a3fff-2b66-4a82-9741-fb4821e4b364
        // 2016-08-01 20:01:14,852 [DEBUG] subscription-manager:30729 @connection.py:602 - Response: status=500
        // 2016-08-01 20:01:14,852 [ERROR] subscription-manager:30729 @managercli.py:174 - exception caught in subscription-manager
        // 2016-08-01 20:01:14,852 [ERROR] subscription-manager:30729 @managercli.py:175 - Unable to serialize objects to JSON.
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 81, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 72, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 2732, in main
        // return CLI.main(self)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/cli.py", line 160, in main
        // return cmd.main()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 526, in main
        // return_code = self._do_command()
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 2321, in _do_command
        // filter_string=self.options.filter_string)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 314, in get_available_entitlements
        // overlapping, uninstalled, text, filter_string)
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 519, in get_filtered_pools_list
        // self.identity.uuid, self.facts, active_on=active_on, filter_string=filter_string):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managerlib.py", line 278, in list_pools
        // active_on=active_on, owner=ownerid, filter_string=filter_string)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1260, in getPoolsList
        // results = self.conn.request_get(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 694, in request_get
        // return self._request("GET", method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 611, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 661, in validateResponse
        // raise RestlibException(response['status'], error_msg, response.get('headers'))
        // RestlibException: Unable to serialize objects to JSON.
        issue = "Unable to serialize objects to JSON.";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1362535 - Unable to serialize objects to JSON.
            String bugId = "1362535";
            // Bug 1362535 - Unable to serialize objects to JSON.
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-08-12 00:33:35.448  FINE: ssh root@wolverine.idmqe.lab.eng.bos.redhat.com subscription-manager subscribe --pool=8a99f9815582f734015585f99da5513f --pool=8a99f9815582f734015585f99c47511c --pool=8a99f9815582f734015585f99add50f4 --pool=8a99f9815582f734015585f99a0950b9 --pool=8a99f9815582f734015585f99e9d519d --pool=8a99f9815582f734015585f9a7c952b9 --pool=8a99f9815582f734015585f9a654524a --pool=8a99f9815582f734015585f9989d5047 --pool=8a99f9815582f734015585f9a0c4521d --pool=8a99f9815582f734015585f99f4051c9 --pool=8a99f9815582f734015585f9995e5080 --pool=8a99f9815582f734015585f99fc851ee --pool=8a99f9815582f734015585f99b72510a
        // 2016-08-12 00:33:57.891  FINE: Stdout:
        // Successfully attached a subscription for: Red Hat Enterprise Linux for Virtual Datacenters, Premium
        // Successfully attached a subscription for: Red Hat Enterprise Linux Server, Standard (Physical or Virtual Nodes)
        // Successfully attached a subscription for: Red Hat Enterprise Linux for IBM POWER, Standard (4 sockets) (Up to 30 LPARs) with Smart Management
        // Successfully attached a subscription for: Red Hat Enterprise Linux Workstation, Standard
        // 
        // 2016-08-12 00:33:57.891  FINE: Stderr: Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
        // 
        // 2016-08-12 00:33:57.891  FINE: ExitCode: 70
        // 2016-08-12 00:33:57.892  FINE: ssh root@wolverine.idmqe.lab.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2016-08-12 00:33:58.129  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2016-08-12 00:33:50,844 [DEBUG] subscription-manager:2690:MainThread @connection.py:573 - Making request: POST /subscription/consumers/b8f5adea-bfea-44c4-a522-955f58a61a70/entitlements?pool=8a99f9815582f734015585f99e9d519d
        // 2016-08-12 00:33:51,919 [DEBUG] subscription-manager:2690:MainThread @connection.py:602 - Response: status=500
        // 2016-08-12 00:33:51,920 [ERROR] subscription-manager:2690:MainThread @managercli.py:1566 - Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
        // Traceback (most recent call last):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 1556, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1169, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 697, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 611, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 661, in validateResponse
        // raise RestlibException(response['status'], error_msg, response.get('headers'))
        // RestlibException: Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
        issue = "Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1366772 - Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
            String bugId = "1366772";
            // Bug 1366772 - Runtime Error org.hibernate.exception.LockAcquisitionException: could not execute statement at sun.reflect.NativeConstructorAccessorImpl.newInstance0:-2
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-09-08 06:15:07.029  FINE: ssh root@hp-moonshot-03-c07.lab.eng.rdu.redhat.com subscription-manager subscribe --pool=8a99f9815582f734015585f9989d5047
        // 2016-09-08 06:16:08.858  FINE: Stdout:
        // 2016-09-08 06:16:08.858  FINE: Stderr: Runtime Error could not obtain pessimistic lock at com.mysql.jdbc.SQLError.createSQLException:1,078
        // 
        // 2016-09-08 06:16:08.859  FINE: ExitCode: 70
        // 2016-09-08 06:16:08.859  FINE: ssh root@hp-moonshot-03-c07.lab.eng.rdu.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2016-09-08 06:16:08.957  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2016-09-08 06:15:20,797 [DEBUG] subscription-manager:30381:MainThread @connection.py:573 - Making request: POST /subscription/consumers/25f1a0fb-59cf-48aa-a089-897a01282502/entitlements?pool=8a99f9815582f734015585f9989d5047
        // 2016-09-08 06:16:12,079 [DEBUG] subscription-manager:30381:MainThread @connection.py:602 - Response: status=500
        // 2016-09-08 06:16:12,080 [ERROR] subscription-manager:30381:MainThread @managercli.py:1570 - Runtime Error could not obtain pessimistic lock at com.mysql.jdbc.SQLError.createSQLException:1,078
        // Traceback (most recent call last):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 1560, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1169, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 697, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 611, in _request
        // self.validateResponse(result, request_type, handler)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 661, in validateResponse
        // raise RestlibException(response['status'], error_msg, response.get('headers'))
        // RestlibException: Runtime Error could not obtain pessimistic lock at com.mysql.jdbc.SQLError.createSQLException:1,078
        issue = "Runtime Error could not obtain pessimistic lock";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1374448 - Runtime Error could not obtain pessimistic lock at com.mysql.jdbc.SQLError.createSQLException:1,078
            String bugId = "1374448";
            // Bug 1374448 - Runtime Error could not obtain pessimistic lock at com.mysql.jdbc.SQLError.createSQLException:1,078
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2016-09-08 06:14:00.215  FINE: ssh root@ibm-z10-13.rhts.eng.bos.redhat.com subscription-manager subscribe --pool=8a99f9815582f734015585f99973509a
        // 2016-09-08 06:16:53.512  FINE: Stdout:
        // 2016-09-08 06:16:53.512  FINE: Stderr: ''
        // 
        // 2016-09-08 06:16:53.512  FINE: ExitCode: 70
        // 2016-09-08 06:16:53.512  FINE: ssh root@ibm-z10-13.rhts.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2016-09-08 06:16:54.358  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2016-09-08 06:14:16,544 [DEBUG] subscription-manager:3495:MainThread @connection.py:573 - Making request: POST /subscription/consumers/ec27ba92-b7e5-4f97-8872-d9a3de796bcb/entitlements?pool=8a99f9815582f734015585f99973509a
        // 2016-09-08 06:16:51,800 [ERROR] subscription-manager:3495:MainThread @managercli.py:174 - Unable to attach: ''
        // 2016-09-08 06:16:51,800 [ERROR] subscription-manager:3495:MainThread @managercli.py:175 - ''
        // Traceback (most recent call last):
        // File "/usr/lib/python2.7/site-packages/subscription_manager/managercli.py", line 1560, in _do_command
        // ents = self.cp.bindByEntitlementPool(self.identity.uuid, pool, self.options.quantity)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 1169, in bindByEntitlementPool
        // return self.conn.request_post(method)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 697, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.7/site-packages/rhsm/connection.py", line 591, in _request
        // response = conn.getresponse()
        // File "/usr/lib64/python2.7/httplib.py", line 1089, in getresponse
        // response.begin()
        // File "/usr/lib64/python2.7/httplib.py", line 444, in begin
        // version, status, reason = self._read_status()
        // File "/usr/lib64/python2.7/httplib.py", line 408, in _read_status
        // raise BadStatusLine(line)
        // BadStatusLine: ''
        issue = "BadStatusLine: ''";
        // 2017-02-24 22:10:28.427  FINE: ssh root@ivanova.idmqe.lab.eng.bos.redhat.com subscription-manager register --username=stage_auto_testuser1 --password=redhat --autosubscribe --servicelevel=stAndarD --force
        // 2017-02-24 22:13:10.090  FINE: Stdout:
        // The system with UUID 032c087d-0c70-48a2-96b3-3db2233d7628 has been unregistered
        // Registering to: subscription.rhsm.stage.redhat.com:443/subscription
        // 
        // 2017-02-24 22:13:10.091  FINE: Stderr: Remote server error. Please check the connection details, or see /var/log/rhsm/rhsm.log for more information.
        // 
        // 2017-02-24 22:13:10.091  FINE: ExitCode: 70
        // 2017-02-24 22:13:10.091  FINE: ssh root@ivanova.idmqe.lab.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2017-02-24 22:13:10.359  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2017-02-24 22:10:38,227 [DEBUG] subscription-manager:6475:MainThread @connection.py:490 - Making request: POST /subscription/consumers?owner=7964055
        // 2017-02-24 22:13:16,278 [ERROR] subscription-manager:6475:MainThread @managercli.py:177 - Error during registration:
        // 2017-02-24 22:13:16,279 [ERROR] subscription-manager:6475:MainThread @managercli.py:178 -
        // Traceback (most recent call last):
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managercli.py", line 1149, in _do_command
        // content_tags=self.installed_mgr.tags)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 856, in registerConsumer
        // return self.conn.request_post(url, params)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 626, in request_post
        // return self._request("POST", method, params)
        // File "/usr/lib64/python2.6/site-packages/rhsm/connection.py", line 512, in _request
        // response = conn.getresponse()
        // File "/usr/lib64/python2.6/site-packages/rhsm/m2cryptohttp.py", line 182, in getresponse
        // return self._connection.getresponse(*args, **kwargs)
        // File "/usr/lib64/python2.6/httplib.py", line 1049, in getresponse
        // response.begin()
        // File "/usr/lib64/python2.6/httplib.py", line 433, in begin
        // version, status, reason = self._read_status()
        // File "/usr/lib64/python2.6/httplib.py", line 397, in _read_status
        // raise BadStatusLine(line)
        // BadStatusLine
        issue = "BadStatusLine";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1374460 - sometimes stage candlepin does not return any error message; appears as a BadStatusLine: ''
            String bugId = "1374460";
            // Bug 1374460 - sometimes stage candlepin does not return any error message; appears as a BadStatusLine: ''
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // 2017-02-27 00:06:57.786  FINE: ssh root@qe-blade-09.idmqe.lab.eng.bos.redhat.com subscription-manager register --username=stage_2013_data_center_test --password=redhat --org=7965071
        // 2017-02-27 00:07:00.089  FINE: Stdout: Registering to: subscription.rhsm.stage.redhat.com:443/subscription
        // 
        // 2017-02-27 00:07:00.090  FINE: Stderr: 'idCert'
        // 
        // 2017-02-27 00:07:00.090  FINE: ExitCode: 70
        // 2017-02-27 00:07:00.090  FINE: ssh root@qe-blade-09.idmqe.lab.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // 2017-02-27 00:07:00.273  WARNING: Last request from /var/log/rhsm/rhsm.log:
        // 2017-02-27 00:07:06,243 [DEBUG] subscription-manager:11832:MainThread @connection.py:490 - Making request: POST /subscription/consumers?owner=7965071
        // 2017-02-27 00:07:07,042 [INFO] subscription-manager:11832:MainThread @connection.py:525 - Response: status=202, request="POST /subscription/consumers?owner=7965071"
        // 2017-02-27 00:07:07,044 [DEBUG] subscription-manager:11832:MainThread @cache.py:110 - Wrote cache: /var/lib/rhsm/cache/installed_products.json
        // 2017-02-27 00:07:07,044 [ERROR] subscription-manager:11832:MainThread @managercli.py:177 - exception caught in subscription-manager
        // 2017-02-27 00:07:07,044 [ERROR] subscription-manager:11832:MainThread @managercli.py:178 - 'idCert'
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 85, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 76, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managercli.py", line 2768, in main
        // return CLI.main(self)
        // File "/usr/lib/python2.6/site-packages/subscription_manager/cli.py", line 160, in main
        // return cmd.main()
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managercli.py", line 537, in main
        // return_code = self._do_command()
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managercli.py", line 1159, in _do_command
        // consumer_info = self._persist_identity_cert(consumer)
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managercli.py", line 1225, in _persist_identity_cert
        // return managerlib.persist_consumer_cert(consumer)
        // File "/usr/lib/python2.6/site-packages/subscription_manager/managerlib.py", line 72, in persist_consumer_cert
        // consumer = identity.ConsumerIdentity(consumerinfo['idCert']['key'],
        // KeyError: 'idCert'
        issue = "KeyError: 'idCert'";
        if (getTracebackCommandResult.getStdout().contains(issue) || result.getStderr().contains(issue)) {
            // Bug 1393965 - Fail to register a system to stage
            String bugId = "1393965";
            // Bug 1393965 - Fail to register a system to stage
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
        // END OF WORKAROUND
        // TEMPORARY WORKAROUND FOR BUG
        // ssh root@gizmo.idmqe.lab.eng.bos.redhat.com subscription-manager register --username=stage_auto_testuser1 --password=redhat --force
        // Stdout:
        // Unregistering from: subscription.rhsm.stage.redhat.com:443/subscription
        // The system with UUID 66381640-b650-4f82-9889-b8e61102268e has been unregistered
        // All local data removed
        // Registering to: subscription.rhsm.stage.redhat.com:443/subscription
        // The system has been registered with ID: cafe07a8-94e0-49e3-86d4-b3ed0492de23
        // The registered system name is: gizmo.idmqe.lab.eng.bos.redhat.com
        // Stderr:
        // ExitCode: 70
        // ssh root@gizmo.idmqe.lab.eng.bos.redhat.com LINE_NUMBER=$(grep --line-number 'Making request:' /var/log/rhsm/rhsm.log | tail --lines=1 | cut --delimiter=':' --field=1); if [ -n "$LINE_NUMBER" ]; then tail -n +$LINE_NUMBER /var/log/rhsm/rhsm.log; fi;
        // Last request from /var/log/rhsm/rhsm.log:
        // 2018-01-25 20:42:30,494 [DEBUG] subscription-manager:5750:MainThread @connection.py:543 - Making request: PUT /subscription/consumers/cafe07a8-94e0-49e3-86d4-b3ed0492de23/packages
        // 2018-01-25 20:43:01,344 [INFO] subscription-manager:5750:MainThread @connection.py:586 - Response: status=500, request="PUT /subscription/consumers/cafe07a8-94e0-49e3-86d4-b3ed0492de23/packages"
        // 2018-01-25 20:43:01,346 [ERROR] subscription-manager:5750:MainThread @managercli.py:181 - exception caught in subscription-manager
        // 2018-01-25 20:43:01,347 [ERROR] subscription-manager:5750:MainThread @managercli.py:182 -
        // Traceback (most recent call last):
        // File "/usr/sbin/subscription-manager", line 96, in <module>
        // sys.exit(abs(main() or 0))
        // File "/usr/sbin/subscription-manager", line 87, in main
        // return managercli.ManagerCLI().main()
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/managercli.py", line 2622, in main
        // ret = CLI.main(self)
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/cli.py", line 181, in main
        // return cmd.main()
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/managercli.py", line 496, in main
        // return_code = self._do_command()
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/managercli.py", line 1148, in _do_command
        // profile_mgr.update_check(self.cp, consumer['uuid'], True)
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/cache.py", line 417, in update_check
        // return CacheManager.update_check(self, uep, consumer_uuid, force)
        // File "/usr/lib64/python2.7/site-packages/subscription_manager/cache.py", line 178, in update_check
        // raise re
        // RestlibException
        issue = "Response: status=500 from a PUT on /subscription/consumers/{UUID}/packages";
        if (SubscriptionManagerCLITestScript.doesStringContainMatches(getTracebackCommandResult.getStdout(), "request=\"PUT /subscription/consumers/[a-f,0-9,\\-]{36}/packages\"")) {
            // Bug 1539115 - encountering frequent 500 responses from stage candlepin from a PUT on /subscription/consumers/{UUID}/packages
            String bugId = "1539115";
            // Bug 1539115 - encountering frequent 500 responses from stage candlepin from a PUT on /subscription/consumers/{UUID}/packages
            boolean invokeWorkaroundWhileBugIsOpen = true;
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                throw new SkipException("Encountered a '" + issue + "' from the server and could not complete this test while bug '" + bugId + "' is open.");
            }
        }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // 2015-10-12 17:58:54,620 [DEBUG] subscription-manager:44349 @connection.py:523 - Making request: PUT /subscription/consumers/d8018dbc-7e66-4c0a-b322-9c28037fd8cf
    // 2015-10-12 17:58:55,094 [DEBUG] subscription-manager:44349 @connection.py:555 - Response: status=429
    // 2015-10-12 17:58:55,095 [ERROR] subscription-manager:44349 @managercli.py:1746 -
    // Traceback (most recent call last):
    // File "/usr/share/rhsm/subscription_manager/managercli.py", line 1744, in _do_command
    // facts.update_check(self.cp, identity.uuid, force=True)
    // File "/usr/share/rhsm/subscription_manager/cache.py", line 148, in update_check
    // raise re
    // RateLimitExceededException
    // Note: Candlepin-IT has introduced Throttling
    // The number of requests within a 30 min periord cannot exceed 60 calls for a specific consumer.
    // Here are the throttled API rules from aedwards...
    // #uri max_request lifetime_seconds methods
    // "^/subscription/consumers/([^/]+)/?$" := "60 1800 GET POST PUT",
    // "^/subscription/consumers/([^/]+)/entitlements?$" := "60 1800 GET POST",
    // "^/subscription/consumers/([^/]+)/entitlements/dry-run?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/events?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/guests?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/host?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/release?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/compliance?$" := "60 1800 GET",
    // "^/subscription/consumers/([^/]+)/certificates?$" := "60 1800 GET PUT",
    // "^/subscription/consumers/([^/]+)/certificates/serials?$" := "60 1800 GET",
    // "^/subscription/hypervisors?$" := "10 600 POST",
    // END OF WORKAROUND
    }
}
Also used : BigInteger(java.math.BigInteger) SSHCommandResult(com.redhat.qe.tools.SSHCommandResult) SkipException(org.testng.SkipException) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException)

Example 17 with BugzillaAPIException

use of com.redhat.qe.auto.bugzilla.BugzillaAPIException in project rhsm-qe by RedHatQE.

the class SubscriptionManagerTasks method register.

public SSHCommandResult register(String username, String password, String org, String environment, ConsumerType type, String name, String consumerid, Boolean autosubscribe, String servicelevel, String release, List<String> activationkeys, String serverurl, Boolean insecure, String baseurl, Boolean force, Boolean autoheal, String proxy, String proxyuser, String proxypassword, String noproxy) {
    boolean alreadyRegistered = this.currentlyRegisteredUsername == null ? false : true;
    String currentConsumerId = alreadyRegistered ? getCurrentConsumerId() : null;
    if (alreadyRegistered && currentConsumerId == null) {
        log.warning("AUTOMATION ERROR: Detected a bad state of the SubscriptionManagerTasks.  Conflicting variables: alreadyRegistered='" + alreadyRegistered + "' && currentConsumerId='" + currentConsumerId + "'.  Instruct the automator of this testware to troubleshoot the cause.  Proceeding with the assumption that currentConsumerId is correct.");
        alreadyRegistered = false;
        this.currentlyRegisteredType = null;
        this.currentlyRegisteredUsername = null;
        this.currentlyRegisteredPassword = null;
        this.currentlyRegisteredOrg = null;
    }
    String msg;
    SSHCommandResult sshCommandResult = register_(username, password, org, environment, type, name, consumerid, autosubscribe, servicelevel, release, activationkeys, serverurl, insecure, baseurl, force, autoheal, proxy, proxyuser, proxypassword, noproxy);
    // assert results when already registered
    if (alreadyRegistered) {
        if (force == null || !force) {
            // already registered while attempting to register without using force
            if (isPackageVersion("subscription-manager", ">=", "1.13.8-1")) {
                // post commit df95529a5edd0be456b3528b74344be283c4d258 bug 1119688
                Assert.assertEquals(sshCommandResult.getExitCode(), Integer.valueOf(64), "The exit code from the register command indicates we are already registered.");
                Assert.assertEquals(sshCommandResult.getStderr().trim(), "This system is already registered. Use --force to override");
                Assert.assertEquals(sshCommandResult.getStdout().trim(), "");
            } else {
                Assert.assertEquals(sshCommandResult.getExitCode(), Integer.valueOf(1), "The exit code from the register command indicates we are already registered.");
                Assert.assertEquals(sshCommandResult.getStdout().trim(), "This system is already registered. Use --force to override");
                Assert.assertEquals(sshCommandResult.getStderr().trim(), "");
            }
            return sshCommandResult;
        }
        if (force != null && force) {
            // 201705041325:42.376 - FINE: ExitCode: 0
            if (isPackageVersion("subscription-manager", ">=", "1.19.11-1")) {
                // commit 217c3863448478d06c5008694e327e048cc54f54 Bug 1443101: Provide feedback for force register
                String unregisterFromServer = getConfFileParameter(rhsmConfFile, "server", "hostname") + ":" + getConfFileParameter(rhsmConfFile, "server", "port") + getConfFileParameter(rhsmConfFile, "server", "prefix");
                // introduced by commit 217c3863448478d06c5008694e327e048cc54f54 Bug 1443101: Provide feedback for force register
                String unregisterFromMsg = String.format("Unregistering from: %s\nThe system with UUID %s has been unregistered\nAll local data removed", unregisterFromServer, currentConsumerId);
                if (isRhnSystemRegistered()) {
                    String unregisterFromMsgWithInteroperabilityWarning = msg_InteroperabilityWarning + "\n" + unregisterFromMsg;
                    Assert.assertTrue(sshCommandResult.getStdout().trim().startsWith(unregisterFromMsgWithInteroperabilityWarning), "Stdout from an attempt to register with force while already being registered (to both RHN Classic and RHSM) starts with message '" + unregisterFromMsgWithInteroperabilityWarning + "'.");
                } else {
                    Assert.assertTrue(sshCommandResult.getStdout().trim().startsWith(unregisterFromMsg), "Stdout from an attempt to register with force while already being registered starts with message '" + unregisterFromMsg + "'.");
                }
            }
        }
    }
    // assert results for a successful registration exit code
    if (isPackageVersion("subscription-manager", "<", "1.10")) {
        if (// https://bugzilla.redhat.com/show_bug.cgi?id=689608
        autosubscribe == null || !autosubscribe)
            Assert.assertEquals(sshCommandResult.getExitCode(), Integer.valueOf(0), "The exit code from the register command indicates a success.");
    } else {
        // Bug 689608 - Return error code when auto subscribing doesn't find any subscriptions
        if ((autosubscribe != null && Boolean.valueOf(autosubscribe)) || // Bug 1145835 - subscription-manager register --consumerid throws return code 1 even though it was successful	// commit 6f82c03f05804dcc28eb66d8126453f73c250488
        (consumerid != null && isPackageVersion("subscription-manager", "<", "1.13.4-1")) || (activationkeys != null && !activationkeys.isEmpty())) {
        // skip exit code assertion
        } else {
            Assert.assertEquals(sshCommandResult.getExitCode(), Integer.valueOf(0), "The exit code from the register command indicates a success.");
        }
    }
    // assert the heading for the current status of the installed products (applicable to register with autosubscribe|consumerid|activationkey)
    msg = "Installed Product Current Status:";
    if (isPackageVersion("subscription-manager", "<", "1.10")) {
        if (autosubscribe == null || !autosubscribe)
            Assert.assertFalse(sshCommandResult.getStdout().contains(msg), "register without autosubscribe should not show a list of the \"" + msg + "\".");
        else
            Assert.assertTrue(sshCommandResult.getStdout().contains(msg), "register with autosubscribe should show a list of the \"" + msg + "\".");
    } else {
        // bug 962545
        if (getCurrentProductCertFiles().isEmpty())
            msg = "No products installed.";
        if (isPackageVersion("subscription-manager", ">=", "1.13.4-1")) {
            // applicable to register with autosubscribe|activationkey - is no longer applicable to consumerid after bugs 1122001 bug 1145835
            if ((autosubscribe != null && Boolean.valueOf(autosubscribe)) || (activationkeys != null && !activationkeys.isEmpty())) {
                // TODO subscription-manager commit afd16e96d89ff38f74c60fd23613b67e27da17c5 for 1132981: Fixed exit code when registering system with no products installed is causing the following assert to fail.  Need resolution for comment https://bugzilla.redhat.com/show_bug.cgi?id=1132981#c10
                Assert.assertTrue(sshCommandResult.getStdout().contains(msg), "register with autosubscribe|activationkey should list \"" + msg + "\".");
            } else {
                Assert.assertTrue(!sshCommandResult.getStdout().contains(msg), "register without autosubscribe|activationkey should NOT list \"" + msg + "\".");
            }
        } else {
            // applicable to register with autosubscribe|consumerid|activationkey
            if ((autosubscribe != null && Boolean.valueOf(autosubscribe)) || (consumerid != null) || (activationkeys != null && !activationkeys.isEmpty())) {
                Assert.assertTrue(sshCommandResult.getStdout().contains(msg), "register with autosubscribe|consumerid|activationkey should list \"" + msg + "\".");
            } else {
                Assert.assertTrue(!sshCommandResult.getStdout().contains(msg), "register without autosubscribe|consumerid|activationkey should NOT list \"" + msg + "\".");
            }
        }
    }
    // https://bugzilla.redhat.com/show_bug.cgi?id=661130
    if (type == ConsumerType.person)
        name = username;
    // https://bugzilla.redhat.com/show_bug.cgi?id=669395
    if (name == null)
        name = this.hostname;
    // Assert.assertContainsMatch(sshCommandResult.getStdout().trim(), "[a-f,0-9,\\-]{36} "+name);	// applicable to RHEL61 and RHEL57. changed in RHEL62 due to feedback from mmccune https://engineering.redhat.com/trac/kalpana/wiki/SubscriptionManagerReview - jsefler 6/28/2011
    // Assert.assertContainsMatch(sshCommandResult.getStdout().trim(), "The system has been registered with id: [a-f,0-9,\\-]{36}");
    // msg = "The system has been registered with id: [a-f,0-9,\\-]{36}";	// msgid changed by bug 878634
    msg = "The system has been registered with ID: [a-f,0-9,\\-]{36}";
    Assert.assertTrue(Pattern.compile(".*" + msg + ".*", Pattern.DOTALL).matcher(sshCommandResult.getStdout()).find(), "Stdout from register contains a match to expected msg: " + msg);
    // assert that register with consumerId returns the expected uuid
    if (consumerid != null) {
        // Assert.assertEquals(sshCommandResult.getStdout().trim(), consumerId+" "+username, "register to an exiting consumer was a success");
        // Assert.assertContainsMatch(sshCommandResult.getStdout().trim(), "^"+consumerId, "register to an exiting consumer was a success");	// removed name from assert to account for https://bugzilla.redhat.com/show_bug.cgi?id=669395	// applicable to RHEL61 and RHEL57.
        // Assert.assertContainsMatch(sshCommandResult.getStdout().trim(), "The system has been registered with id: "+consumerid, "register to an exiting consumer was a success");	// removed name from assert to account for https://bugzilla.redhat.com/show_bug.cgi?id=669395
        // msg = "The system has been registered with id: "+consumerid;	// msgid changed by bug 878634
        msg = "The system has been registered with ID: " + consumerid;
        Assert.assertTrue(sshCommandResult.getStdout().contains(msg), "Stdout from register contains a match to expected msg: " + msg);
    }
    // assert certificate files are installed into /etc/pki/consumer
    Assert.assertTrue(RemoteFileTasks.testExists(sshCommandRunner, this.consumerKeyFile()), "Consumer key file '" + this.consumerKeyFile() + "' must exist after register.");
    Assert.assertTrue(RemoteFileTasks.testExists(sshCommandRunner, this.consumerCertFile()), "Consumer cert file '" + this.consumerCertFile() + "' must exist after register.");
    // TEMPORARY WORKAROUND FOR BUG: https://bugzilla.redhat.com/show_bug.cgi?id=639417 - jsefler 10/1/2010
    // Status: 	CLOSED CURRENTRELEASE
    boolean invokeWorkaroundWhileBugIsOpen = false;
    String bugId = "639417";
    try {
        if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
            log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
            SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
        } else {
            invokeWorkaroundWhileBugIsOpen = false;
        }
    } catch (BugzillaAPIException be) {
    /* ignore exception */
    } catch (RuntimeException re) {
    /* ignore exception */
    }
    if (invokeWorkaroundWhileBugIsOpen) {
        restart_rhsmcertd(Integer.valueOf(getConfFileParameter(rhsmConfFile, "certFrequency")), null, null);
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR Bug 797243 - manual changes to redhat.repo are too sticky
    invokeWorkaroundWhileBugIsOpen = true;
    bugId = "797243";
    try {
        if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
            log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
            SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
        } else {
            invokeWorkaroundWhileBugIsOpen = false;
        }
    } catch (BugzillaAPIException be) {
    /* ignore exception */
    } catch (RuntimeException re) {
    /* ignore exception */
    }
    if (invokeWorkaroundWhileBugIsOpen) {
        log.warning("Triggering a yum transaction to insure the redhat.repo file is wiped clean");
        // --disableplugin=rhnplugin helps avoid: up2date_client.up2dateErrors.AbuseError
        sshCommandRunner.runCommandAndWait("yum repolist --disableplugin=rhnplugin");
    }
    // from the register command
    return sshCommandResult;
}
Also used : SSHCommandResult(com.redhat.qe.tools.SSHCommandResult) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException)

Example 18 with BugzillaAPIException

use of com.redhat.qe.auto.bugzilla.BugzillaAPIException in project rhsm-qe by RedHatQE.

the class SubscriptionManagerTasks method assertNoAvailableSubscriptionPoolsToList.

public void assertNoAvailableSubscriptionPoolsToList(boolean ignoreMuliEntitlementSubscriptionPools, String assertMsg) {
    boolean invokeWorkaroundWhileBugIsOpen = true;
    // TEMPORARY WORKAROUND FOR BUG
    // true;	// Status: CLOSED ERRATA	// Bug 613635 - “connection.UEPConnection instance “ displays while availability check
    invokeWorkaroundWhileBugIsOpen = false;
    try {
        String bugId = "613635";
        if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
            log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
            SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
        } else {
            invokeWorkaroundWhileBugIsOpen = false;
        }
    } catch (BugzillaAPIException be) {
    /* ignore exception */
    } catch (RuntimeException re) {
    /* ignore exception */
    }
    if (invokeWorkaroundWhileBugIsOpen) {
        Assert.assertContainsMatch(listAvailableSubscriptionPools().getStdout(), "^No available subscription pools to list$", assertMsg);
        return;
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // true;	// Status: CLOSED ERRATA	// Bug 622839 - extraneous user hash code appears in stdout after executing list --available
    invokeWorkaroundWhileBugIsOpen = false;
    try {
        String bugId = "622839";
        if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
            log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
            SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
        } else {
            invokeWorkaroundWhileBugIsOpen = false;
        }
    } catch (BugzillaAPIException be) {
    /* ignore exception */
    } catch (RuntimeException re) {
    /* ignore exception */
    }
    if (invokeWorkaroundWhileBugIsOpen) {
        Assert.assertContainsMatch(listAvailableSubscriptionPools().getStdout(), "^No available subscription pools to list$", assertMsg);
        return;
    }
    // END OF WORKAROUND
    // TEMPORARY WORKAROUND FOR BUG
    // true;	// Status: CLOSED DUPLICATE of bug 623481	// Bug 623657 - extraneous self.conn output appears in stdout after executing list --available
    invokeWorkaroundWhileBugIsOpen = false;
    try {
        String bugId = "623657";
        if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
            log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
            SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
        } else {
            invokeWorkaroundWhileBugIsOpen = false;
        }
    } catch (BugzillaAPIException be) {
    /* ignore exception */
    } catch (RuntimeException re) {
    /* ignore exception */
    }
    if (invokeWorkaroundWhileBugIsOpen) {
        Assert.assertContainsMatch(listAvailableSubscriptionPools().getStdout(), "^No available subscription pools to list$", assertMsg);
        return;
    }
    // END OF WORKAROUND
    // determine which available pools are multi-entitlement pools
    List<SubscriptionPool> poolsAvailableExcludingMuliEntitlement = new ArrayList<SubscriptionPool>();
    List<SubscriptionPool> poolsAvailable = getCurrentlyAvailableSubscriptionPools();
    for (SubscriptionPool pool : poolsAvailable) {
        try {
            String authenticator = this.currentlyRegisteredUsername != null ? this.currentlyRegisteredUsername : candlepinAdminUsername;
            String password = this.currentlyRegisteredPassword != null ? this.currentlyRegisteredPassword : candlepinAdminPassword;
            if (!CandlepinTasks.isPoolProductMultiEntitlement(authenticator, password, candlepinUrl, pool.poolId)) {
                poolsAvailableExcludingMuliEntitlement.add(pool);
            }
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail(e.getMessage());
        }
    }
    // assert
    if (ignoreMuliEntitlementSubscriptionPools) {
        Assert.assertEquals(poolsAvailableExcludingMuliEntitlement.size(), 0, assertMsg + " (muti-entitlement pools were excluded.)");
    } else {
        Assert.assertEquals(poolsAvailable.size(), 0, assertMsg + " (muti-entitlement pools were excluded.)");
        Assert.assertEquals(listAvailableSubscriptionPools().getStdout().trim(), "No available subscription pools to list", assertMsg);
    }
}
Also used : ArrayList(java.util.ArrayList) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException) SubscriptionPool(rhsm.data.SubscriptionPool) SkipException(org.testng.SkipException) JSONException(org.json.JSONException) IOException(java.io.IOException) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException)

Example 19 with BugzillaAPIException

use of com.redhat.qe.auto.bugzilla.BugzillaAPIException in project rhsm-qe by RedHatQE.

the class SubscriptionManagerTasks method subscribeToSubscriptionPool.

/**
 * subscribe to the given SubscriptionPool (assumes pool came from the list of available pools)
 * @return the newly installed EntitlementCert file to the newly consumed ProductSubscriptions
 */
public File subscribeToSubscriptionPool(SubscriptionPool pool, String quantity, String authenticator, String password, String serverUrl) {
    List<ProductSubscription> beforeProductSubscriptions = getCurrentlyConsumedProductSubscriptions();
    List<File> beforeEntitlementCertFiles = getCurrentEntitlementCertFiles();
    log.info("Subscribing to subscription pool: " + pool);
    SSHCommandResult sshCommandResult = subscribe(null, null, pool.poolId, null, null, quantity, null, null, null, null, null, null, null);
    // is this pool multi-entitleable?
    /* This information is now in the SubscriptionPool itself
		boolean isPoolMultiEntitlement = false;
		try {
			isPoolMultiEntitlement = CandlepinTasks.isPoolProductMultiEntitlement(this.currentlyRegisteredUsername,this.currentlyRegisteredPassword,SubscriptionManagerBaseTestScript.sm_serverUrl,pool.poolId);
		} catch (Exception e) {
			e.printStackTrace();
			Assert.fail(e.getMessage());
		}
		*/
    // get the pool's product "arch" attribute that this subscription pool supports
    String poolProductAttributeArch = "";
    List<String> poolProductAttributeArches = new ArrayList<String>();
    if (authenticator != null && password != null && serverUrl != null) {
        try {
            poolProductAttributeArch = CandlepinTasks.getPoolProductAttributeValue(authenticator, password, serverUrl, pool.poolId, "arch");
            if (poolProductAttributeArch != null && !poolProductAttributeArch.trim().isEmpty()) {
                // Note: the arch attribute can be a comma separated list of values
                poolProductAttributeArches.addAll(Arrays.asList(poolProductAttributeArch.trim().split(" *, *")));
                // Note: x86 is a general arch to cover all 32-bit intel microprocessors
                if (poolProductAttributeArches.contains("x86"))
                    poolProductAttributeArches.addAll(Arrays.asList("i386", "i486", "i586", "i686"));
            // if (productSupportedArches.contains("ALL")) productSupportedArches.add(arch);
            }
        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail(e.getMessage());
        }
    }
    // assert that the remaining SubscriptionPools does NOT contain the pool just subscribed to (unless it is multi-entitleable)
    List<SubscriptionPool> afterSubscriptionPools = getCurrentlyAvailableSubscriptionPools();
    if (pool.subscriptionType != null && pool.subscriptionType.equals("Other")) {
        Assert.fail("Encountered a subscription pool of type '" + pool.subscriptionType + "'.  Do not know how to assert the remaining availability of this pool after subscribing to it: " + pool);
    } else if (pool.multiEntitlement == null && pool.subscriptionType != null && pool.subscriptionType.isEmpty()) {
        log.warning("Encountered a pool with an empty value for subscriptionType (indicative of an older candlepin server): " + pool);
        log.warning("Skipping assertion of the pool's expected availability after having subscribed to it.");
    } else if (!pool.quantity.equalsIgnoreCase("unlimited") && Integer.valueOf(pool.quantity) <= 1) {
        Assert.assertTrue(!afterSubscriptionPools.contains(pool), "When the final quantity from the pool was consumed, the remaining available subscription pools no longer contains the just subscribed to pool: " + pool);
    } else if (pool.multiEntitlement != null && !pool.multiEntitlement) {
        Assert.assertTrue(!afterSubscriptionPools.contains(pool), "When the pool is not multi-entitleable, the remaining available subscription pools no longer contains the just subscribed to pool: " + pool);
    } else if (pool.subscriptionType != null && (!pool.subscriptionType.equals("Stackable") && !pool.subscriptionType.equals("Multi-Entitleable") && !pool.subscriptionType.equals("Instance Based") && !pool.subscriptionType.equals("Stackable (Temporary)") && !pool.subscriptionType.equals("Multi-Entitleable (Temporary)") && !pool.subscriptionType.equals("Instance Based (Temporary)"))) {
        // see https://bugzilla.redhat.com/show_bug.cgi?id=1029968#c2
        Assert.assertTrue(!afterSubscriptionPools.contains(pool), "When the pool is not multi-entitleable (not Stackable && not Multi-Entitleable && not Instance Based), the remaining available subscription pools no longer contains the just subscribed to pool: " + pool);
    } else if (!poolProductAttributeArches.isEmpty() && !poolProductAttributeArches.contains("ALL") && !poolProductAttributeArches.contains(arch)) {
        Assert.assertTrue(!afterSubscriptionPools.contains(pool), "When the pools product attribute arch '" + poolProductAttributeArch + "' does not support this system arch '" + arch + "', the remaining available subscription pools should never contain the just subscribed to pool: " + pool);
    } else {
        Assert.assertTrue(afterSubscriptionPools.contains(pool), // TODO fix the assertions for "if this fails"
        "When the pool is multi-entitleable, the remaining available subscription pools still contains the just subscribed to pool: " + pool + " (TODO: if this fails, then we likely attached the final entitlements from the pool)");
    }
    // assert that the remaining SubscriptionPools do NOT contain the same productId just subscribed to
    // log.warning("We will no longer assert that the remaining available pools do not contain the same productId ("+pool.productId+") as the pool that was just subscribed.  Reference: https://bugzilla.redhat.com/show_bug.cgi?id=663455");
    /*
		for (SubscriptionPool afterSubscriptionPool : afterSubscriptionPools) {
			Assert.assertTrue(!afterSubscriptionPool.productId.equals(pool.productId),
					"This remaining available pool "+afterSubscriptionPool+" does NOT contain the same productId ("+pool.productId+") after subscribing to pool: "+pool);
		}
		*/
    // is this a personal subpool?
    String poolProductId = pool.productId;
    boolean isSubpool = false;
    try {
        JSONArray personSubscriptionPoolProductData;
        // personSubscriptionPoolProductData = new JSONArray(System.getProperty("sm.person.subscriptionPoolProductData", "<>").replaceAll("<", "[").replaceAll(">", "]")); // hudson parameters use <> instead of []
        // hudson JSONArray parameters get surrounded with double quotes that need to be stripped
        personSubscriptionPoolProductData = new JSONArray(SubscriptionManagerBaseTestScript.getProperty("sm.person.subscriptionPoolProductData", "[]").replaceFirst("^\"", "").replaceFirst("\"$", "").replaceAll("<", "[").replaceAll(">", "]"));
        for (int j = 0; j < personSubscriptionPoolProductData.length(); j++) {
            JSONObject poolProductDataAsJSONObject = (JSONObject) personSubscriptionPoolProductData.get(j);
            String personProductId = poolProductDataAsJSONObject.getString("personProductId");
            JSONObject subpoolProductDataAsJSONObject = poolProductDataAsJSONObject.getJSONObject("subPoolProductData");
            String systemProductId = subpoolProductDataAsJSONObject.getString("systemProductId");
            if (poolProductId.equals(systemProductId)) {
                // special case when pool's productId is really a personal subpool
                poolProductId = personProductId;
                isSubpool = true;
                break;
            }
        }
    } catch (JSONException e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
    // figure out which entitlement cert file has been newly installed into /etc/pki/entitlement after attempting to subscribe to pool
    /* OLD - THIS ALGORITHM BREAKS DOWN WHEN MODIFIER ENTITLEMENTS ARE IN PLAY
		File newCertFile = null;
		List<File> afterEntitlementCertFiles = getCurrentEntitlementCertFiles();
		for (File file : afterEntitlementCertFiles) {
			if (!beforeEntitlementCertFiles.contains(file)) {
				newCertFile = file; break;
			}
		}
		*/
    /* VALID BUT INEFFICIENT
		List<File> afterEntitlementCertFiles = getCurrentEntitlementCertFiles();
		File newCertFile = null;
		Map<BigInteger, SubscriptionPool> map = new HashMap<BigInteger, SubscriptionPool>();
		try {
			map = getCurrentSerialMapToSubscriptionPools(this.currentAuthenticator,this.currentAuthenticatorPassword);
		} catch (Exception e) {
			e.printStackTrace();
			Assert.fail(e.getMessage());
		}
		for (BigInteger serial: map.keySet()) {
			if (map.get(serial).poolId.equals(pool.poolId)) {
				newCertFile = new File(this.entitlementCertDir+"/"+serial+".pem");
				break;
			}
		}
		*/
    // NOTE: this block of code is somewhat duplicated in getEntitlementCertCorrespondingToSubscribedPool(...)
    File newCertFile = null;
    List<File> afterEntitlementCertFiles = getCurrentEntitlementCertFiles("-t");
    if (authenticator != null && password != null && serverUrl != null) {
        for (File entitlementCertFile : afterEntitlementCertFiles) {
            if (!beforeEntitlementCertFiles.contains(entitlementCertFile)) {
                EntitlementCert entitlementCert = getEntitlementCertFromEntitlementCertFile(entitlementCertFile);
                try {
                    // JSONObject jsonEntitlement = CandlepinTasks.getEntitlementUsingRESTfulAPI(authenticator,password,serverUrl,entitlementCert.id);	// is throwing a 500 in stage, but only for qa@redhat.com credentials - I don't know why
                    JSONObject jsonEntitlement = CandlepinTasks.getEntitlementUsingRESTfulAPI(currentlyRegisteredUsername, currentlyRegisteredPassword, serverUrl, entitlementCert.id);
                    JSONObject jsonPool = new JSONObject(CandlepinTasks.getResourceUsingRESTfulAPI(authenticator, password, serverUrl, jsonEntitlement.getJSONObject("pool").getString("href")));
                    if (jsonPool.getString("id").equals(pool.poolId)) {
                        newCertFile = entitlementCertFile;
                        break;
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    Assert.fail(e.getMessage());
                }
            }
        }
    }
    // when the pool is already subscribed to...
    if (sshCommandResult.getStdout().startsWith("This consumer is already subscribed")) {
        // assert that NO new entitlement cert file has been installed in /etc/pki/entitlement
        /*Assert.assertNull(newCertFile,
					"A new entitlement certificate has NOT been installed after attempting to subscribe to an already subscribed to pool: "+pool);
			*/
        Assert.assertEquals(beforeEntitlementCertFiles.size(), afterEntitlementCertFiles.size(), "The existing entitlement certificate count remains unchanged after attempting to subscribe to an already subscribed to pool: " + pool);
        // find the existing entitlement cert file corresponding to the already subscribed pool
        /* ALREADY FOUND USING ALGORITHM ABOVE 
			EntitlementCert entitlementCert = null;
			for (File thisEntitlementCertFile : getCurrentEntitlementCertFiles()) {
				EntitlementCert thisEntitlementCert = getEntitlementCertFromEntitlementCertFile(thisEntitlementCertFile);
				if (thisEntitlementCert.orderNamespace.productId.equals(poolProductId)) {
					entitlementCert = thisEntitlementCert;
					break;
				}
			}
			Assert.assertNotNull(entitlementCert, isSubpool?
					"Found an already existing Entitlement Cert whose personal productId matches the system productId from the subscription pool: "+pool:
					"Found an already existing Entitlement Cert whose productId matches the productId from the subscription pool: "+pool);
			newCertFile = getEntitlementCertFileFromEntitlementCert(entitlementCert); // not really new, just already existing
			*/
        // assert that consumed ProductSubscriptions has NOT changed
        List<ProductSubscription> afterProductSubscriptions = getCurrentlyConsumedProductSubscriptions();
        Assert.assertTrue(afterProductSubscriptions.size() == beforeProductSubscriptions.size() && afterProductSubscriptions.size() > 0, "The list of currently consumed product subscriptions has not changed (from " + beforeProductSubscriptions.size() + " to " + afterProductSubscriptions.size() + ") since the productId of the pool we are trying to subscribe to is already consumed.");
    // when no free entitlements exist...		// No entitlements are available from the pool with id '8a90f8143611c33f013611c4797b0456'.	// No subscriptions are available from the pool with id '8a90f8303c98703a013c98715ca80494'.  Bug 876758
    } else if (sshCommandResult.getStdout().startsWith("No entitlements are available") || sshCommandResult.getStdout().startsWith("No subscriptions are available")) {
        // assert that the depleted pool Quantity is zero
        SubscriptionPool depletedPool = SubscriptionPool.findFirstInstanceWithMatchingFieldFromList("poolId", pool.poolId, getCurrentlyAllAvailableSubscriptionPools());
        /* behavior changed on list --all --available  (3/4/2011)
			Assert.assertNotNull(depletedPool,
					"Found the depleted pool amongst --all --available after having consumed all of its available entitlements: ");
			*/
        Assert.assertNull(depletedPool, "Should no longer find the depleted pool amongst --all --available after having consumed all of its available entitlements: ");
        // "Asserting the pool's quantity after having consumed all of its available entitlements is zero.");
        if (authenticator != null && password != null && serverUrl != null) {
            JSONObject jsonPool = null;
            int consumed = 0;
            int quantityAvailable = Integer.valueOf(pool.quantity);
            try {
                jsonPool = new JSONObject(CandlepinTasks.getResourceUsingRESTfulAPI(authenticator, password, serverUrl, "/pools/" + pool.poolId));
                consumed = jsonPool.getInt("consumed");
                quantityAvailable = jsonPool.getInt("quantity");
            } catch (Exception e) {
                e.printStackTrace();
                Assert.fail(e.getMessage());
            }
            Assert.assertEquals(consumed, quantityAvailable, "Asserting the pool's consumed attribute equals it's total quantity after having consumed all of its available entitlements.");
        }
        // assert that NO new entitlement cert file has been installed in /etc/pki/entitlement
        Assert.assertNull(newCertFile, "A new entitlement certificate has NOT been installed after attempting to subscribe to depleted pool: " + depletedPool);
        Assert.assertEquals(beforeEntitlementCertFiles.size(), afterEntitlementCertFiles.size(), "The existing entitlement certificate count remains unchanged after attempting to subscribe to depleted pool: " + depletedPool);
    // otherwise, the pool is NOT already subscribe to...
    } else {
        // assert that only ONE new entitlement cert file has been installed in /etc/pki/entitlement
        // https://bugzilla.redhat.com/show_bug.cgi?id=640338
        Assert.assertTrue(afterEntitlementCertFiles.size() == beforeEntitlementCertFiles.size() + 1, "Only ONE new entitlement certificate has been installed (count was '" + beforeEntitlementCertFiles.size() + "'; is now '" + afterEntitlementCertFiles.size() + "') after subscribing to pool: " + pool);
        if (authenticator != null && password != null && serverUrl != null) {
            // assert the new entitlement cert file has been installed in /etc/pki/entitlement
            Assert.assertNotNull(newCertFile, "A new entitlement certificate has been installed after subscribing to pool: " + pool);
            log.info("The new entitlement certificate file is: " + newCertFile);
            // assert that the productId from the pool matches the entitlement productId
            // TEMPORARY WORKAROUND FOR BUG: https://bugzilla.redhat.com/show_bug.cgi?id=650278 - jsefler 11/05/2010
            // TEMPORARY WORKAROUND FOR BUG: https://bugzilla.redhat.com/show_bug.cgi?id=806986 - jsefler 06/28/2012
            boolean invokeWorkaroundWhileBugIsOpen = true;
            String bugId1 = "650278";
            String bugId2 = "806986";
            try {
                if (invokeWorkaroundWhileBugIsOpen && (BzChecker.getInstance().isBugOpen(bugId1) || BzChecker.getInstance().isBugOpen(bugId2))) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId1).toString() + " Bugzilla " + bugId1 + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId1 + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId1);
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId2).toString() + " Bugzilla " + bugId2 + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId2 + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId2);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                log.warning("Skipping assert that the productId from the pool matches the entitlement productId");
            } else {
                // END OF WORKAROUND
                EntitlementCert entitlementCert = getEntitlementCertFromEntitlementCertFile(newCertFile);
                File newCertKeyFile = getEntitlementCertKeyFileFromEntitlementCert(entitlementCert);
                Assert.assertEquals(entitlementCert.orderNamespace.productId, poolProductId, isSubpool ? "New EntitlementCert productId '" + entitlementCert.orderNamespace.productId + "' matches originating Personal SubscriptionPool productId '" + poolProductId + "' after subscribing to the subpool." : "New EntitlementCert productId '" + entitlementCert.orderNamespace.productId + "' matches originating SubscriptionPool productId '" + poolProductId + "' after subscribing to the pool.");
                Assert.assertTrue(RemoteFileTasks.testExists(sshCommandRunner, newCertFile.getPath()), "New EntitlementCert file exists after subscribing to SubscriptionPool '" + pool.poolId + "'.");
                Assert.assertTrue(RemoteFileTasks.testExists(sshCommandRunner, newCertKeyFile.getPath()), "New EntitlementCert key file exists after subscribing to SubscriptionPool '" + pool.poolId + "'.");
            }
        }
        // assert that consumed ProductSubscriptions has NOT decreased
        List<ProductSubscription> afterProductSubscriptions = getCurrentlyConsumedProductSubscriptions();
        // this assertion was valid prior to bug Bug 801187 - collapse list of provided products for subscription-manager list --consumed
        // Assert.assertTrue(afterProductSubscriptions.size() >= beforeProductSubscriptions.size() && afterProductSubscriptions.size() > 0,
        // "The list of currently consumed product subscriptions has increased (from "+beforeProductSubscriptions.size()+" to "+afterProductSubscriptions.size()+"), or has remained the same after subscribing (using poolID="+pool.poolId+") to pool: "+pool+"  Note: The list of consumed product subscriptions can remain the same when all the products from this subscription pool are a subset of those from a previously subscribed pool.");
        Assert.assertTrue(afterProductSubscriptions.size() == beforeProductSubscriptions.size() + 1, "The list of currently consumed product subscriptions has increased by 1 (from " + beforeProductSubscriptions.size() + " to " + afterProductSubscriptions.size() + "), after subscribing to pool: " + pool);
    }
    return newCertFile;
}
Also used : EntitlementCert(rhsm.data.EntitlementCert) ProductSubscription(rhsm.data.ProductSubscription) ArrayList(java.util.ArrayList) JSONArray(org.json.JSONArray) JSONException(org.json.JSONException) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException) SkipException(org.testng.SkipException) JSONException(org.json.JSONException) IOException(java.io.IOException) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException) JSONObject(org.json.JSONObject) SSHCommandResult(com.redhat.qe.tools.SSHCommandResult) ArrayList(java.util.ArrayList) List(java.util.List) File(java.io.File) SubscriptionPool(rhsm.data.SubscriptionPool)

Example 20 with BugzillaAPIException

use of com.redhat.qe.auto.bugzilla.BugzillaAPIException in project rhsm-qe by RedHatQE.

the class ActivationKeyTests method testMultiClientAttemptToDeadLockOnRegisterWithActivationKeys.

@// update=true,	// uncomment to make TestDefinition changes update Polarion testcases through the polarize testcase importer
TestDefinition(projectID = { Project.RHEL6, Project.RedHatEnterpriseLinux7 }, testCaseID = { "RHEL6-21791", "RHEL7-51608" }, level = DefTypes.Level.COMPONENT, component = "subscription-manager", testtype = @TestType(testtype = DefTypes.TestTypes.FUNCTIONAL, subtype1 = DefTypes.Subtypes.RELIABILITY, subtype2 = DefTypes.Subtypes.EMPTY), posneg = PosNeg.NEGATIVE, importance = DefTypes.Importance.HIGH, automation = DefTypes.Automation.AUTOMATED, tags = "Tier3")
@Test(description = "attempt to register two different consumers with multiple activation keys (in reverse order) containing many subscriptions", groups = { "Tier3Tests", "blockedByBug-1095939" })
public // @ImplementsNitrateTest(caseId=)
void testMultiClientAttemptToDeadLockOnRegisterWithActivationKeys() throws JSONException, Exception {
    if (client2tasks == null)
        throw new SkipException("This multi-client test requires a second client.");
    // register two clients
    String client1ConsumerId = client1tasks.getCurrentConsumerId(client1tasks.register(sm_client1Username, sm_client1Password, sm_client1Org, null, null, null, null, null, null, null, (List<String>) null, null, null, null, true, false, null, null, null, null));
    String client2ConsumerId = client2tasks.getCurrentConsumerId(client2tasks.register(sm_client2Username, sm_client2Password, sm_client2Org, null, null, null, null, null, null, null, (List<String>) null, null, null, null, true, false, null, null, null, null));
    String client1OwnerKey = CandlepinTasks.getOwnerKeyOfConsumerId(sm_client1Username, sm_client1Password, sm_serverUrl, client1ConsumerId);
    String client2OwnerKey = CandlepinTasks.getOwnerKeyOfConsumerId(sm_client2Username, sm_client2Password, sm_serverUrl, client2ConsumerId);
    if (!client1OwnerKey.equals(client2OwnerKey))
        throw new SkipException("This multi-client test requires that both client registerers belong to the same owner. (client1: username=" + sm_client1Username + " ownerkey=" + client1OwnerKey + ") (client2: username=" + sm_client2Username + " ownerkey=" + client2OwnerKey + ")");
    // get all of the pools belonging to ownerKey
    JSONArray jsonPools = new JSONArray(CandlepinTasks.getResourceUsingRESTfulAPI(sm_clientUsername, sm_clientPassword, sm_serverUrl, "/owners/" + client1OwnerKey + "/pools?listall=true"));
    if (!(jsonPools.length() > 1))
        throw new SkipException("This test requires more than one pool for org '" + sm_client1Org + "'.");
    jsonPools = clienttasks.workaroundForBug1040101(jsonPools);
    // create two activation key each containing the same pools (added in reverse order)
    long currentTimeMillis = System.currentTimeMillis();
    final String activationKeyName1 = String.format("ActivationKey1_%sWithMultiplePoolsForOrgKey_%s", currentTimeMillis, client1OwnerKey);
    final String activationKeyName2 = String.format("ActivationKey2_%sWithMultiplePoolsForOrgKey_%s", currentTimeMillis, client2OwnerKey);
    Map<String, String> mapActivationKeyRequest1 = new HashMap<String, String>() {

        {
            put("name", activationKeyName1);
        }
    };
    Map<String, String> mapActivationKeyRequest2 = new HashMap<String, String>() {

        {
            put("name", activationKeyName2);
        }
    };
    JSONObject jsonActivationKey1 = new JSONObject(CandlepinTasks.postResourceUsingRESTfulAPI(sm_serverAdminUsername, sm_serverAdminPassword, sm_serverUrl, "/owners/" + client1OwnerKey + "/activation_keys", new JSONObject(mapActivationKeyRequest1).toString()));
    JSONObject jsonActivationKey2 = new JSONObject(CandlepinTasks.postResourceUsingRESTfulAPI(sm_serverAdminUsername, sm_serverAdminPassword, sm_serverUrl, "/owners/" + client2OwnerKey + "/activation_keys", new JSONObject(mapActivationKeyRequest2).toString()));
    // process each of the pools to choosing friendly ones to add to the activation keys
    List<String> poolIds = new ArrayList<String>();
    boolean isSystemVirtual = Boolean.valueOf(clienttasks.getFactValue("virt.is_guest"));
    for (int i = 0; i < jsonPools.length(); i++) {
        JSONObject jsonPool = (JSONObject) jsonPools.get(i);
        // for the purpose of this test, skip pools with no available entitlements (consumed>=quantity) (quantity=-1 is unlimited)
        if (jsonPool.getInt("quantity") > 0 && jsonPool.getInt("consumed") >= jsonPool.getInt("quantity"))
            continue;
        // for the purpose of this test, skip pools that do not have at least 4 available entitlements
        if (jsonPool.getInt("quantity") >= 0 && (jsonPool.getInt("quantity") - jsonPool.getInt("consumed")) < 4)
            continue;
        // for the purpose of this test, skip non-system pools otherwise the register will fail with "Consumers of this type are not allowed to subscribe to the pool with id '8a90f8c631ab7ccc0131ab7e46ca0619'."
        if (!CandlepinTasks.isPoolProductConsumableByConsumerType(sm_clientUsername, sm_clientPassword, sm_serverUrl, jsonPool.getString("id"), ConsumerType.system))
            continue;
        // for the purpose of this test, skip physical_only pools when system is virtual otherwise the register will fail with "Pool is restricted to physical systems: '8a9086d344549b0c0144549bf9ae0dd4'."
        if (isSystemVirtual && CandlepinTasks.isPoolRestrictedToPhysicalSystems(sm_clientUsername, sm_clientPassword, sm_serverUrl, jsonPool.getString("id")))
            continue;
        // for the purpose of this test, skip virt_only pools when system is physical otherwise the register will fail with "Pool is restricted to virtual guests: '8a9086d344549b0c0144549bf9ae0dd4'."
        if (!isSystemVirtual && CandlepinTasks.isPoolRestrictedToVirtualSystems(sm_clientUsername, sm_clientPassword, sm_serverUrl, jsonPool.getString("id")))
            continue;
        // for the purpose of this test, skip virt_only derived_pool when server is standalone otherwise the register will fail with "Unable to entitle consumer to the pool with id '8a90f85733d86b130133d88c09410e5e'.: virt.guest.host.does.not.match.pool.owner"
        if (servertasks.statusStandalone) {
            String pool_derived = CandlepinTasks.getPoolAttributeValue(jsonPool, "pool_derived");
            String virt_only = CandlepinTasks.getPoolAttributeValue(jsonPool, "virt_only");
            if (pool_derived != null && virt_only != null && Boolean.valueOf(pool_derived) && Boolean.valueOf(virt_only)) {
                continue;
            }
        }
        // for the purpose of this test, skip non-multi-entitlement pools otherwise the multi-activation key register will fail with "This unit has already had the subscription matching pool ID '8a9087e345a9f5f90145b36429073724' attached."
        if (!CandlepinTasks.isPoolProductMultiEntitlement(sm_clientUsername, sm_clientPassword, sm_serverUrl, jsonPool.getString("id")))
            continue;
        poolIds.add(jsonPool.getString("id"));
    }
    // add each of the pools to jsonActivationKey1
    for (int i = 0; i < poolIds.size(); i++) {
        JSONObject jsonResult1 = new JSONObject(CandlepinTasks.postResourceUsingRESTfulAPI(sm_client1Username, sm_client1Password, sm_serverUrl, "/activation_keys/" + jsonActivationKey1.getString("id") + "/pools/" + poolIds.get(i), /* + (addQuantity==null?"":"?quantity="+addQuantity)*/
        null));
        // if (clienttasks.isVersion(servertasks.statusVersion, ">", "0.9.30-1")) the POST now returns the jsonActivationKey and formerly returned the jsonPoolAddedToActivationKey	// candlepin commit 82b9af5dc2c63b58447366e680fcf6f156c6049f
        if (jsonResult1.has("displayMessage")) {
            Assert.fail("Failed to add pool '" + poolIds.get(i) + "' to activation key '" + jsonActivationKey1.getString("id") + "'.  DisplayMessage: " + jsonResult1.getString("displayMessage"));
        }
    }
    // add each of the pools to jsonActivationKey2 (in reverse order)
    for (int i = poolIds.size() - 1; i >= 0; i--) {
        JSONObject jsonResult2 = new JSONObject(CandlepinTasks.postResourceUsingRESTfulAPI(sm_client2Username, sm_client2Password, sm_serverUrl, "/activation_keys/" + jsonActivationKey2.getString("id") + "/pools/" + poolIds.get(i), /* + (addQuantity==null?"":"?quantity="+addQuantity)*/
        null));
        // if (clienttasks.isVersion(servertasks.statusVersion, ">", "0.9.30-1")) the POST now returns the jsonActivationKey and formerly returned the jsonPoolAddedToActivationKey	// candlepin commit 82b9af5dc2c63b58447366e680fcf6f156c6049f
        if (jsonResult2.has("displayMessage")) {
            Assert.fail("Failed to add pool '" + poolIds.get(i) + "' to activation key '" + jsonActivationKey2.getString("id") + "'.  DisplayMessage: " + jsonResult2.getString("displayMessage"));
        }
    }
    // attempt this test more than once
    for (int attempt = 1; attempt < 5; attempt++) {
        client1tasks.unregister(null, null, null, null);
        client2tasks.unregister(null, null, null, null);
        // register each client simultaneously using the activation keys (one in reverse order of the other)
        log.info("Simultaneously attempting to register with activation keys on '" + client1tasks.hostname + "' and '" + client2tasks.hostname + "'...");
        client1.runCommand(/*AndWait*/
        client1tasks.registerCommand(null, null, client1OwnerKey, null, null, null, null, null, null, null, new ArrayList<String>() {

            {
                add(activationKeyName1);
                add(activationKeyName2);
            }
        }, null, null, null, null, null, null, null, null, null), TestRecords.action());
        client2.runCommand(/*AndWait*/
        client2tasks.registerCommand(null, null, client2OwnerKey, null, null, null, null, null, null, null, new ArrayList<String>() {

            {
                add(activationKeyName2);
                add(activationKeyName1);
            }
        }, null, null, null, null, null, null, null, null, null), TestRecords.action());
        // timeout after 10 min
        client1.waitForWithTimeout(new Long(10 * 60 * 1000));
        // timeout after 10 min
        client2.waitForWithTimeout(new Long(10 * 60 * 1000));
        SSHCommandResult client1Result = client1.getSSHCommandResult();
        SSHCommandResult client2Result = client2.getSSHCommandResult();
        // 201405091623:44.055 - INFO: SSHCommandResult from an attempt to register with activation keys on 'jsefler-7server.usersys.redhat.com':
        // exitCode=255
        // stdout=''
        // stderr='Problem creating unit Consumer [id = 8a9087e345a9f5f90145e2a740c200ad, type = ConsumerType [id=1000, label=system], getName() = jsefler-7server.usersys.redhat.com]'
        // assert the results
        log.info("SSHCommandResult from an attempt to register with activation keys on '" + client1tasks.hostname + "': \n" + client1Result);
        log.info("SSHCommandResult from an attempt to register with activation keys on '" + client2tasks.hostname + "': \n" + client2Result);
        Assert.assertEquals(client1Result.getExitCode(), Integer.valueOf(0), "The exit code from register with activation keys on '" + client1tasks.hostname + "'.");
        // TEMPORARY WORKAROUND FOR BUG: 1183122 - rhsmd/subman dbus traceback on 'attach --pool'
        if (client1Result.getStderr().contains("KeyError: 'product_id'")) {
            boolean invokeWorkaroundWhileBugIsOpen = true;
            String bugId = "1183122";
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                log.warning("Encountered bug '" + bugId + "'. Skipping stderr/exitCode assertion from the prior register with activationkey command while bug '" + bugId + "' is open.");
            }
        } else
            // Assert client1Result.getStderr()
            // END OF WORKAROUND
            Assert.assertEquals(client1Result.getStderr(), "", "Stderr from the unsubscribe all on '" + client1tasks.hostname + "'.");
        Assert.assertEquals(client2Result.getExitCode(), Integer.valueOf(0), "The exit code from register with activation keys on '" + client2tasks.hostname + "'.");
        // TEMPORARY WORKAROUND FOR BUG: 1183122 - rhsmd/subman dbus traceback on 'attach --pool'
        if (client2Result.getStderr().contains("KeyError: 'product_id'")) {
            boolean invokeWorkaroundWhileBugIsOpen = true;
            String bugId = "1183122";
            try {
                if (invokeWorkaroundWhileBugIsOpen && BzChecker.getInstance().isBugOpen(bugId)) {
                    log.fine("Invoking workaround for " + BzChecker.getInstance().getBugState(bugId).toString() + " Bugzilla " + bugId + ".  (https://bugzilla.redhat.com/show_bug.cgi?id=" + bugId + ")");
                    SubscriptionManagerCLITestScript.addInvokedWorkaround(bugId);
                } else {
                    invokeWorkaroundWhileBugIsOpen = false;
                }
            } catch (BugzillaAPIException be) {
            /* ignore exception */
            } catch (RuntimeException re) {
            /* ignore exception */
            }
            if (invokeWorkaroundWhileBugIsOpen) {
                log.warning("Encountered bug '" + bugId + "'. Skipping stderr/exitCode assertion from the prior register with activationkey command while bug '" + bugId + "' is open.");
            }
        } else
            // Assert client2Result.getStderr()
            // END OF WORKAROUND
            Assert.assertEquals(client2Result.getStderr(), "", "Stderr from the unsubscribe all on '" + client2tasks.hostname + "'.");
    }
}
Also used : HashMap(java.util.HashMap) JSONArray(org.json.JSONArray) ArrayList(java.util.ArrayList) BugzillaAPIException(com.redhat.qe.auto.bugzilla.BugzillaAPIException) JSONObject(org.json.JSONObject) SSHCommandResult(com.redhat.qe.tools.SSHCommandResult) ArrayList(java.util.ArrayList) List(java.util.List) SkipException(org.testng.SkipException) TestDefinition(com.github.redhatqe.polarize.metadata.TestDefinition) Test(org.testng.annotations.Test)

Aggregations

BugzillaAPIException (com.redhat.qe.auto.bugzilla.BugzillaAPIException)134 Test (org.testng.annotations.Test)91 SkipException (org.testng.SkipException)89 TestDefinition (com.github.redhatqe.polarize.metadata.TestDefinition)88 SSHCommandResult (com.redhat.qe.tools.SSHCommandResult)77 ImplementsNitrateTest (com.redhat.qe.auto.tcms.ImplementsNitrateTest)55 ArrayList (java.util.ArrayList)44 SubscriptionPool (rhsm.data.SubscriptionPool)28 File (java.io.File)23 ProductSubscription (rhsm.data.ProductSubscription)23 ProductCert (rhsm.data.ProductCert)22 JSONObject (org.json.JSONObject)20 EntitlementCert (rhsm.data.EntitlementCert)16 InstalledProduct (rhsm.data.InstalledProduct)14 BigInteger (java.math.BigInteger)13 List (java.util.List)12 ConsumerCert (rhsm.data.ConsumerCert)11 SyndFeed (com.sun.syndication.feed.synd.SyndFeed)10 Calendar (java.util.Calendar)10 HashMap (java.util.HashMap)10