From 1eeeedb9d7937170ee91122f0f06abe02c347a3d Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Tue, 7 Jul 2026 12:26:34 -0700 Subject: [PATCH 01/29] first push --- src/azure-cli/HISTORY.rst | 1 + .../cli/command_modules/appservice/_help.py | 36 +++ .../cli/command_modules/appservice/_params.py | 9 + .../command_modules/appservice/commands.py | 22 ++ .../cli/command_modules/appservice/custom.py | 208 ++++++++++++++++++ .../latest/test_webapp_commands_thru_mock.py | 180 ++++++++++++++- 6 files changed, 455 insertions(+), 1 deletion(-) diff --git a/src/azure-cli/HISTORY.rst b/src/azure-cli/HISTORY.rst index 7bbafa33310..8cc961875a5 100644 --- a/src/azure-cli/HISTORY.rst +++ b/src/azure-cli/HISTORY.rst @@ -139,6 +139,7 @@ Release History * `az webapp list-runtimes`: Add `--runtime` and `--support` filter parameters (#32903) * [BREAKING CHANGE] `az webapp list-runtimes`: Remove deprecated `--linux` and `--show-runtime-details` parameters (#32903) * `az webapp log startup`: Add commands to list and view Linux container startup logs (#33256) +* `az webapp troubleshoot config`: Add preview command that runs built-in configuration checks against a Linux web app and surfaces the last runtime error from ARM * `az webapp create`: Add `--site-scoped-certs` parameter to support enabling or disabling site-scoped certificates (#33306) * `az webapp up`: Add warning message for future deprecation (#33410) * `az functionapp deployment source config-zip`: Fix `KeyError` `'FUNCTIONS_WORKER_RUNTIME'` for Go function apps on Flex Consumption (#33404) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_help.py b/src/azure-cli/azure/cli/command_modules/appservice/_help.py index eea863ae35d..6c173d22213 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_help.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_help.py @@ -2432,6 +2432,42 @@ text: az webapp log startup show --name MyWebApp --resource-group MyResourceGroup --instance lw0sdlwk000002 """ +helps['webapp troubleshoot'] = """ +type: group +short-summary: Diagnose common Linux web app problems. +long-summary: > + Preview command group that pairs built-in configuration checks (from + KuduLite on the worker) with the most recent runtime error reported by + Azure Resource Manager. Use when a Linux app is failing to start, returning + HTTP 502/503, or exhibiting other post-deployment misbehavior. +""" + +helps['webapp troubleshoot config'] = """ +type: command +short-summary: Validate configuration for a Linux web app and surface the last runtime error. +long-summary: > + Aggregates two data sources into a single report: + + (1) Built-in configuration checks from KuduLite — a set of common + Linux App Service settings (linuxFxVersion, port binding, startup + command, alwaysOn, health check path, ...) evaluated against the + running site's configuration snapshot. + + (2) The most recent runtime error reported by Azure Resource Manager + (`/siteStatus`) — echoed verbatim from the platform. + + By default the command returns a structured payload so the standard + `-o json/yaml/tsv/table` formatters handle output. Pass `--report` to + print a human-readable two-section report to stdout instead. +examples: + - name: Run the built-in configuration checks and show the runtime error, if any + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup + - name: Print the human-readable report + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --report + - name: Target a deployment slot + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --slot staging +""" + helps['functionapp log'] = """ type: group short-summary: Manage function app logs. diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_params.py b/src/azure-cli/azure/cli/command_modules/appservice/_params.py index 45ef30f76d3..a6b477253c2 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_params.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_params.py @@ -852,6 +852,15 @@ def load_arguments(self, _): with self.argument_context('webapp log startup show') as c: c.argument('filename', options_list=['--filename', '-f'], help='Name of a specific startup log file to display. If not specified, shows the latest log (preferring failures).') + with self.argument_context('webapp troubleshoot') as c: + c.argument('name', arg_type=webapp_name_arg_type, id_part=None) + c.argument('resource_group_name', arg_type=resource_group_name_type) + c.argument('slot', options_list=['--slot', '-s'], help="the name of the slot. Default to the production slot if not specified") + + with self.argument_context('webapp troubleshoot config') as c: + c.argument('report', options_list=['--report'], arg_type=get_three_state_flag(), + help='Print a human-readable report instead of the structured payload.') + with self.argument_context('functionapp log deployment show') as c: c.argument('name', arg_type=functionapp_name_arg_type, id_part=None) c.argument('resource_group', arg_type=resource_group_name_type) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/commands.py b/src/azure-cli/azure/cli/command_modules/appservice/commands.py index 31be90a3615..d10de5e727b 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/commands.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/commands.py @@ -48,6 +48,24 @@ def transform_runtime_list_output(result): ]) for r in result] +def transform_troubleshoot_config_output(result): + """Flatten the troubleshoot config payload into a per-setting table. + + Falls back gracefully for non-dict / empty payloads (e.g. --report was + passed and the command returned ``None``). + """ + from collections import OrderedDict + if not isinstance(result, dict): + return [] + settings = result.get('settings') or [] + return [OrderedDict([ + ('Setting', s.get('Setting') or s.get('setting') or ''), + ('Value', s.get('Value') if s.get('Value') is not None else s.get('value') or ''), + ('Details', s.get('Details') or s.get('details') or ''), + ]) for s in settings if isinstance(s, dict)] + + + def ex_handler_factory(creating_plan=False): def _ex_handler(ex): ex = _polish_bad_errors(ex, creating_plan) @@ -259,6 +277,10 @@ def load_command_table(self, _): g.custom_command('list', 'list_startup_logs') g.custom_show_command('show', 'show_startup_log') + with self.command_group('webapp troubleshoot', is_preview=True) as g: + g.custom_command('config', 'troubleshoot_config', + table_transformer=transform_troubleshoot_config_output) + with self.command_group('functionapp log deployment') as g: g.custom_show_command('show', 'show_deployment_log') g.custom_command('list', 'list_deployment_logs') diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index a3b809afa62..d3896d228e6 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6560,6 +6560,214 @@ def show_startup_log(cmd, resource_group, name, slot=None, filename=None, instan return response.json() +# ----------------------------------------------------------------------------- +# az webapp troubleshoot config +# ----------------------------------------------------------------------------- + +def _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot=None): + client = web_client_factory(cmd.cli_ctx) + if slot: + app = client.web_apps.get_slot(resource_group_name, name, slot) + else: + app = client.web_apps.get(resource_group_name, name) + if app is None or not is_linux_webapp(app): + raise ArgumentUsageError( + "'az webapp troubleshoot config' is only supported for Linux web apps.") + + +def _extract_runtime_error(arm_response): + """Return the freshest runtime-error block from an ARM /siteStatus response. + + /siteStatus returns per-instance status under 'properties' (a list); the + single-instance form returns a dict. We normalize both, then pick the entry + with the latest ``lastErrorTimestamp`` that also has a non-empty + ``lastError``. Returns ``None`` when no runtime error is reported. + """ + if not isinstance(arm_response, dict): + return None + properties = arm_response.get('properties') + if isinstance(properties, list): + items = properties + elif isinstance(properties, dict): + items = [properties] + else: + return None + candidates = [item for item in items if isinstance(item, dict) and item.get('lastError')] + if not candidates: + return None + + def _ts_key(item): + return item.get('lastErrorTimestamp') or '' + + candidates.sort(key=_ts_key, reverse=True) + return candidates[0] + + +def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False): + """Aggregate built-in KuduLite config-check findings plus the most recent ARM + /siteStatus runtime error for a Linux web app. + + Data sources: + * Built-in checks come from KuduLite (SCM): + GET https://{scm-host}/api/troubleshoot/config + * Last runtime error comes from ARM: + GET /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web + /sites/{name}[/slots/{slot}]/siteStatus?api-version=... + + By default returns the structured payload so the standard + ``-o json/yaml/tsv/table`` formatters handle output. Pass ``--report`` to + print the human-readable two-section report to stdout instead. + """ + import requests + from azure.cli.core.commands.client_factory import get_subscription_id + from azure.core.exceptions import HttpResponseError as _Hre + + _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot) + + # ---- 1. Built-in checks from KuduLite ---- + scm_url = _get_scm_url(cmd, resource_group_name, name, slot) + headers = get_scm_site_headers(cmd.cli_ctx, name, resource_group_name, slot) + config_url = '{}/api/troubleshoot/config'.format(scm_url) + + built_in = {'siteName': None, 'instanceId': None, 'writtenAt': None, 'settings': []} + try: + config_response = requests.get(config_url, headers=headers, timeout=30) + except requests.RequestException as ex: + logger.warning("Failed to call '%s': %s", config_url, ex) + config_response = None + + if config_response is not None: + if config_response.status_code == 200: + try: + body = config_response.json() + except ValueError: + body = None + if isinstance(body, dict): + built_in['siteName'] = body.get('SiteName') or body.get('siteName') + built_in['instanceId'] = body.get('InstanceId') or body.get('instanceId') + built_in['writtenAt'] = body.get('WrittenAt') or body.get('writtenAt') + settings = body.get('Settings') or body.get('settings') or [] + if isinstance(settings, list): + built_in['settings'] = [s for s in settings if isinstance(s, dict)] + elif config_response.status_code == 404: + # Endpoint not rolled out yet or app has never written a config snapshot. + logger.warning( + 'Built-in configuration checks are not available for this app. ' + "This feature requires a platform version that may not have rolled " + "out to your app's region yet.") + else: + logger.warning( + "Failed to retrieve built-in configuration checks from '%s' " + "(status %s %s).", + config_url, config_response.status_code, config_response.reason) + + # ---- 2. Site runtime status from ARM /siteStatus ---- + client = web_client_factory(cmd.cli_ctx) + subscription_id = get_subscription_id(cmd.cli_ctx) + # pylint: disable=protected-access + api_version = client._config.api_version + # pylint: enable=protected-access + slot_segment = '/slots/{}'.format(slot) if slot else '' + arm_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/siteStatus?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group_name, name=name, + slot_seg=slot_segment, ver=api_version) + + runtime_error = None + try: + arm_response = send_raw_request(cmd.cli_ctx, 'GET', arm_url).json() + runtime_error = _extract_runtime_error(arm_response) + except _Hre as ex: + logger.warning("Failed to retrieve site runtime status from '%s': %s", arm_url, ex) + except ValueError as ex: + logger.warning("Failed to parse site runtime status response: %s", ex) + + payload = { + 'name': name, + 'resourceGroup': resource_group_name, + 'siteName': built_in['siteName'], + 'instanceId': built_in['instanceId'], + 'writtenAt': built_in['writtenAt'], + 'settings': built_in['settings'], + 'runtimeError': runtime_error, + } + if report: + _render_troubleshoot_config(payload) + return None + return payload + + +def _render_troubleshoot_config(payload): + """Print the human-readable report: BUILT-IN CHECKS + SITE RUNTIME ERROR + RECOMMENDATION. Invoked when ``--report`` is passed.""" + from azure.cli.core.style import Style, print_styled_text + + def _out(*objs): + print_styled_text(*objs, file=sys.stdout) + + def _row(*objs): + _out(list(objs)) + + settings = payload.get('settings') or [] + runtime_error = payload.get('runtimeError') + + # ---- Section 1: Built-in checks ---- + _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) + _out() + if not settings: + _row((Style.WARNING, 'No built-in configuration checks reported.')) + else: + setting_w = max(20, min(40, max(len(str(s.get('Setting') or '')) for s in settings) + 2)) + value_w = max(15, min(30, max(len(str(s.get('Value') or '')) for s in settings) + 2)) + + header = '{sname:<{sw}}{vname:<{vw}}{dname}'.format( + sname='Setting', sw=setting_w, vname='Value', vw=value_w, dname='Details') + _row((Style.PRIMARY, header)) + _row((Style.PRIMARY, '{s}{v}{d}'.format( + s=('─' * (setting_w - 2)).ljust(setting_w), + v=('─' * (value_w - 2)).ljust(value_w), + d='─' * 40))) + for setting in settings: + name_v = str(setting.get('Setting') or '') + value_v = str(setting.get('Value') if setting.get('Value') is not None else '') + details_v = str(setting.get('Details') or '') + line = '{s:<{sw}}{v:<{vw}}{d}'.format( + s=name_v, sw=setting_w, v=value_v, vw=value_w, d=details_v) + # Highlight problem findings (anything other than "No issues detected"). + is_ok = details_v.strip().lower().startswith('no issues') + style = Style.SUCCESS if is_ok else Style.WARNING + _row((style, line)) + + _out() + + # ---- Section 2: Site runtime error recommendation ---- + _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) + _out() + if runtime_error is None: + _row((Style.SUCCESS, 'No runtime error reported.')) + else: + state = str(runtime_error.get('state') or '') + action = str(runtime_error.get('lastErrorAction') or runtime_error.get('action') or '') + last_error = str(runtime_error.get('lastError') or '') + details = str(runtime_error.get('lastErrorDetails') or runtime_error.get('details') or '') + + _row((Style.PRIMARY, 'Last runtime error')) + if state: + _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) + if action: + _row((Style.PRIMARY, ' Action: '), (Style.WARNING, action)) + if last_error: + _row((Style.PRIMARY, ' Error: '), (Style.ERROR, last_error)) + if details: + _row((Style.PRIMARY, ' Details: '), (Style.WARNING, details)) + + _out() + _row((Style.PRIMARY, '─' * 75)) + + def config_slot_auto_swap(cmd, resource_group_name, webapp, slot, auto_swap_slot=None, disable=None): client = web_client_factory(cmd.cli_ctx) site_config = client.web_apps.get_configuration_slot(resource_group_name, webapp, slot) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py index ec6a96e6502..8f620dde0a8 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py @@ -39,6 +39,8 @@ update_webapp, list_startup_logs, show_startup_log, + troubleshoot_config, + _extract_runtime_error, create_webapp) # pylint: disable=line-too-long @@ -1794,7 +1796,183 @@ def test_get_java_runtimes_from_container_settings_reads_mapping(self): runtimes = _StackRuntimeHelper._get_java_runtimes_from_container_settings(settings) self.assertEqual({name for name, _, _ in runtimes}, self.EXPECTED) self.assertTrue(all(is_auto for _, _, is_auto in runtimes)) +class TestTroubleshootConfigMocked(unittest.TestCase): + """Tests for az webapp troubleshoot config.""" + + def setUp(self): + is_linux_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=True) + client_factory_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.web_client_factory') + is_linux_patch.start() + cf_mock = client_factory_patch.start() + # Give the real code path something to unpack when it reads + # client._config.api_version. + cf_mock.return_value._config.api_version = '2022-03-01' + self.addCleanup(is_linux_patch.stop) + self.addCleanup(client_factory_patch.stop) + + def _scm_response(self, status_code=200, json_data=None, reason=''): + resp = mock.MagicMock() + resp.status_code = status_code + resp.reason = reason + resp.json.return_value = json_data + return resp + + def _arm_response(self, json_data): + resp = mock.MagicMock() + resp.json.return_value = json_data + return resp + + # ---- _extract_runtime_error ---- + + def test_extract_runtime_error_picks_latest_timestamp(self): + arm = {'properties': [ + {'state': 'Started', 'lastError': None}, + {'state': 'Stopped', 'lastError': 'A', 'lastErrorTimestamp': '2026-07-01T00:00:00Z'}, + {'state': 'Stopped', 'lastError': 'B', 'lastErrorTimestamp': '2026-07-02T00:00:00Z'}, + ]} + self.assertEqual(_extract_runtime_error(arm)['lastError'], 'B') + + def test_extract_runtime_error_returns_none_when_all_started(self): + arm = {'properties': [{'state': 'Started', 'lastError': None}]} + self.assertIsNone(_extract_runtime_error(arm)) + + def test_extract_runtime_error_handles_single_dict_properties(self): + arm = {'properties': {'state': 'Stopped', 'lastError': 'X'}} + self.assertEqual(_extract_runtime_error(arm)['lastError'], 'X') + + def test_extract_runtime_error_handles_missing_properties(self): + self.assertIsNone(_extract_runtime_error({})) + self.assertIsNone(_extract_runtime_error(None)) + + # ---- troubleshoot_config ---- + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, + _headers_mock, send_raw_request_mock): + settings = [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', 'Details': 'No issues detected'}, + {'Setting': 'alwaysOn', 'Value': 'false', 'Details': 'App may be unloaded when idle'}, + ] + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', + 'InstanceId': 'abc123', + 'WrittenAt': '2026-07-07T18:12:29+00:00', + 'Settings': settings, + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'Container did not respond', 'lastErrorAction': 'WaitingForSiteToStart', + 'lastErrorTimestamp': '2026-07-07T18:00:00Z'}, + ]}) + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertEqual(result['name'], 'myApp') + self.assertEqual(result['resourceGroup'], 'myRG') + self.assertEqual(result['settings'], settings) + self.assertEqual(result['runtimeError']['lastError'], 'ContainerTimeout') + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_scm_404_returns_empty_settings(self, requests_get_mock, + _scm_url_mock, _headers_mock, + send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(404) + send_raw_request_mock.return_value = self._arm_response({'properties': []}) + + with mock.patch('azure.cli.command_modules.appservice.custom.logger') as logger_mock: + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertEqual(result['settings'], []) + self.assertIsNone(result['runtimeError']) + logger_mock.warning.assert_called() + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_mock, + _headers_mock, send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [{'Setting': 'alwaysOn', 'Value': 'true', + 'Details': 'No issues detected'}], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Started', 'lastError': None}, + ]}) + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertIsNone(result['runtimeError']) + self.assertEqual(len(result['settings']), 1) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_mock, + _scm_url_mock, _headers_mock, + send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected'}, + {'Setting': 'alwaysOn', 'Value': 'false', + 'Details': 'App may be unloaded when idle'}, + ], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ImagePullUnauthorizedFailure', + 'lastErrorDetails': 'forbidden', 'lastErrorAction': 'StartingSiteContainers', + 'lastErrorTimestamp': '2026-07-07T18:00:00Z'}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + self.assertIsNone(result) + # BUILT-IN CHECKS + SITE RUNTIME ERROR RECOMMENDATION headers should + # have been printed. + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('BUILT-IN CHECKS', printed_text) + self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertIn('ImagePullUnauthorizedFailure', printed_text) + + def test_troubleshoot_config_raises_on_windows(self): + with mock.patch( + 'azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=False): + with self.assertRaises(ArgumentUsageError) as cm: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myWindowsApp') + self.assertIn('Linux', str(cm.exception)) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 839b4b7e7fd1d0bb79b5f70a857c282b0f906fb1 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Wed, 8 Jul 2026 13:56:07 -0700 Subject: [PATCH 02/29] output updates, nest config check --- .../command_modules/appservice/commands.py | 10 +- .../cli/command_modules/appservice/custom.py | 191 ++++++++++++++---- .../latest/test_webapp_commands_thru_mock.py | 162 ++++++++++++++- 3 files changed, 312 insertions(+), 51 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/commands.py b/src/azure-cli/azure/cli/command_modules/appservice/commands.py index d10de5e727b..aea1c45fb55 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/commands.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/commands.py @@ -51,13 +51,17 @@ def transform_runtime_list_output(result): def transform_troubleshoot_config_output(result): """Flatten the troubleshoot config payload into a per-setting table. - Falls back gracefully for non-dict / empty payloads (e.g. --report was - passed and the command returned ``None``). + Reads Settings out of the nested ``configCheck`` field (the verbatim SCM + body). Falls back gracefully for non-dict / empty payloads (e.g. --report + was passed and the command returned ``None``). """ from collections import OrderedDict if not isinstance(result, dict): return [] - settings = result.get('settings') or [] + config_check = result.get('configCheck') or {} + settings = config_check.get('Settings') or config_check.get('settings') or [] + if not isinstance(settings, list): + return [] return [OrderedDict([ ('Setting', s.get('Setting') or s.get('setting') or ''), ('Value', s.get('Value') if s.get('Value') is not None else s.get('value') or ''), diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index d3896d228e6..c0ea9baa758 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6625,41 +6625,114 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot) # ---- 1. Built-in checks from KuduLite ---- + # + # KuduLite reads the config snapshot from ``/appsvctmp/config_check_{siteName}.json`` + # on the worker where the request lands. On multi-worker plans only the + # instance that most recently ran the site's startup pipeline has the + # file, so ARR-affinity routing to any other worker returns 404. Retry + # per-instance until one worker responds with data. scm_url = _get_scm_url(cmd, resource_group_name, name, slot) headers = get_scm_site_headers(cmd.cli_ctx, name, resource_group_name, slot) config_url = '{}/api/troubleshoot/config'.format(scm_url) - built_in = {'siteName': None, 'instanceId': None, 'writtenAt': None, 'settings': []} - try: - config_response = requests.get(config_url, headers=headers, timeout=30) - except requests.RequestException as ex: - logger.warning("Failed to call '%s': %s", config_url, ex) - config_response = None + config_check = None + last_status = None + last_body_text = '' + tried_instances = [] + + def _try_config(cookies=None): + try: + return requests.get(config_url, headers=headers, cookies=cookies, + timeout=30, allow_redirects=False) + except requests.RequestException as ex: + logger.warning("Failed to call '%s': %s", config_url, ex) + return None - if config_response is not None: - if config_response.status_code == 200: + def _consume(resp): + nonlocal config_check, last_status, last_body_text + if resp is None: + return False + last_status = resp.status_code + last_body_text = (resp.text or '').strip() + if last_status == 200: try: - body = config_response.json() + body = resp.json() except ValueError: body = None if isinstance(body, dict): - built_in['siteName'] = body.get('SiteName') or body.get('siteName') - built_in['instanceId'] = body.get('InstanceId') or body.get('instanceId') - built_in['writtenAt'] = body.get('WrittenAt') or body.get('writtenAt') - settings = body.get('Settings') or body.get('settings') or [] - if isinstance(settings, list): - built_in['settings'] = [s for s in settings if isinstance(s, dict)] - elif config_response.status_code == 404: - # Endpoint not rolled out yet or app has never written a config snapshot. + config_check = body + return True + snippet = last_body_text[:200] logger.warning( - 'Built-in configuration checks are not available for this app. ' - "This feature requires a platform version that may not have rolled " - "out to your app's region yet.") - else: + "Built-in configuration checks endpoint '%s' returned 200 but the body " + "wasn't the expected JSON object. First 200 chars: %r", + config_url, snippet) + return False + + # First attempt: whichever worker ARR picks. + if not _consume(_try_config()): + # On 404, walk instances and retry with ARR affinity pinned to each. + if last_status == 404: + try: + client = web_client_factory(cmd.cli_ctx) + # pylint: disable=protected-access + api_version = client._config.api_version + # pylint: enable=protected-access + subscription_id = get_subscription_id(cmd.cli_ctx) + slot_segment = '/slots/{}'.format(slot) if slot else '' + instances_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/instances?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group_name, name=name, + slot_seg=slot_segment, ver=api_version) + instances_payload = send_raw_request(cmd.cli_ctx, 'GET', instances_url).json() + instance_ids = [e.get('name') for e in (instances_payload.get('value') or []) + if isinstance(e, dict) and e.get('name')] + except Exception as ex: # pylint: disable=broad-except + logger.warning('Failed to enumerate site instances for retry: %s', ex) + instance_ids = [] + + for instance_id in instance_ids: + tried_instances.append(instance_id) + cookies = {'ARRAffinity': instance_id, 'ARRAffinitySameSite': instance_id} + if _consume(_try_config(cookies=cookies)): + break + + if config_check is None: + status = last_status + if status == 404: + if last_body_text: + logger.warning( + "Built-in configuration checks endpoint returned 404 on all " + "workers (tried %d instance(s)). Response body: %s " + "This usually means the site container has not started " + "successfully yet, so no configuration snapshot was written. " + "See the runtime-error section below or run 'az webapp log tail' " + "for startup details.", + max(1, len(tried_instances)), last_body_text) + else: + logger.warning( + "Built-in configuration checks are not available for this app " + "(SCM returned 404 for %s). This feature requires a platform " + "version that may not have rolled out to your app's region yet.", + config_url) + elif status in (401, 403): + logger.warning( + "Access to built-in configuration checks was denied by the SCM " + "endpoint (status %s). Make sure basic auth is enabled for SCM on " + "this site, or that your credentials have SCM access.", status) + elif status in (301, 302, 303, 307, 308): + logger.warning( + "Built-in configuration checks endpoint '%s' returned a redirect " + "(status %s). This usually means SCM authentication is " + "misconfigured for this app.", config_url, status) + elif status is not None: logger.warning( "Failed to retrieve built-in configuration checks from '%s' " - "(status %s %s).", - config_url, config_response.status_code, config_response.reason) + "(status %s).", + config_url, status) # ---- 2. Site runtime status from ARM /siteStatus ---- client = web_client_factory(cmd.cli_ctx) @@ -6688,10 +6761,7 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) payload = { 'name': name, 'resourceGroup': resource_group_name, - 'siteName': built_in['siteName'], - 'instanceId': built_in['instanceId'], - 'writtenAt': built_in['writtenAt'], - 'settings': built_in['settings'], + 'configCheck': config_check, 'runtimeError': runtime_error, } if report: @@ -6711,9 +6781,50 @@ def _out(*objs): def _row(*objs): _out(list(objs)) - settings = payload.get('settings') or [] + config_check = payload.get('configCheck') or {} + settings = config_check.get('Settings') or config_check.get('settings') or [] + if not isinstance(settings, list): + settings = [] + settings = [s for s in settings if isinstance(s, dict)] runtime_error = payload.get('runtimeError') + def _issue_detected(setting): + # KuduLite emits IssueDetected as a JSON string ("True"/"False") today, + # but tolerate a real bool too in case the contract firms up later. + raw = setting.get('IssueDetected') + if raw is None: + raw = setting.get('issueDetected') + if isinstance(raw, bool): + return raw + if isinstance(raw, str): + return raw.strip().lower() == 'true' + return False + + any_issue = any(_issue_detected(s) for s in settings) + + def _last_error_within(minutes): + # Runtime error is considered "fresh" if its timestamp is within the + # last N minutes (UTC). ARM emits lastErrorTimestamp as an ISO 8601 + # string; tolerate a trailing 'Z' and missing tzinfo (treated as UTC). + if not runtime_error: + return False + raw = runtime_error.get('lastErrorTimestamp') + if not raw: + return False + from datetime import datetime, timezone, timedelta + try: + ts = str(raw).strip() + if ts.endswith('Z'): + ts = ts[:-1] + '+00:00' + parsed = datetime.fromisoformat(ts) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + return False + return (datetime.now(timezone.utc) - parsed) <= timedelta(minutes=minutes) + + show_runtime = any_issue or _last_error_within(15) + # ---- Section 1: Built-in checks ---- _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) _out() @@ -6736,33 +6847,37 @@ def _row(*objs): details_v = str(setting.get('Details') or '') line = '{s:<{sw}}{v:<{vw}}{d}'.format( s=name_v, sw=setting_w, v=value_v, vw=value_w, d=details_v) - # Highlight problem findings (anything other than "No issues detected"). - is_ok = details_v.strip().lower().startswith('no issues') - style = Style.SUCCESS if is_ok else Style.WARNING + style = Style.WARNING if _issue_detected(setting) else Style.SUCCESS _row((style, line)) _out() # ---- Section 2: Site runtime error recommendation ---- + # Rendered when the built-in checks flagged at least one setting with + # IssueDetected == true, OR when the ARM lastErrorTimestamp is within the + # last 15 minutes (fresh failures are worth surfacing even if the SCM + # checks look clean). + if not show_runtime: + return + _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: _row((Style.SUCCESS, 'No runtime error reported.')) else: state = str(runtime_error.get('state') or '') - action = str(runtime_error.get('lastErrorAction') or runtime_error.get('action') or '') last_error = str(runtime_error.get('lastError') or '') details = str(runtime_error.get('lastErrorDetails') or runtime_error.get('details') or '') + timestamp = str(runtime_error.get('lastErrorTimestamp') or '') - _row((Style.PRIMARY, 'Last runtime error')) if state: - _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) - if action: - _row((Style.PRIMARY, ' Action: '), (Style.WARNING, action)) + _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) if last_error: - _row((Style.PRIMARY, ' Error: '), (Style.ERROR, last_error)) + _row((Style.PRIMARY, ' Last Error: '), (Style.ERROR, last_error)) if details: - _row((Style.PRIMARY, ' Details: '), (Style.WARNING, details)) + _row((Style.PRIMARY, ' Last Error Details: '), (Style.WARNING, details)) + if timestamp: + _row((Style.PRIMARY, ' Last Error Timestamp: '), (Style.WARNING, timestamp)) _out() _row((Style.PRIMARY, '─' * 75)) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py index 8f620dde0a8..bad82015274 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py @@ -1861,12 +1861,13 @@ def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', 'Details': 'No issues detected'}, {'Setting': 'alwaysOn', 'Value': 'false', 'Details': 'App may be unloaded when idle'}, ] - requests_get_mock.return_value = self._scm_response(200, json_data={ + scm_body = { 'SiteName': 'myApp', 'InstanceId': 'abc123', 'WrittenAt': '2026-07-07T18:12:29+00:00', 'Settings': settings, - }) + } + requests_get_mock.return_value = self._scm_response(200, json_data=scm_body) send_raw_request_mock.return_value = self._arm_response({'properties': [ {'state': 'Stopped', 'lastError': 'ContainerTimeout', 'lastErrorDetails': 'Container did not respond', 'lastErrorAction': 'WaitingForSiteToStart', @@ -1877,7 +1878,9 @@ def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, self.assertEqual(result['name'], 'myApp') self.assertEqual(result['resourceGroup'], 'myRG') - self.assertEqual(result['settings'], settings) + # configCheck is the verbatim SCM body — PascalCase keys preserved. + self.assertEqual(result['configCheck'], scm_body) + self.assertEqual(result['configCheck']['Settings'], settings) self.assertEqual(result['runtimeError']['lastError'], 'ContainerTimeout') @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') @@ -1895,7 +1898,7 @@ def test_troubleshoot_config_scm_404_returns_empty_settings(self, requests_get_m with mock.patch('azure.cli.command_modules.appservice.custom.logger') as logger_mock: result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') - self.assertEqual(result['settings'], []) + self.assertIsNone(result['configCheck']) self.assertIsNone(result['runtimeError']) logger_mock.warning.assert_called() @@ -1919,7 +1922,8 @@ def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_ result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') self.assertIsNone(result['runtimeError']) - self.assertEqual(len(result['settings']), 1) + self.assertEqual(len(result['configCheck']['Settings']), 1) + self.assertEqual(result['configCheck']['SiteName'], 'myApp') @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', @@ -1930,13 +1934,16 @@ def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # At least one setting has IssueDetected: True so the runtime-error + # section will render. requests_get_mock.return_value = self._scm_response(200, json_data={ 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', 'Settings': [ {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', - 'Details': 'No issues detected'}, - {'Setting': 'alwaysOn', 'Value': 'false', - 'Details': 'App may be unloaded when idle'}, + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + {'Setting': 'websitesPort', 'Value': '', + 'Details': "App doesn't respond on expected port", + 'IssueDetected': 'True'}, ], }) send_raw_request_mock.return_value = self._arm_response({'properties': [ @@ -1950,8 +1957,6 @@ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_m result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) self.assertIsNone(result) - # BUILT-IN CHECKS + SITE RUNTIME ERROR RECOMMENDATION headers should - # have been printed. printed_text = '' for call in print_mock.call_args_list: for arg in call.args: @@ -1964,6 +1969,98 @@ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_m self.assertIn('BUILT-IN CHECKS', printed_text) self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) self.assertIn('ImagePullUnauthorizedFailure', printed_text) + self.assertIn('Last Error:', printed_text) + self.assertIn('Last Error Details:', printed_text) + self.assertIn('Last Error Timestamp:', printed_text) + # Removed labels should NOT appear. + self.assertNotIn('Last runtime error', printed_text) + self.assertNotIn('Action:', printed_text) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_suppresses_runtime_when_no_issues( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # Every setting has IssueDetected == "False" -> runtime section suppressed. + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + {'Setting': 'alwaysOn', 'Value': 'true', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + ], + }) + # Even if ARM reports a stale runtime error, we should not render the + # recommendation section when no setting has an issue. + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Started', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'old error', 'lastErrorAction': 'None', + 'lastErrorTimestamp': '2026-07-01T00:00:00Z'}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('BUILT-IN CHECKS', printed_text) + self.assertNotIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertNotIn('ContainerTimeout', printed_text) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_shows_runtime_when_error_is_fresh( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # All settings clean, but ARM reports a runtime error whose timestamp + # is within the last 15 minutes -> section should still render. + from datetime import datetime, timezone, timedelta + fresh_ts = (datetime.now(timezone.utc) - timedelta(minutes=2)).strftime( + '%Y-%m-%dT%H:%M:%SZ') + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + ], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'fresh error', 'lastErrorAction': 'None', + 'lastErrorTimestamp': fresh_ts}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertIn('ContainerTimeout', printed_text) + self.assertIn('fresh error', printed_text) def test_troubleshoot_config_raises_on_windows(self): with mock.patch( @@ -1973,6 +2070,51 @@ def test_troubleshoot_config_raises_on_windows(self): troubleshoot_config(_get_test_cmd(), 'myRG', 'myWindowsApp') self.assertIn('Linux', str(cm.exception)) + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_retries_across_instances_on_404( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # First (unpinned) call and the first instance retry return 404 with + # KuduLite's "not available" body; the second instance retry returns + # the snapshot. + settings = [{'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected'}] + good = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'inst2', 'WrittenAt': 't', + 'Settings': settings, + }) + bad = mock.MagicMock() + bad.status_code = 404 + bad.text = 'Config check information is not available.' + bad.json.side_effect = ValueError('not json') + + requests_get_mock.side_effect = [bad, bad, good] + + send_raw_request_mock.side_effect = [ + # ARM /instances response (retry lookup). + self._arm_response({'value': [{'name': 'inst1'}, {'name': 'inst2'}]}), + # ARM /siteStatus response. + self._arm_response({'properties': []}), + ] + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertIsNotNone(result['configCheck']) + self.assertEqual(result['configCheck']['InstanceId'], 'inst2') + # 3 SCM calls: unpinned, then per-instance retry twice. + self.assertEqual(requests_get_mock.call_count, 3) + # Second and third calls should carry ARR affinity cookies. + self.assertEqual( + requests_get_mock.call_args_list[1].kwargs['cookies'], + {'ARRAffinity': 'inst1', 'ARRAffinitySameSite': 'inst1'}) + self.assertEqual( + requests_get_mock.call_args_list[2].kwargs['cookies'], + {'ARRAffinity': 'inst2', 'ARRAffinitySameSite': 'inst2'}) + if __name__ == '__main__': unittest.main() From 4ebcb50df16a368304253644d9c451e681ab57bf Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Thu, 9 Jul 2026 12:29:30 -0700 Subject: [PATCH 03/29] Merge status branch into combined test branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picks the entire origin/developer/amberamos/troubleshootcommand branch on top of the config command changes so both 'az webapp troubleshoot config' and 'az webapp troubleshoot status' preview commands can be tested together. Conflict resolution: additive merge — both commands, their help entries, param contexts, table transformers, and tests coexist. --- src/azure-cli/HISTORY.rst | 1 + .../cli/command_modules/appservice/_help.py | 38 ++ .../cli/command_modules/appservice/_params.py | 7 + .../command_modules/appservice/commands.py | 51 ++ .../cli/command_modules/appservice/custom.py | 543 +++++++++++++++++- .../latest/test_webapp_commands_thru_mock.py | 307 +++++++++- 6 files changed, 941 insertions(+), 6 deletions(-) diff --git a/src/azure-cli/HISTORY.rst b/src/azure-cli/HISTORY.rst index 8cc961875a5..d0204b47a6e 100644 --- a/src/azure-cli/HISTORY.rst +++ b/src/azure-cli/HISTORY.rst @@ -140,6 +140,7 @@ Release History * [BREAKING CHANGE] `az webapp list-runtimes`: Remove deprecated `--linux` and `--show-runtime-details` parameters (#32903) * `az webapp log startup`: Add commands to list and view Linux container startup logs (#33256) * `az webapp troubleshoot config`: Add preview command that runs built-in configuration checks against a Linux web app and surfaces the last runtime error from ARM +* `az webapp troubleshoot status`: Add new command group and command to show per-instance Site Runtime Status and recent startup summary for Linux web apps (#33673) * `az webapp create`: Add `--site-scoped-certs` parameter to support enabling or disabling site-scoped certificates (#33306) * `az webapp up`: Add warning message for future deprecation (#33410) * `az functionapp deployment source config-zip`: Fix `KeyError` `'FUNCTIONS_WORKER_RUNTIME'` for Go function apps on Flex Consumption (#33404) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_help.py b/src/azure-cli/azure/cli/command_modules/appservice/_help.py index 6c173d22213..847da439b86 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_help.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_help.py @@ -2468,6 +2468,44 @@ text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --slot staging """ +helps['webapp troubleshoot status'] = """ +type: command +short-summary: Show site runtime status and recent startup summary for a Linux web app. +long-summary: | + Aggregates two data sources: + + - Site Runtime Status: ARM /siteStatus[/{instanceId}] (per-instance state, + action, last error, details). + - Startup summary: KuduLite (SCM) /api/startuplogs/summary (counts of + successful and failed startup attempts in the last 24h, plus the + most recent success and failure timestamps). + + Use --instance to scope both to a single worker. By default returns the + structured payload (works with -o json, -o yaml, and -o table). Pass + --report to print a human-readable, color-coded report instead. +examples: + - name: Show status for all instances of a web app (JSON by default) + text: az webapp troubleshoot status --name MyWebApp --resource-group MyResourceGroup + - name: Print the human-readable report + text: az webapp troubleshoot status --name MyWebApp --resource-group MyResourceGroup --report + - name: Show status scoped to a single worker instance + text: az webapp troubleshoot status --name MyWebApp --resource-group MyResourceGroup --instance 7c2d9 +parameters: + - name: --instance + short-summary: Scope the report to a single worker instance. + long-summary: > + Accepts either the hex instanceId (from `az webapp list-instances`) or the + machine name (e.g. `lw0sdlwk0007AB`). When omitted, returns an overview of + every instance seen in the last 24 hours. + - name: --report + short-summary: Print a human-readable, color-coded report instead of returning structured data. + long-summary: > + When set, the command writes a formatted report (overview table plus + per-instance Last runtime status and Startup summary) to stdout and + returns no machine-readable output. Omit --report to keep the default + structured payload that works with `-o json`, `-o yaml`, and `-o table`. +""" + helps['functionapp log'] = """ type: group short-summary: Manage function app logs. diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_params.py b/src/azure-cli/azure/cli/command_modules/appservice/_params.py index a6b477253c2..a3cb8229081 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_params.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_params.py @@ -861,6 +861,13 @@ def load_arguments(self, _): c.argument('report', options_list=['--report'], arg_type=get_three_state_flag(), help='Print a human-readable report instead of the structured payload.') + with self.argument_context('webapp troubleshoot status') as c: + c.argument('name', arg_type=webapp_name_arg_type, id_part=None) + c.argument('resource_group', arg_type=resource_group_name_type) + c.argument('slot', options_list=['--slot', '-s'], help="the name of the slot. Default to the production slot if not specified") + c.argument('instance', options_list=['--instance']) + c.argument('report', options_list=['--report'], arg_type=get_three_state_flag()) + with self.argument_context('functionapp log deployment show') as c: c.argument('name', arg_type=functionapp_name_arg_type, id_part=None) c.argument('resource_group', arg_type=resource_group_name_type) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/commands.py b/src/azure-cli/azure/cli/command_modules/appservice/commands.py index aea1c45fb55..4494b187ec0 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/commands.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/commands.py @@ -69,6 +69,55 @@ def transform_troubleshoot_config_output(result): ]) for s in settings if isinstance(s, dict)] +def transform_troubleshoot_status_output(result): + """Flatten the nested `instances` payload into one row per worker for `-o table`. + Column layout: InstanceId / State / (LastError / LastErrorTimestamp only when + any row has an error) / Successful / Failed / Updated. + + The framework's default table renderer would only surface top-level scalars + (name, resourceGroup) and drop every meaningful field.""" + from collections import OrderedDict + from .custom import _format_dt, _most_recent_startup, _startup_fetch_failed + + items = (result or {}).get('instances') or [] + + # LastError is nullable on the backend SiteRuntimeStatusOnWorker contract. + # A 'Started' instance is healthy, so any LastError it still carries is stale + # and must not be shown. Surface the LastError columns only when an instance + # reports a LastError while NOT 'Started'. + def _has_visible_error(item): + return bool(item.get('lastError')) and item.get('state') != 'Started' + + show_errors = any(_has_visible_error(item) for item in items) + + rows = [] + for item in items: + startup = item.get('startup') or {} + # KuduLite returns a SummaryFetchStatus failure reason when it couldn't + # read this worker's log directory; count/timestamp fields are + # meaningless in that case. + has_startup_error = bool(_startup_fetch_failed(startup)) + succeeded = None if has_startup_error else startup.get('Succeeded') + failed = None if has_startup_error else startup.get('Failed') + updated = None if has_startup_error else _format_dt(_most_recent_startup(startup)) + + row = OrderedDict([ + ('InstanceId', item.get('instanceId')), + ('State', item.get('state')), + ]) + if show_errors: + if _has_visible_error(item): + row['LastError'] = item.get('lastError') + row['LastErrorTimestamp'] = item.get('lastErrorTimestamp') + else: + row['LastError'] = None + row['LastErrorTimestamp'] = None + row['Succeeded'] = succeeded + row['Failed'] = failed + row['Updated'] = updated + rows.append(row) + return rows + def ex_handler_factory(creating_plan=False): def _ex_handler(ex): @@ -284,6 +333,8 @@ def load_command_table(self, _): with self.command_group('webapp troubleshoot', is_preview=True) as g: g.custom_command('config', 'troubleshoot_config', table_transformer=transform_troubleshoot_config_output) + g.custom_command('status', 'troubleshoot_status', + table_transformer=transform_troubleshoot_status_output) with self.command_group('functionapp log deployment') as g: g.custom_show_command('show', 'show_deployment_log') diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index c0ea9baa758..710ccf046fa 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -5,6 +5,8 @@ import ast import base64 +import os +import sys import threading import time import re @@ -387,10 +389,12 @@ def create_webapp(cmd, resource_group_name, name, plan, runtime=None, startup_fi _enable_basic_auth(cmd, name, None, resource_group_name, basic_auth.lower()) # Only suggest deployment command when no deployment method is already configured - if not using_webapp_up and not any([container_image_name, deployment_container_image_name, - multicontainer_config_type, sitecontainers_app, - deployment_source_url, deployment_local_git]): - logger.warning("Webapp '%s' created. Deploy your code with: az webapp deploy", name) + if not using_webapp_up: + if not any([container_image_name, deployment_container_image_name, + multicontainer_config_type, sitecontainers_app, + deployment_source_url, deployment_local_git]): + logger.warning("Webapp '%s' created. Deploy your code with: az webapp deploy", name) + _log_webapp_status_tip(name, resource_group_name, is_linux) return webapp @@ -2462,6 +2466,28 @@ def show_app(cmd, resource_group_name, name, slot=None): return app +def _log_webapp_status_tip(name, resource_group_name, is_linux): + # Per-instance runtime status (siteStatus) is a Linux App Service feature, + # so only surface the tip for Linux webapps. + if not is_linux: + return + logger.warning("Tip: run 'az webapp status --name %s --resource-group %s' " + "to see per-instance runtime status.", + name, resource_group_name) + + +def _extract_webapp_status_items(result): + # The siteStatus response holds per-instance status under 'properties': + # a list for /siteStatus, a single object for /siteStatus/{instanceId}. + # Normalize both shapes into a list for uniform formatting. + properties = result.get('properties') + if isinstance(properties, list): + return properties + if isinstance(properties, dict): + return [properties] + return [] + + def _list_app(cli_ctx, resource_group_name=None, show_details=False): client = web_client_factory(cli_ctx) if resource_group_name: @@ -6456,6 +6482,11 @@ def list_deployment_logs(cmd, resource_group, name, slot=None): def _ensure_linux_webapp_for_startup_logs(cmd, resource_group, name, slot=None): + _ensure_linux_webapp(cmd, resource_group, name, slot, + command_label="'az webapp log startup'") + + +def _ensure_linux_webapp(cmd, resource_group, name, slot=None, command_label='This command'): client = web_client_factory(cmd.cli_ctx) if slot: app = client.web_apps.get_slot(resource_group, name, slot) @@ -6463,7 +6494,7 @@ def _ensure_linux_webapp_for_startup_logs(cmd, resource_group, name, slot=None): app = client.web_apps.get(resource_group, name) if app is None or not is_linux_webapp(app): raise ArgumentUsageError( - "'az webapp log startup' is only supported for Linux web apps.") + "{} is only supported for Linux web apps.".format(command_label)) def list_startup_logs(cmd, resource_group, name, slot=None, outcome=None, instance=None): @@ -6883,6 +6914,498 @@ def _last_error_within(minutes): _row((Style.PRIMARY, '─' * 75)) + +# --------------------------------------------------------------------------- +# az webapp troubleshoot status +# --------------------------------------------------------------------------- + +def troubleshoot_status(cmd, resource_group, name, slot=None, instance=None, report=False): + """Fetch runtime + startup status for a Linux web app. + + Data sources: + * Site Runtime Status comes from ARM: + GET /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web + /sites/{name}[/slots/{slot}]/siteStatus[/{instanceId}]?api-version=... + * Startup summary comes from KuduLite (SCM): + GET https://{scm-host}/api/startuplogs/summary[?instance={id}] + + By default returns the structured payload (list of instances + startup + summary) so the standard `-o json/yaml/tsv/table` formatters handle output. + Pass --report to print the human-readable report to stdout instead. + """ + import requests + from azure.cli.core.commands.client_factory import get_subscription_id + from azure.core.exceptions import HttpResponseError as _Hre + + _ensure_linux_webapp(cmd, resource_group, name, slot, + command_label="'az webapp troubleshoot status'") + + # --- 1. Map ARM hex instanceId <-> friendly machineName via ARM /instances. + # We fetch this first so we can accept either form on --instance and resolve + # the right value before calling /siteStatus (ARM) and /api/startuplogs/summary (SCM). + client = web_client_factory(cmd.cli_ctx) + subscription_id = get_subscription_id(cmd.cli_ctx) + api_version = client._config.api_version + slot_segment = '/slots/{}'.format(slot) if slot else '' + instances_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/instances?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group, name=name, + slot_seg=slot_segment, ver=api_version) + id_to_machine = {} + machine_to_id = {} + try: + instances_payload = send_raw_request(cmd.cli_ctx, 'GET', instances_url).json() + for entry in instances_payload.get('value') or []: + entry_name = entry.get('name') + machine = (entry.get('properties') or {}).get('machineName') + if entry_name and machine: + id_to_machine[entry_name] = machine + machine_to_id[machine] = entry_name + except _Hre as ex: + logger.warning("Failed to retrieve machine names from '%s': %s", instances_url, ex) + + # Resolve --instance: accept either hex GUID (ARM form) or machineName (SCM form). + arm_instance_id = instance + if instance and instance in machine_to_id: + arm_instance_id = machine_to_id[instance] + elif instance and instance not in id_to_machine and machine_to_id: + # User passed something that matches neither known id nor machineName. + raise ResourceNotFoundError( + "Instance '{}' was not found for this webapp. " + "Run 'az webapp list-instances' to see available instance IDs.".format(instance)) + + # --- 2. Site Runtime Status from ARM /siteStatus[/{hex-instanceId}] --- + instance_segment = '/{}'.format(arm_instance_id) if arm_instance_id else '' + arm_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/siteStatus{inst_seg}?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group, name=name, + slot_seg=slot_segment, inst_seg=instance_segment, ver=api_version) + + try: + arm_response = send_raw_request(cmd.cli_ctx, 'GET', arm_url).json() + except _Hre as ex: + if instance and getattr(ex, 'status_code', None) == 404: + raise ResourceNotFoundError( + "Instance '{}' was not found for this webapp. " + "Run 'az webapp list-instances' to see available instance IDs.".format(instance)) + raise + + runtime_items = _extract_webapp_status_items(arm_response) + for item in runtime_items: + machine = id_to_machine.get(item.get('instanceId')) + if machine: + item['machineName'] = machine + + # --- 3. Per-instance Startup summary from SCM /api/startuplogs/summary --- + # KuduLite's summary endpoint already returns one entry per instance + # ([{InstanceId, Startup}, ...]) for everything seen in the last 24h, so we + # make a single round-trip and pair the entries with runtime_items locally + # instead of fanning out one call per instance. + scm_url = _get_scm_url(cmd, resource_group, name, slot) + headers = get_scm_site_headers(cmd.cli_ctx, name, resource_group, slot) + + # When --instance was supplied we still pass the filter to KuduLite so it + # only walks the requested worker's log directory (and the response stays + # small). Otherwise we fetch the full set in one call. + target_machine = None + if instance and len(runtime_items) == 1: + target_machine = runtime_items[0].get('machineName') + + summary_url = '{}/api/startuplogs/summary'.format(scm_url) + if target_machine: + summary_url = '{}?instance={}'.format(summary_url, quote(target_machine, safe='')) + + startup_by_machine = {} + # KuduLite may return a plaintext (non-JSON) body when the endpoint can't + # produce a per-instance summary at all — for example when the customer sets + # WEBSITE_DISABLE_CONTAINER_STARTUP_LOGS=true. That message applies to every + # worker on the app, so we hoist it to the instance-level SummaryFetchStatus + # below (after per-instance JSON entries, if any, are already loaded). + app_wide_fetch_status = None + try: + summary_response = requests.get(summary_url, headers=headers, timeout=30) + except requests.RequestException as ex: + logger.warning("Failed to call '%s': %s", summary_url, ex) + summary_response = None + + if summary_response is not None: + if summary_response.status_code == 200: + try: + body = summary_response.json() + except ValueError: + body = None + if isinstance(body, list): + for entry in body: + if not isinstance(entry, dict): + continue + # KuduLite serializes its C# POCO with default settings -> PascalCase + # (InstanceId, Startup, Succeeded, MostRecentSuccess, ...). When KuduLite + # ships a [JsonPropertyName] / camelCase contract, switch the keys + # here (and in _print_startup_block / overview-table renderer) accordingly. + key = entry.get('InstanceId') + if not key: + continue + startup_by_machine[key] = entry.get('Startup') or entry + elif isinstance(body, dict): + # Tolerate a single-object response (older KuduLite shape). + inner = body.get('Startup') or body + key = body.get('InstanceId') or target_machine + if key: + startup_by_machine[key] = inner + else: + # Plaintext body (e.g. WEBSITE_DISABLE_CONTAINER_STARTUP_LOGS notice). + text = (summary_response.text or '').strip() + if text: + app_wide_fetch_status = text + else: + # Any non-200 response means KuduLite couldn't produce a startup + # summary for this app. In practice we see: + # * 404: newer KuduLite doesn't recognize the /summary route yet + # (feature not rolled out to this region / stamp). + # * 400 "Invalid startup log filename.": older KuduLite routes + # /api/startuplogs/{filename} — no /summary sub-route — so it + # treats 'summary' as a filename and rejects it. + # * 5xx / auth errors: transient or config problem. + # Surface the status + body so users aren't misled into thinking + # their site had no startup attempts. + body_snippet = (summary_response.text or '').strip() + if len(body_snippet) > 200: + body_snippet = body_snippet[:200] + '...' + reason = summary_response.reason or '' + detail = '{} {}'.format(summary_response.status_code, reason).strip() + if body_snippet: + detail = '{}: {}'.format(detail, body_snippet) + app_wide_fetch_status = ( + "Startup summary is not available for this app " + "(SCM returned {} for {}). This feature requires a platform " + "version that may not have rolled out to your app's region yet." + ).format(detail, summary_url) + + for item in runtime_items: + machine = item.get('machineName') + # Without machineName we can't correlate this runtime entry to a KuduLite + # entry (KuduLite keys its summaries by machine name). + per_instance = startup_by_machine.get(machine) if machine else None + if per_instance is None and app_wide_fetch_status: + # Synthesize a SummaryFetchStatus entry so the existing renderer / + # table transformer surface the app-wide message per instance. + per_instance = {'SummaryFetchStatus': app_wide_fetch_status} + item['startup'] = per_instance + + # --- 3. Assemble payload --- + payload = { + 'name': name, + 'resourceGroup': resource_group, + 'instances': runtime_items, + } + if report: + _render_troubleshoot_status(payload) + return None + return payload + + +def _render_troubleshoot_status(payload): + """Print the human-readable report (Site Runtime Status + per-instance Startup summary). + Invoked by 'az webapp troubleshoot status' when --report is passed.""" + from azure.cli.core.style import Style, print_styled_text + + instances = payload.get('instances') or [] + app_name = payload.get('name') or '' + resource_group = payload.get('resourceGroup') + + def _out(*objs): + print_styled_text(*objs, file=sys.stdout) + + if not instances: + _out("No runtime status reported for '{}'.".format(app_name)) + return + + _out('') + _out((Style.HIGHLIGHT, "Application status for {}.".format(app_name))) + _out('') + + # Overview table (skip when only one instance is present, e.g. --instance filter). + if len(instances) > 1: + col_widths = (14, 20, 12, 24) + header = "{:<{w0}}{:<{w1}}{:<{w2}}{}".format( + 'INSTANCE', 'MACHINE', 'STATE', 'UPDATED', + w0=col_widths[0], w1=col_widths[1], w2=col_widths[2]) + _out((Style.HIGHLIGHT, header)) + _out('-' * sum(col_widths)) + for inst in instances: + startup = inst.get('startup') or {} + updated = _format_dt(_most_recent_startup(startup)) or '-' + state = inst.get('state') or '-' + # Pad plain text first, then wrap the STATE segment in its style — the + # framework's color escapes don't perturb the visible column width. + _out([ + (Style.PRIMARY, '{:<{w}}'.format(_short_id(inst.get('instanceId')), w=col_widths[0])), + (Style.PRIMARY, '{:<{w}}'.format(inst.get('machineName') or '-', w=col_widths[1])), + (_state_style(state), '{:<{w}}'.format(state, w=col_widths[2])), + (Style.PRIMARY, updated), + ]) + _out('') + _out('') + + # Per-instance Site Runtime Status + Startup summary. + for inst in instances: + machine = inst.get('machineName') + label = machine if machine else _short_id(inst.get('instanceId')) + sep = '-' * 66 + _out(sep) + _out((Style.HIGHLIGHT, 'Instance {} Full Status Report'.format(label))) + _out(sep) + _out((Style.HIGHLIGHT, 'Last runtime status')) + _print_runtime_block(inst, _out) + _out('') + _out((Style.HIGHLIGHT, 'Startup summary (last 24h)')) + if not machine: + # Without machineName we couldn't query KuduLite for this instance, so + # distinguish the "couldn't ask" case from "asked, nothing recorded". + _out(' Startup summary unavailable: machine name could not be determined for this instance.') + _out('') + else: + _print_startup_block(inst.get('startup'), _out) + + # Hint footer — surfaced only when at least one instance has a real + # failure in the report's window (Failed > 0 + lastError set). + has_error = any( + (inst.get('lastError') and _failed_count(inst.get('startup')) > 0) + for inst in instances + ) + if has_error: + rg = resource_group or '' + _out((Style.WARNING, 'Hint:')) + _out(' Check application logs: az webapp log tail -n {} -g {}'.format(app_name, rg)) + _out(' Check startup logs: az webapp log startup show -n {} -g {}'.format(app_name, rg)) + + +def _state_style(state): + """Map a runtime state string to an azure-cli Style for print_styled_text.""" + from azure.cli.core.style import Style + if not state: + return Style.PRIMARY + s = state.lower() + if s == 'started': + return Style.SUCCESS + if s in ('stopped', 'failed', 'crashed', 'unhealthy'): + return Style.ERROR + if s in ('starting', 'pullingimage', 'pulling', 'pending'): + return Style.WARNING + return Style.PRIMARY + + +def _outcome_style(outcome): + from azure.cli.core.style import Style + if not outcome: + return Style.PRIMARY + o = outcome.upper() + if o == 'STARTED': + return Style.SUCCESS + if o in ('FAILED', 'CRASHED'): + return Style.ERROR + return Style.PRIMARY + + +def _count_style(count, kind): + """Style for a numeric count. kind='failed' -> ERROR when > 0; 'successful' -> SUCCESS when > 0. + Accepts either an int/str integer (e.g. 3, "3") or a KuduLite capped-count + string like "50+" (parsed as the leading integer for the > 0 test).""" + from azure.cli.core.style import Style + text = str(count) + try: + n = int(text) + except (TypeError, ValueError): + # Handle capped forms like "50+" — parse the leading digits. + import re as _re + m = _re.match(r'\d+', text) + n = int(m.group(0)) if m else None + if n is None: + return Style.PRIMARY, text + if kind == 'failed' and n > 0: + return Style.ERROR, text + if kind == 'successful' and n > 0: + return Style.SUCCESS, text + return Style.PRIMARY, text + + +def _short_id(instance_id): + """Truncate a long hex ARM instanceId for table display.""" + if not instance_id: + return '-' + if len(instance_id) > 12: + return instance_id[:10] + return instance_id + + +def _format_dt(value): + if not value: + return None + # Pass through ISO strings; trim sub-second/timezone noise for the table view. + if isinstance(value, str): + v = value.replace('T', ' ') + is_utc = v.endswith('Z') + if '.' in v: + v = v.split('.', 1)[0] + if is_utc: + if v.endswith('Z'): + v = v[:-1] + v = v + ' UTC' + elif '+' in v: + v = v.split('+', 1)[0] + return v + return str(value) + + +def _relative_age(iso_value): + """Return a short 'Nh Mm ago' / 'Nm ago' / 'just now' / 'in the future' string + for an ISO-8601 UTC timestamp, or None if the input is unparseable/missing.""" + if not iso_value or not isinstance(iso_value, str): + return None + from datetime import datetime, timezone + v = iso_value + if '.' in v: + # datetime.fromisoformat pre-3.11 chokes on fractional seconds with 'Z' — strip both. + head, _, tail = v.partition('.') + tz = '' + for suffix in ('Z', '+', '-'): + if suffix in tail: + idx = tail.find(suffix) + tz = tail[idx:] + break + v = head + tz + v = v.replace('Z', '+00:00') + try: + dt = datetime.fromisoformat(v) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - dt + total_seconds = int(delta.total_seconds()) + if total_seconds < 0: + return 'in the future' + if total_seconds < 60: + return 'just now' + minutes = total_seconds // 60 + if minutes < 60: + return '{}m ago'.format(minutes) + hours = minutes // 60 + rem_min = minutes % 60 + if hours < 24: + return '{}h {}m ago'.format(hours, rem_min) if rem_min else '{}h ago'.format(hours) + days = hours // 24 + rem_hr = hours % 24 + return '{}d {}h ago'.format(days, rem_hr) if rem_hr else '{}d ago'.format(days) + + +def _failed_count(startup): + """Parse Startup.Failed into an int (0 when missing/invalid). Accepts int, + numeric string, or KuduLite's capped '50+' form (leading digits win).""" + if not startup: + return 0 + raw = startup.get('Failed') + if raw is None: + return 0 + text = str(raw) + try: + return int(text) + except ValueError: + import re as _re + m = _re.match(r'\d+', text) + return int(m.group(0)) if m else 0 + + +def _print_runtime_block(inst, emit): + """Print one Site Runtime Status block from an ARM /siteStatus item.""" + from azure.cli.core.style import Style + if not inst: + emit(' (no runtime status reported)') + return + state = inst.get('state') or '-' + details = inst.get('details') or '-' + emit([(Style.PRIMARY, ' State '), (_state_style(state), state)]) + emit(' Details {}'.format(details)) + # LastError may be stale after a recovery, so only surface it when this + # worker has actually had failed startup attempts in the report's window. + has_visible_error = bool(inst.get('lastError')) and _failed_count(inst.get('startup')) > 0 + if has_visible_error: + last_error = inst.get('lastError') or '-' + last_error_details = inst.get('lastErrorDetails') or '-' + # Treat .NET DateTime.MinValue (0001-01-01...) as "no error ever" and hide it. + last_error_ts_raw = inst.get('lastErrorTimestamp') + if isinstance(last_error_ts_raw, str) and last_error_ts_raw.startswith('0001-01-01'): + last_error_ts_raw = None + last_error_ts = _format_dt(last_error_ts_raw) or '-' + age = _relative_age(last_error_ts_raw) if last_error_ts_raw else None + if age: + last_error_ts = '{} ({})'.format(last_error_ts, age) + emit(' Last Error {}'.format(last_error)) + emit(' Last Error Details {}'.format(last_error_details)) + emit(' Last Error Timestamp {}'.format(last_error_ts)) + + +def _most_recent_startup(startup): + """Return the most recent of MostRecentSuccess / MostRecentFailure (ISO strings), + or None if both are missing. Lexicographic max is correct for RFC3339/ISO-8601 UTC.""" + if not startup: + return None + candidates = [ts for ts in (startup.get('MostRecentSuccess'), + startup.get('MostRecentFailure')) if ts] + return max(candidates) if candidates else None + + +def _startup_fetch_failed(startup): + """Return the SummaryFetchStatus string only when it indicates a fetch failure + (i.e. not the KuduLite success sentinel). None means success or missing.""" + if not startup: + return None + status = startup.get('SummaryFetchStatus') + if not status: + return None + # KuduLite success sentinel starts with 'Successfully'; anything else is a + # user-facing failure reason we want to surface. + if str(status).startswith('Successfully'): + return None + return status + + +def _print_startup_block(s, emit): + from azure.cli.core.style import Style + if not s: + emit(' No startup attempts recorded in the last 24 hours') + emit('') + return + # KuduLite sets SummaryFetchStatus to a failure reason when it couldn't read + # the log directory for this worker; other fields are meaningless then. + fetch_error = _startup_fetch_failed(s) + if fetch_error: + emit([(Style.PRIMARY, ' Startup summary unavailable: '), + (Style.WARNING, str(fetch_error))]) + emit('') + return + succeeded = s.get('Succeeded', 0) + failed = s.get('Failed', 0) + emit([(Style.PRIMARY, ' Succeeded '), _count_style(succeeded, 'successful')]) + emit([(Style.PRIMARY, ' Failed '), _count_style(failed, 'failed')]) + most_recent_success = _format_dt(s.get('MostRecentSuccess')) + most_recent_failure = _format_dt(s.get('MostRecentFailure')) + if most_recent_success: + emit([(Style.PRIMARY, ' Most recent success '), + (_outcome_style('STARTED'), most_recent_success)]) + if most_recent_failure: + emit([(Style.PRIMARY, ' Most recent failure '), + (_outcome_style('FAILED'), most_recent_failure)]) + emit('') + + + def config_slot_auto_swap(cmd, resource_group_name, webapp, slot, auto_swap_slot=None, disable=None): client = web_client_factory(cmd.cli_ctx) site_config = client.web_apps.get_configuration_slot(resource_group_name, webapp, slot) @@ -10287,6 +10810,7 @@ def _poll_deployment_runtime_status(cmd, resource_group_name, webapp_name, slot, time_elapsed = 0 deployment_status = None response_body = None + status_tip_logged = False while time_elapsed < max_time_sec: try: response_body = send_raw_request(cmd.cli_ctx, "GET", deploymentstatusapi_url).json() @@ -10301,12 +10825,19 @@ def _poll_deployment_runtime_status(cmd, resource_group_name, webapp_name, slot, status = deployment_status if status is None else status logger.warning("Status: %s Time: %s(s)", status, time_elapsed) if deployment_status == "RuntimeStarting": + if not status_tip_logged: + _log_webapp_status_tip(webapp_name, resource_group_name, True) + status_tip_logged = True logger.info("InprogressInstances: %s, SuccessfulInstances: %s", deployment_properties.get('numberOfInstancesInProgress'), deployment_properties.get('numberOfInstancesSuccessful')) if deployment_status == "RuntimeSuccessful": + if not status_tip_logged: + _log_webapp_status_tip(webapp_name, resource_group_name, True) break if deployment_status == "RuntimeFailed": + if not status_tip_logged: + _log_webapp_status_tip(webapp_name, resource_group_name, True) error_text = "" total_num_instances = int(deployment_properties.get('numberOfInstancesInProgress')) + \ int(deployment_properties.get('numberOfInstancesSuccessful')) + \ @@ -11190,6 +11721,8 @@ def webapp_up(cmd, name=None, resource_group_name=None, plan=None, location=None logger.warning("You can launch the app at %s", _url) create_json.update({'URL': _url}) + _log_webapp_status_tip(name, rg_name, _is_linux) + if logs: _configure_default_logging(cmd, rg_name, name) try: diff --git a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py index bad82015274..8694d5ced62 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py @@ -15,7 +15,8 @@ from azure.cli.core.azclierror import (InvalidArgumentValueError, MutuallyExclusiveArgumentError, ArgumentUsageError, - AzureResponseError) + AzureResponseError, + ResourceNotFoundError) from azure.cli.command_modules.appservice.custom import (set_deployment_user, update_git_token, add_hostname, update_site_configs, @@ -41,6 +42,7 @@ show_startup_log, troubleshoot_config, _extract_runtime_error, + troubleshoot_status, create_webapp) # pylint: disable=line-too-long @@ -972,6 +974,309 @@ def test_show_startup_log_404_with_instance(self, requests_get_mock, _scm_url_mo self.assertEqual(logger_mock.warning.call_args[0][1], 'lw0sdlwk000002') +class TestTroubleshootStatusMocked(unittest.TestCase): + """Tests for az webapp troubleshoot status (ARM siteStatus + SCM startuplogs/summary).""" + + def setUp(self): + is_linux_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=True) + client_factory_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.web_client_factory') + sub_id_patch = mock.patch( + 'azure.cli.core.commands.client_factory.get_subscription_id', + return_value='00000000-0000-0000-0000-000000000000') + self.client_factory_mock = client_factory_patch.start() + is_linux_patch.start() + sub_id_patch.start() + self.addCleanup(is_linux_patch.stop) + self.addCleanup(client_factory_patch.stop) + self.addCleanup(sub_id_patch.stop) + # Pin API version reported by the SDK config so URL assertions are stable. + self.client_factory_mock.return_value._config.api_version = '2025-05-01' + + self.cmd = _get_test_cmd() + self.cmd.cli_ctx.cloud.endpoints.resource_manager = 'https://management.azure.com' + + @staticmethod + def _arm_response(items): + return {'properties': items} + + @staticmethod + def _instances_payload(mapping): + """Build an ARM /instances response from {hex_id: machineName} mapping.""" + return {'value': [{'name': hex_id, 'properties': {'machineName': mn}} + for hex_id, mn in mapping.items()]} + + @staticmethod + def _make_response(status_code=200, json_data=None, reason='', text=''): + resp = mock.MagicMock() + resp.status_code = status_code + resp.reason = reason + resp.text = text + resp.json.return_value = json_data + return resp + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_all_instances(self, requests_get_mock, send_raw_request_mock, + _scm_url_mock, _headers_mock): + arm_items = [ + {'instanceId': 'a3f1b', 'state': 'Started', 'action': 'SiteStarted', + 'lastError': None, 'lastErrorDetails': None, 'lastErrorTimestamp': None, + 'details': 'Site is running', 'detailsLevel': 'Information'}, + {'instanceId': 'b4d22', 'state': 'Starting', 'action': 'PullingImage', + 'lastError': None, 'lastErrorDetails': None, 'lastErrorTimestamp': None, + 'details': 'Pulling image', 'detailsLevel': 'Warning'}, + ] + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'a3f1b': 'lw0sdlwk0008PB', 'b4d22': 'lw1sdlwk0009EF'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_items))), + ] + # Real KuduLite response is a single list with one entry per instance. + a3f1b_startup = {'Succeeded': 1, 'Failed': 0} + b4d22_startup = {'Succeeded': 0, 'Failed': 3} + requests_get_mock.return_value = self._make_response(200, json_data=[ + {'InstanceId': 'lw0sdlwk0008PB', 'Startup': a3f1b_startup}, + {'InstanceId': 'lw1sdlwk0009EF', 'Startup': b4d22_startup}, + ]) + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp') + + self.assertEqual(result['instances'][0]['startup'], a3f1b_startup) + self.assertEqual(result['instances'][1]['startup'], b4d22_startup) + self.assertEqual(result['instances'][0]['machineName'], 'lw0sdlwk0008PB') + self.assertEqual(result['instances'][1]['machineName'], 'lw1sdlwk0009EF') + # ARM calls: instances FIRST (so we can resolve --instance), then siteStatus. + arm_urls = [call.args[2] for call in send_raw_request_mock.call_args_list] + self.assertEqual(arm_urls, [ + 'https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000' + '/resourceGroups/myRG/providers/Microsoft.Web/sites/myApp/instances?api-version=2025-05-01', + 'https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000' + '/resourceGroups/myRG/providers/Microsoft.Web/sites/myApp/siteStatus?api-version=2025-05-01', + ]) + # Single unfiltered SCM call returns every instance in one response. + requests_get_mock.assert_called_once_with( + 'https://myapp.scm.azurewebsites.net/api/startuplogs/summary', + headers={'Authorization': 'Bearer token'}, + timeout=30, + ) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_single_instance(self, requests_get_mock, send_raw_request_mock, + _scm_url_mock, _headers_mock): + arm_item = {'instanceId': '7c2d9', 'state': 'Stopped', 'action': 'SiteStopped', + 'lastError': 'NoResponse', 'lastErrorDetails': 'Worker not reachable', + 'lastErrorTimestamp': '2026-05-20T18:50:44Z', + 'details': 'Stopped', 'detailsLevel': 'Error'} + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'7c2d9': 'lw0sdlwk0007AB'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_item))), + ] + startup_summary = {'Succeeded': 0, 'Failed': 4} + requests_get_mock.return_value = self._make_response( + 200, json_data=[{'InstanceId': 'lw0sdlwk0007AB', 'Startup': startup_summary}]) + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp', instance='7c2d9') + + self.assertEqual(result['instances'][0]['instanceId'], '7c2d9') + self.assertEqual(result['instances'][0]['startup'], startup_summary) + self.assertEqual(result['instances'][0]['machineName'], 'lw0sdlwk0007AB') + arm_urls = [call.args[2] for call in send_raw_request_mock.call_args_list] + self.assertEqual(arm_urls, [ + 'https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000' + '/resourceGroups/myRG/providers/Microsoft.Web/sites/myApp/instances?api-version=2025-05-01', + 'https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000' + '/resourceGroups/myRG/providers/Microsoft.Web/sites/myApp/siteStatus/7c2d9' + '?api-version=2025-05-01', + ]) + requests_get_mock.assert_called_once_with( + 'https://myapp.scm.azurewebsites.net/api/startuplogs/summary?instance=lw0sdlwk0007AB', + headers={'Authorization': 'Bearer token'}, + timeout=30, + ) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_summary_404_returns_empty_startup( + self, requests_get_mock, send_raw_request_mock, _scm_url_mock, _headers_mock): + arm_items = [{'instanceId': 'abcde', 'state': 'Started', 'action': 'SiteStarted'}] + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'abcde': 'lw0sdlwk0001AA'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_items))), + ] + requests_get_mock.return_value = self._make_response(404) + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp') + + # A 404 from the /api/startuplogs/summary endpoint means the KuduLite + # build doesn't recognize the route yet (feature not rolled out). + # Surface that as a SummaryFetchStatus so users aren't misled into + # thinking their site had no startup attempts. + startup = result['instances'][0]['startup'] + self.assertIsNotNone(startup) + self.assertIn('SCM returned 404', startup.get('SummaryFetchStatus', '')) + self.assertIn('not available', startup.get('SummaryFetchStatus', '').lower()) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_summary_400_invalid_filename_surfaces_message( + self, requests_get_mock, send_raw_request_mock, _scm_url_mock, _headers_mock): + # Older KuduLite build routes /api/startuplogs/{filename} and has no + # /summary sub-route, so it returns 400 "Invalid startup log filename." + # This should surface as feature-not-available, not as "no startups". + arm_items = [{'instanceId': 'abcde', 'state': 'Started', 'action': 'SiteStarted'}] + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'abcde': 'lw0sdlwk0001AA'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_items))), + ] + requests_get_mock.return_value = self._make_response( + 400, reason='BadRequest', text='Invalid startup log filename.') + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp') + + startup = result['instances'][0]['startup'] + self.assertIsNotNone(startup) + msg = startup.get('SummaryFetchStatus', '') + self.assertIn('SCM returned 400', msg) + self.assertIn('Invalid startup log filename', msg) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + def test_troubleshoot_status_arm_404_with_instance(self, send_raw_request_mock, + _scm_url_mock, _headers_mock): + error = HttpResponseError(message='Not found') + error.status_code = 404 + send_raw_request_mock.side_effect = error + + with self.assertRaises(ResourceNotFoundError): + troubleshoot_status(self.cmd, 'myRG', 'myApp', instance='7c2d9') + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_summary_500_surfaces_message( + self, requests_get_mock, send_raw_request_mock, _scm_url_mock, _headers_mock): + arm_items = [{'instanceId': 'abcde', 'state': 'Started', 'action': 'SiteStarted'}] + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'abcde': 'lw0sdlwk0001AA'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_items))), + ] + requests_get_mock.return_value = self._make_response(500, reason='Internal Server Error') + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp') + + # Non-200 -> feature-not-available message, not silent drop. + startup = result['instances'][0]['startup'] + self.assertIsNotNone(startup) + self.assertIn('SCM returned 500', startup.get('SummaryFetchStatus', '')) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + def test_troubleshoot_status_machine_name_as_instance( + self, requests_get_mock, send_raw_request_mock, _scm_url_mock, _headers_mock): + """User passes a friendly machineName for --instance; we should resolve it to + the hex ARM instanceId before calling /siteStatus.""" + arm_item = {'instanceId': '7c2d9', 'state': 'Started', 'action': 'SiteStarted'} + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'7c2d9': 'lw0sdlwk0007AB'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_item))), + ] + requests_get_mock.return_value = self._make_response( + 200, json_data=[{'InstanceId': 'lw0sdlwk0007AB', + 'Startup': {'Succeeded': 1, 'Failed': 0}}]) + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp', instance='lw0sdlwk0007AB') + + # ARM /siteStatus must use the hex id even though user passed the machine name. + arm_urls = [call.args[2] for call in send_raw_request_mock.call_args_list] + self.assertIn('/siteStatus/7c2d9?', arm_urls[1]) + self.assertEqual(result['instances'][0]['machineName'], 'lw0sdlwk0007AB') + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + def test_troubleshoot_status_unknown_instance_raises( + self, send_raw_request_mock, _scm_url_mock, _headers_mock): + """User passes an --instance that matches neither hex id nor machineName.""" + send_raw_request_mock.return_value = mock.MagicMock( + json=mock.MagicMock(return_value=self._instances_payload({'7c2d9': 'lw0sdlwk0007AB'}))) + + with self.assertRaises(ResourceNotFoundError): + troubleshoot_status(self.cmd, 'myRG', 'myApp', instance='does-not-exist') + + def test_troubleshoot_status_raises_on_windows(self): + with mock.patch('azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=False): + with self.assertRaises(ArgumentUsageError) as cm: + troubleshoot_status(self.cmd, 'myRG', 'myWindowsApp') + self.assertIn('Linux', str(cm.exception)) + + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': 'Bearer token'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('requests.get') + @mock.patch('azure.cli.command_modules.appservice.custom._render_troubleshoot_status') + def test_troubleshoot_status_report_flag_renders_and_returns_none( + self, render_mock, requests_get_mock, send_raw_request_mock, + _scm_url_mock, _headers_mock): + """With --report, command calls the renderer and returns None (no structured payload).""" + arm_item = {'instanceId': '7c2d9', 'state': 'Running'} + send_raw_request_mock.side_effect = [ + mock.MagicMock(json=mock.MagicMock(return_value=self._instances_payload( + {'7c2d9': 'lw0sdlwk0007AB'}))), + mock.MagicMock(json=mock.MagicMock(return_value=self._arm_response(arm_item))), + ] + requests_get_mock.return_value = self._make_response( + 200, json_data=[{'InstanceId': 'lw0sdlwk0007AB', + 'Startup': {'Succeeded': 1, 'Failed': 0}}]) + + result = troubleshoot_status(self.cmd, 'myRG', 'myApp', report=True) + + self.assertIsNone(result) + render_mock.assert_called_once() + rendered_payload = render_mock.call_args.args[0] + self.assertEqual(rendered_payload['name'], 'myApp') + self.assertEqual(rendered_payload['instances'][0]['instanceId'], '7c2d9') + + class TestRuntimeFailedHintMocked(unittest.TestCase): """Tests that the TIP hint appears in RuntimeFailed and timeout errors.""" From cf78e845aa7463de5d68f06a24ea485c9a0bdbeb Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:37:27 -0700 Subject: [PATCH 04/29] troubleshoot: dash for null most-recent success/failure + friendly config failure message Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cli/command_modules/appservice/custom.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 710ccf046fa..43ebaaaa33b 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6731,7 +6731,7 @@ def _consume(resp): if _consume(_try_config(cookies=cookies)): break - if config_check is None: + if config_check is None and not report: status = last_status if status == 404: if last_body_text: @@ -6813,6 +6813,7 @@ def _row(*objs): _out(list(objs)) config_check = payload.get('configCheck') or {} + config_check_failed = payload.get('configCheck') is None settings = config_check.get('Settings') or config_check.get('settings') or [] if not isinstance(settings, list): settings = [] @@ -6859,7 +6860,14 @@ def _last_error_within(minutes): # ---- Section 1: Built-in checks ---- _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) _out() - if not settings: + if config_check_failed: + _row((Style.WARNING, + 'Failed to retrieve built-in configuration checks. Restart the ' + 'application (\'az webapp restart\') and try again. If the issue ' + 'persists, confirm the app is running and reachable, verify you ' + 'have permission to call the SCM (Kudu) endpoint, and check for ' + 'an ongoing App Service incident in this region.')) + elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) else: setting_w = max(20, min(40, max(len(str(s.get('Setting') or '')) for s in settings) + 2)) @@ -7396,12 +7404,12 @@ def _print_startup_block(s, emit): emit([(Style.PRIMARY, ' Failed '), _count_style(failed, 'failed')]) most_recent_success = _format_dt(s.get('MostRecentSuccess')) most_recent_failure = _format_dt(s.get('MostRecentFailure')) - if most_recent_success: - emit([(Style.PRIMARY, ' Most recent success '), - (_outcome_style('STARTED'), most_recent_success)]) - if most_recent_failure: - emit([(Style.PRIMARY, ' Most recent failure '), - (_outcome_style('FAILED'), most_recent_failure)]) + emit([(Style.PRIMARY, ' Most recent success '), + (_outcome_style('STARTED') if most_recent_success else Style.PRIMARY, + most_recent_success or '-')]) + emit([(Style.PRIMARY, ' Most recent failure '), + (_outcome_style('FAILED') if most_recent_failure else Style.PRIMARY, + most_recent_failure or '-')]) emit('') From 6c6fab4d83939ba494cd8c6dbca57f157ed20e50 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:42:42 -0700 Subject: [PATCH 05/29] troubleshoot config: shorter failure message Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cli/command_modules/appservice/custom.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 43ebaaaa33b..4e169e85e66 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6863,10 +6863,8 @@ def _last_error_within(minutes): if config_check_failed: _row((Style.WARNING, 'Failed to retrieve built-in configuration checks. Restart the ' - 'application (\'az webapp restart\') and try again. If the issue ' - 'persists, confirm the app is running and reachable, verify you ' - 'have permission to call the SCM (Kudu) endpoint, and check for ' - 'an ongoing App Service incident in this region.')) + 'application (\'az webapp restart\') and confirm the SCM (Kudu) ' + 'is running and reachable.')) elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) else: From f293fb10f5406b90e4e5fb3d1229269baa6bacc7 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:55:12 -0700 Subject: [PATCH 06/29] troubleshoot config: drop trailing blank line when runtime section not rendered Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure-cli/azure/cli/command_modules/appservice/custom.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 4e169e85e66..e13ed1711fb 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6887,8 +6887,6 @@ def _last_error_within(minutes): style = Style.WARNING if _issue_detected(setting) else Style.SUCCESS _row((style, line)) - _out() - # ---- Section 2: Site runtime error recommendation ---- # Rendered when the built-in checks flagged at least one setting with # IssueDetected == true, OR when the ARM lastErrorTimestamp is within the @@ -6897,6 +6895,7 @@ def _last_error_within(minutes): if not show_runtime: return + _out() _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: From c092b872f9740e5faa4edf8d9074f85835749bbf Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 10:24:57 -0700 Subject: [PATCH 07/29] troubleshoot config: report polish - Add extra blank line above SITE RUNTIME ERROR RECOMMENDATION. - Render 'No runtime error reported.' in primary (white) instead of green. - Reword SCM failure message to lead with retry before restart. - Drop trailing horizontal rule at end of report. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cli/command_modules/appservice/custom.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index e13ed1711fb..23a9317e5ab 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6862,8 +6862,8 @@ def _last_error_within(minutes): _out() if config_check_failed: _row((Style.WARNING, - 'Failed to retrieve built-in configuration checks. Restart the ' - 'application (\'az webapp restart\') and confirm the SCM (Kudu) ' + 'Failed to retrieve built-in configuration checks. Please try again. ' + 'If the issue persists, restart the application (\'az webapp restart\') and confirm the SCM (Kudu) ' 'is running and reachable.')) elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) @@ -6895,11 +6895,12 @@ def _last_error_within(minutes): if not show_runtime: return + _out() _out() _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: - _row((Style.SUCCESS, 'No runtime error reported.')) + _row((Style.PRIMARY, 'No runtime error reported.')) else: state = str(runtime_error.get('state') or '') last_error = str(runtime_error.get('lastError') or '') @@ -6916,8 +6917,6 @@ def _last_error_within(minutes): _row((Style.PRIMARY, ' Last Error Timestamp: '), (Style.WARNING, timestamp)) _out() - _row((Style.PRIMARY, '─' * 75)) - # --------------------------------------------------------------------------- From c8a8448e886cbc591e10a475842f0710c61b8359 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 14:04:38 -0700 Subject: [PATCH 08/29] troubleshoot config + status: retries, correlation, wrapping, hints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combined update to the troubleshoot config and status commands. troubleshoot config: - Retry SCM config fetch on transient 5xx / connection errors (3 attempts, 1.5s backoff). - Read KuduLite's new DetailsLevel field (info/warning/error) with legacy IssueDetected fallback; map to SUCCESS/WARNING/ERROR styles. - Restyle the built-in checks table: HIGHLIGHT header, SECONDARY underlines, per-row Details colored by level; wrap Details column so long values hang-indent under the Details column. - Runtime error section: hanging-indent wrapping for State / Last Error / Last Error Details / Last Error Timestamp, plus relative age next to the timestamp. - Gate runtime section on last-error-within-15-min OR config-check-failed (never on stale flagged settings alone). - Add '▶ Hint:' footer with az webapp config appsettings set / az webapp config set / az webapp log tail commands when any check is warning/error. - _help.py: drop KuduLite / ARM implementation details from long-summary. - Tests: transient-5xx retry cases, config-check-failed keeps runtime section case, refresh stale-timestamp fixtures. troubleshoot status: - Correlate ARM /siteStatus items with KuduLite startup summaries by machineName first, then fall back to cardinality pairing for the common single-instance case; leftover SCM entries surface as an orphanStartups bucket. - When ARM's machineName and SCM's InstanceId disagree, per-instance header shows 'Instance Full Status Report (SCM: )'. - _emit_labeled hanging-indent helper wraps long values under the value column. - Switch overview table header + underline to Style.PRIMARY; '- * 76' separators around instance headers; upgrade Hint: label to '▶ Hint:'. - Suppress the 'machine name could not be determined' fallback when cardinality pairing succeeded. - _help.py: reword status long-summary to match config phrasing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da04b3ec-d5e4-4ab8-aeea-58dd7010ac84 --- .../cli/command_modules/appservice/_help.py | 19 +- .../cli/command_modules/appservice/custom.py | 306 ++++++++++++++---- .../latest/test_webapp_commands_thru_mock.py | 98 +++++- 3 files changed, 342 insertions(+), 81 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_help.py b/src/azure-cli/azure/cli/command_modules/appservice/_help.py index 847da439b86..610180972cb 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_help.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_help.py @@ -2448,19 +2448,18 @@ long-summary: > Aggregates two data sources into a single report: - (1) Built-in configuration checks from KuduLite — a set of common + (1) Built-in configuration checks — a set of common Linux App Service settings (linuxFxVersion, port binding, startup command, alwaysOn, health check path, ...) evaluated against the running site's configuration snapshot. - (2) The most recent runtime error reported by Azure Resource Manager - (`/siteStatus`) — echoed verbatim from the platform. + (2) The most recent site runtime status error reported by App Service By default the command returns a structured payload so the standard - `-o json/yaml/tsv/table` formatters handle output. Pass `--report` to + `-o json/yaml/table` formatters handle output. Pass `--report` to print a human-readable two-section report to stdout instead. examples: - - name: Run the built-in configuration checks and show the runtime error, if any + - name: Run the built-in configuration checks and show the runtime error, if any (JSON by default) text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup - name: Print the human-readable report text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --report @@ -2474,15 +2473,15 @@ long-summary: | Aggregates two data sources: - - Site Runtime Status: ARM /siteStatus[/{instanceId}] (per-instance state, - action, last error, details). + - Site Runtime Status - Startup summary: KuduLite (SCM) /api/startuplogs/summary (counts of successful and failed startup attempts in the last 24h, plus the most recent success and failure timestamps). - Use --instance to scope both to a single worker. By default returns the - structured payload (works with -o json, -o yaml, and -o table). Pass - --report to print a human-readable, color-coded report instead. + Use --instance to scope both to a single worker. By default the command + returns a structured payload so the standard `-o json/yaml/table` formatters + handle output. Pass `--report` to + print a human-readable two-section report to stdout instead. examples: - name: Show status for all instances of a web app (JSON by default) text: az webapp troubleshoot status --name MyWebApp --resource-group MyResourceGroup diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 23a9317e5ab..fe158438a4b 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6671,6 +6671,16 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) last_body_text = '' tried_instances = [] + # SCM (Kudu) is occasionally slow to respond — especially when the app has + # alwaysOn=false (so Kudu itself cold-starts) or the container is thrashing + # during startup. Transient 5xx / timeouts / connection errors resolve after + # a short wait, so retry a few times with backoff before treating the + # failure as terminal. 404 is NOT transient — that's handled separately by + # the per-instance ARR walk below. + _TRANSIENT_STATUSES = {408, 425, 429, 500, 502, 503, 504} + _MAX_ATTEMPTS = 3 + _BACKOFF_SECONDS = 1.5 + def _try_config(cookies=None): try: return requests.get(config_url, headers=headers, cookies=cookies, @@ -6679,6 +6689,18 @@ def _try_config(cookies=None): logger.warning("Failed to call '%s': %s", config_url, ex) return None + def _try_config_with_retry(cookies=None): + import time + resp = None + for attempt in range(1, _MAX_ATTEMPTS + 1): + resp = _try_config(cookies=cookies) + transient = (resp is None) or (resp.status_code in _TRANSIENT_STATUSES) + if not transient: + return resp + if attempt < _MAX_ATTEMPTS: + time.sleep(_BACKOFF_SECONDS * attempt) + return resp + def _consume(resp): nonlocal config_check, last_status, last_body_text if resp is None: @@ -6700,8 +6722,8 @@ def _consume(resp): config_url, snippet) return False - # First attempt: whichever worker ARR picks. - if not _consume(_try_config()): + # First attempt: whichever worker ARR picks (with transient-failure retry). + if not _consume(_try_config_with_retry()): # On 404, walk instances and retry with ARR affinity pinned to each. if last_status == 404: try: @@ -6728,7 +6750,7 @@ def _consume(resp): for instance_id in instance_ids: tried_instances.append(instance_id) cookies = {'ARRAffinity': instance_id, 'ARRAffinitySameSite': instance_id} - if _consume(_try_config(cookies=cookies)): + if _consume(_try_config_with_retry(cookies=cookies)): break if config_check is None and not report: @@ -6812,6 +6834,20 @@ def _out(*objs): def _row(*objs): _out(list(objs)) + def _labeled(label, value): + """Emit '