From 6ed29007513556a50627200bc9ff2f274fad6f42 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Tue, 7 Jul 2026 12:26:34 -0700 Subject: [PATCH 01/17] first push --- src/azure-cli/HISTORY.rst | 1 + .../cli/command_modules/appservice/_help.py | 36 +++ .../cli/command_modules/appservice/_params.py | 9 + .../command_modules/appservice/commands.py | 22 ++ .../cli/command_modules/appservice/custom.py | 208 ++++++++++++++++++ .../latest/test_webapp_commands_thru_mock.py | 180 +++++++++++++++ 6 files changed, 456 insertions(+) diff --git a/src/azure-cli/HISTORY.rst b/src/azure-cli/HISTORY.rst index 59fae05291c..990fc95395a 100644 --- a/src/azure-cli/HISTORY.rst +++ b/src/azure-cli/HISTORY.rst @@ -33,6 +33,7 @@ Release History * `az webapp list-runtimes`: Add `--runtime` and `--support` filter parameters (#32903) * [BREAKING CHANGE] `az webapp list-runtimes`: Remove deprecated `--linux` and `--show-runtime-details` parameters (#32903) * `az webapp log startup`: Add commands to list and view Linux container startup logs (#33256) +* `az webapp troubleshoot config`: Add preview command that runs built-in configuration checks against a Linux web app and surfaces the last runtime error from ARM * `az webapp create`: Add `--site-scoped-certs` parameter to support enabling or disabling site-scoped certificates (#33306) * `az webapp up`: Add warning message for future deprecation (#33410) * `az functionapp deployment source config-zip`: Fix `KeyError` `'FUNCTIONS_WORKER_RUNTIME'` for Go function apps on Flex Consumption (#33404) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_help.py b/src/azure-cli/azure/cli/command_modules/appservice/_help.py index eea863ae35d..6c173d22213 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_help.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_help.py @@ -2432,6 +2432,42 @@ text: az webapp log startup show --name MyWebApp --resource-group MyResourceGroup --instance lw0sdlwk000002 """ +helps['webapp troubleshoot'] = """ +type: group +short-summary: Diagnose common Linux web app problems. +long-summary: > + Preview command group that pairs built-in configuration checks (from + KuduLite on the worker) with the most recent runtime error reported by + Azure Resource Manager. Use when a Linux app is failing to start, returning + HTTP 502/503, or exhibiting other post-deployment misbehavior. +""" + +helps['webapp troubleshoot config'] = """ +type: command +short-summary: Validate configuration for a Linux web app and surface the last runtime error. +long-summary: > + Aggregates two data sources into a single report: + + (1) Built-in configuration checks from KuduLite — a set of common + Linux App Service settings (linuxFxVersion, port binding, startup + command, alwaysOn, health check path, ...) evaluated against the + running site's configuration snapshot. + + (2) The most recent runtime error reported by Azure Resource Manager + (`/siteStatus`) — echoed verbatim from the platform. + + By default the command returns a structured payload so the standard + `-o json/yaml/tsv/table` formatters handle output. Pass `--report` to + print a human-readable two-section report to stdout instead. +examples: + - name: Run the built-in configuration checks and show the runtime error, if any + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup + - name: Print the human-readable report + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --report + - name: Target a deployment slot + text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --slot staging +""" + helps['functionapp log'] = """ type: group short-summary: Manage function app logs. diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_params.py b/src/azure-cli/azure/cli/command_modules/appservice/_params.py index 7958d119c36..fb88f4db0a1 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_params.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_params.py @@ -849,6 +849,15 @@ def load_arguments(self, _): with self.argument_context('webapp log startup show') as c: c.argument('filename', options_list=['--filename', '-f'], help='Name of a specific startup log file to display. If not specified, shows the latest log (preferring failures).') + with self.argument_context('webapp troubleshoot') as c: + c.argument('name', arg_type=webapp_name_arg_type, id_part=None) + c.argument('resource_group_name', arg_type=resource_group_name_type) + c.argument('slot', options_list=['--slot', '-s'], help="the name of the slot. Default to the production slot if not specified") + + with self.argument_context('webapp troubleshoot config') as c: + c.argument('report', options_list=['--report'], arg_type=get_three_state_flag(), + help='Print a human-readable report instead of the structured payload.') + with self.argument_context('functionapp log deployment show') as c: c.argument('name', arg_type=functionapp_name_arg_type, id_part=None) c.argument('resource_group', arg_type=resource_group_name_type) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/commands.py b/src/azure-cli/azure/cli/command_modules/appservice/commands.py index 15462365f96..ce3b6e8c2ab 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/commands.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/commands.py @@ -48,6 +48,24 @@ def transform_runtime_list_output(result): ]) for r in result] +def transform_troubleshoot_config_output(result): + """Flatten the troubleshoot config payload into a per-setting table. + + Falls back gracefully for non-dict / empty payloads (e.g. --report was + passed and the command returned ``None``). + """ + from collections import OrderedDict + if not isinstance(result, dict): + return [] + settings = result.get('settings') or [] + return [OrderedDict([ + ('Setting', s.get('Setting') or s.get('setting') or ''), + ('Value', s.get('Value') if s.get('Value') is not None else s.get('value') or ''), + ('Details', s.get('Details') or s.get('details') or ''), + ]) for s in settings if isinstance(s, dict)] + + + def ex_handler_factory(creating_plan=False): def _ex_handler(ex): ex = _polish_bad_errors(ex, creating_plan) @@ -259,6 +277,10 @@ def load_command_table(self, _): g.custom_command('list', 'list_startup_logs') g.custom_show_command('show', 'show_startup_log') + with self.command_group('webapp troubleshoot', is_preview=True) as g: + g.custom_command('config', 'troubleshoot_config', + table_transformer=transform_troubleshoot_config_output) + with self.command_group('functionapp log deployment') as g: g.custom_show_command('show', 'show_deployment_log') g.custom_command('list', 'list_deployment_logs') diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 31d333d7d28..8a2a02c24e3 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6516,6 +6516,214 @@ def show_startup_log(cmd, resource_group, name, slot=None, filename=None, instan return response.json() +# ----------------------------------------------------------------------------- +# az webapp troubleshoot config +# ----------------------------------------------------------------------------- + +def _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot=None): + client = web_client_factory(cmd.cli_ctx) + if slot: + app = client.web_apps.get_slot(resource_group_name, name, slot) + else: + app = client.web_apps.get(resource_group_name, name) + if app is None or not is_linux_webapp(app): + raise ArgumentUsageError( + "'az webapp troubleshoot config' is only supported for Linux web apps.") + + +def _extract_runtime_error(arm_response): + """Return the freshest runtime-error block from an ARM /siteStatus response. + + /siteStatus returns per-instance status under 'properties' (a list); the + single-instance form returns a dict. We normalize both, then pick the entry + with the latest ``lastErrorTimestamp`` that also has a non-empty + ``lastError``. Returns ``None`` when no runtime error is reported. + """ + if not isinstance(arm_response, dict): + return None + properties = arm_response.get('properties') + if isinstance(properties, list): + items = properties + elif isinstance(properties, dict): + items = [properties] + else: + return None + candidates = [item for item in items if isinstance(item, dict) and item.get('lastError')] + if not candidates: + return None + + def _ts_key(item): + return item.get('lastErrorTimestamp') or '' + + candidates.sort(key=_ts_key, reverse=True) + return candidates[0] + + +def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False): + """Aggregate built-in KuduLite config-check findings plus the most recent ARM + /siteStatus runtime error for a Linux web app. + + Data sources: + * Built-in checks come from KuduLite (SCM): + GET https://{scm-host}/api/troubleshoot/config + * Last runtime error comes from ARM: + GET /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web + /sites/{name}[/slots/{slot}]/siteStatus?api-version=... + + By default returns the structured payload so the standard + ``-o json/yaml/tsv/table`` formatters handle output. Pass ``--report`` to + print the human-readable two-section report to stdout instead. + """ + import requests + from azure.cli.core.commands.client_factory import get_subscription_id + from azure.core.exceptions import HttpResponseError as _Hre + + _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot) + + # ---- 1. Built-in checks from KuduLite ---- + scm_url = _get_scm_url(cmd, resource_group_name, name, slot) + headers = get_scm_site_headers(cmd.cli_ctx, name, resource_group_name, slot) + config_url = '{}/api/troubleshoot/config'.format(scm_url) + + built_in = {'siteName': None, 'instanceId': None, 'writtenAt': None, 'settings': []} + try: + config_response = requests.get(config_url, headers=headers, timeout=30) + except requests.RequestException as ex: + logger.warning("Failed to call '%s': %s", config_url, ex) + config_response = None + + if config_response is not None: + if config_response.status_code == 200: + try: + body = config_response.json() + except ValueError: + body = None + if isinstance(body, dict): + built_in['siteName'] = body.get('SiteName') or body.get('siteName') + built_in['instanceId'] = body.get('InstanceId') or body.get('instanceId') + built_in['writtenAt'] = body.get('WrittenAt') or body.get('writtenAt') + settings = body.get('Settings') or body.get('settings') or [] + if isinstance(settings, list): + built_in['settings'] = [s for s in settings if isinstance(s, dict)] + elif config_response.status_code == 404: + # Endpoint not rolled out yet or app has never written a config snapshot. + logger.warning( + 'Built-in configuration checks are not available for this app. ' + "This feature requires a platform version that may not have rolled " + "out to your app's region yet.") + else: + logger.warning( + "Failed to retrieve built-in configuration checks from '%s' " + "(status %s %s).", + config_url, config_response.status_code, config_response.reason) + + # ---- 2. Site runtime status from ARM /siteStatus ---- + client = web_client_factory(cmd.cli_ctx) + subscription_id = get_subscription_id(cmd.cli_ctx) + # pylint: disable=protected-access + api_version = client._config.api_version + # pylint: enable=protected-access + slot_segment = '/slots/{}'.format(slot) if slot else '' + arm_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/siteStatus?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group_name, name=name, + slot_seg=slot_segment, ver=api_version) + + runtime_error = None + try: + arm_response = send_raw_request(cmd.cli_ctx, 'GET', arm_url).json() + runtime_error = _extract_runtime_error(arm_response) + except _Hre as ex: + logger.warning("Failed to retrieve site runtime status from '%s': %s", arm_url, ex) + except ValueError as ex: + logger.warning("Failed to parse site runtime status response: %s", ex) + + payload = { + 'name': name, + 'resourceGroup': resource_group_name, + 'siteName': built_in['siteName'], + 'instanceId': built_in['instanceId'], + 'writtenAt': built_in['writtenAt'], + 'settings': built_in['settings'], + 'runtimeError': runtime_error, + } + if report: + _render_troubleshoot_config(payload) + return None + return payload + + +def _render_troubleshoot_config(payload): + """Print the human-readable report: BUILT-IN CHECKS + SITE RUNTIME ERROR + RECOMMENDATION. Invoked when ``--report`` is passed.""" + from azure.cli.core.style import Style, print_styled_text + + def _out(*objs): + print_styled_text(*objs, file=sys.stdout) + + def _row(*objs): + _out(list(objs)) + + settings = payload.get('settings') or [] + runtime_error = payload.get('runtimeError') + + # ---- Section 1: Built-in checks ---- + _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) + _out() + if not settings: + _row((Style.WARNING, 'No built-in configuration checks reported.')) + else: + setting_w = max(20, min(40, max(len(str(s.get('Setting') or '')) for s in settings) + 2)) + value_w = max(15, min(30, max(len(str(s.get('Value') or '')) for s in settings) + 2)) + + header = '{sname:<{sw}}{vname:<{vw}}{dname}'.format( + sname='Setting', sw=setting_w, vname='Value', vw=value_w, dname='Details') + _row((Style.PRIMARY, header)) + _row((Style.PRIMARY, '{s}{v}{d}'.format( + s=('─' * (setting_w - 2)).ljust(setting_w), + v=('─' * (value_w - 2)).ljust(value_w), + d='─' * 40))) + for setting in settings: + name_v = str(setting.get('Setting') or '') + value_v = str(setting.get('Value') if setting.get('Value') is not None else '') + details_v = str(setting.get('Details') or '') + line = '{s:<{sw}}{v:<{vw}}{d}'.format( + s=name_v, sw=setting_w, v=value_v, vw=value_w, d=details_v) + # Highlight problem findings (anything other than "No issues detected"). + is_ok = details_v.strip().lower().startswith('no issues') + style = Style.SUCCESS if is_ok else Style.WARNING + _row((style, line)) + + _out() + + # ---- Section 2: Site runtime error recommendation ---- + _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) + _out() + if runtime_error is None: + _row((Style.SUCCESS, 'No runtime error reported.')) + else: + state = str(runtime_error.get('state') or '') + action = str(runtime_error.get('lastErrorAction') or runtime_error.get('action') or '') + last_error = str(runtime_error.get('lastError') or '') + details = str(runtime_error.get('lastErrorDetails') or runtime_error.get('details') or '') + + _row((Style.PRIMARY, 'Last runtime error')) + if state: + _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) + if action: + _row((Style.PRIMARY, ' Action: '), (Style.WARNING, action)) + if last_error: + _row((Style.PRIMARY, ' Error: '), (Style.ERROR, last_error)) + if details: + _row((Style.PRIMARY, ' Details: '), (Style.WARNING, details)) + + _out() + _row((Style.PRIMARY, '─' * 75)) + + def config_slot_auto_swap(cmd, resource_group_name, webapp, slot, auto_swap_slot=None, disable=None): client = web_client_factory(cmd.cli_ctx) site_config = client.web_apps.get_configuration_slot(resource_group_name, webapp, slot) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py index cc19fbcb865..6f9589f642c 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py @@ -37,6 +37,8 @@ update_webapp, list_startup_logs, show_startup_log, + troubleshoot_config, + _extract_runtime_error, create_webapp) # pylint: disable=line-too-long @@ -1605,5 +1607,183 @@ def test_get_visit_url_falls_back_when_no_cache(self, get_url_mock): get_url_mock.assert_called_once_with(params.cmd, 'myRG', 'myApp', None) +class TestTroubleshootConfigMocked(unittest.TestCase): + """Tests for az webapp troubleshoot config.""" + + def setUp(self): + is_linux_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=True) + client_factory_patch = mock.patch( + 'azure.cli.command_modules.appservice.custom.web_client_factory') + is_linux_patch.start() + cf_mock = client_factory_patch.start() + # Give the real code path something to unpack when it reads + # client._config.api_version. + cf_mock.return_value._config.api_version = '2022-03-01' + self.addCleanup(is_linux_patch.stop) + self.addCleanup(client_factory_patch.stop) + + def _scm_response(self, status_code=200, json_data=None, reason=''): + resp = mock.MagicMock() + resp.status_code = status_code + resp.reason = reason + resp.json.return_value = json_data + return resp + + def _arm_response(self, json_data): + resp = mock.MagicMock() + resp.json.return_value = json_data + return resp + + # ---- _extract_runtime_error ---- + + def test_extract_runtime_error_picks_latest_timestamp(self): + arm = {'properties': [ + {'state': 'Started', 'lastError': None}, + {'state': 'Stopped', 'lastError': 'A', 'lastErrorTimestamp': '2026-07-01T00:00:00Z'}, + {'state': 'Stopped', 'lastError': 'B', 'lastErrorTimestamp': '2026-07-02T00:00:00Z'}, + ]} + self.assertEqual(_extract_runtime_error(arm)['lastError'], 'B') + + def test_extract_runtime_error_returns_none_when_all_started(self): + arm = {'properties': [{'state': 'Started', 'lastError': None}]} + self.assertIsNone(_extract_runtime_error(arm)) + + def test_extract_runtime_error_handles_single_dict_properties(self): + arm = {'properties': {'state': 'Stopped', 'lastError': 'X'}} + self.assertEqual(_extract_runtime_error(arm)['lastError'], 'X') + + def test_extract_runtime_error_handles_missing_properties(self): + self.assertIsNone(_extract_runtime_error({})) + self.assertIsNone(_extract_runtime_error(None)) + + # ---- troubleshoot_config ---- + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, + _headers_mock, send_raw_request_mock): + settings = [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', 'Details': 'No issues detected'}, + {'Setting': 'alwaysOn', 'Value': 'false', 'Details': 'App may be unloaded when idle'}, + ] + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', + 'InstanceId': 'abc123', + 'WrittenAt': '2026-07-07T18:12:29+00:00', + 'Settings': settings, + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'Container did not respond', 'lastErrorAction': 'WaitingForSiteToStart', + 'lastErrorTimestamp': '2026-07-07T18:00:00Z'}, + ]}) + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertEqual(result['name'], 'myApp') + self.assertEqual(result['resourceGroup'], 'myRG') + self.assertEqual(result['settings'], settings) + self.assertEqual(result['runtimeError']['lastError'], 'ContainerTimeout') + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_scm_404_returns_empty_settings(self, requests_get_mock, + _scm_url_mock, _headers_mock, + send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(404) + send_raw_request_mock.return_value = self._arm_response({'properties': []}) + + with mock.patch('azure.cli.command_modules.appservice.custom.logger') as logger_mock: + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertEqual(result['settings'], []) + self.assertIsNone(result['runtimeError']) + logger_mock.warning.assert_called() + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_mock, + _headers_mock, send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [{'Setting': 'alwaysOn', 'Value': 'true', + 'Details': 'No issues detected'}], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Started', 'lastError': None}, + ]}) + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertIsNone(result['runtimeError']) + self.assertEqual(len(result['settings']), 1) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_mock, + _scm_url_mock, _headers_mock, + send_raw_request_mock): + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected'}, + {'Setting': 'alwaysOn', 'Value': 'false', + 'Details': 'App may be unloaded when idle'}, + ], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ImagePullUnauthorizedFailure', + 'lastErrorDetails': 'forbidden', 'lastErrorAction': 'StartingSiteContainers', + 'lastErrorTimestamp': '2026-07-07T18:00:00Z'}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + self.assertIsNone(result) + # BUILT-IN CHECKS + SITE RUNTIME ERROR RECOMMENDATION headers should + # have been printed. + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('BUILT-IN CHECKS', printed_text) + self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertIn('ImagePullUnauthorizedFailure', printed_text) + + def test_troubleshoot_config_raises_on_windows(self): + with mock.patch( + 'azure.cli.command_modules.appservice.custom.is_linux_webapp', + return_value=False): + with self.assertRaises(ArgumentUsageError) as cm: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myWindowsApp') + self.assertIn('Linux', str(cm.exception)) + + if __name__ == '__main__': unittest.main() From 40d2042dc5442d600fa66800ef2f19c956e98cb8 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Wed, 8 Jul 2026 13:56:07 -0700 Subject: [PATCH 02/17] output updates, nest config check --- .../command_modules/appservice/commands.py | 10 +- .../cli/command_modules/appservice/custom.py | 191 ++++++++++++++---- .../latest/test_webapp_commands_thru_mock.py | 162 ++++++++++++++- 3 files changed, 312 insertions(+), 51 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/commands.py b/src/azure-cli/azure/cli/command_modules/appservice/commands.py index ce3b6e8c2ab..60f6ed0551f 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/commands.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/commands.py @@ -51,13 +51,17 @@ def transform_runtime_list_output(result): def transform_troubleshoot_config_output(result): """Flatten the troubleshoot config payload into a per-setting table. - Falls back gracefully for non-dict / empty payloads (e.g. --report was - passed and the command returned ``None``). + Reads Settings out of the nested ``configCheck`` field (the verbatim SCM + body). Falls back gracefully for non-dict / empty payloads (e.g. --report + was passed and the command returned ``None``). """ from collections import OrderedDict if not isinstance(result, dict): return [] - settings = result.get('settings') or [] + config_check = result.get('configCheck') or {} + settings = config_check.get('Settings') or config_check.get('settings') or [] + if not isinstance(settings, list): + return [] return [OrderedDict([ ('Setting', s.get('Setting') or s.get('setting') or ''), ('Value', s.get('Value') if s.get('Value') is not None else s.get('value') or ''), diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 8a2a02c24e3..222817d5e3b 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6581,41 +6581,114 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) _ensure_linux_webapp_for_troubleshoot(cmd, resource_group_name, name, slot) # ---- 1. Built-in checks from KuduLite ---- + # + # KuduLite reads the config snapshot from ``/appsvctmp/config_check_{siteName}.json`` + # on the worker where the request lands. On multi-worker plans only the + # instance that most recently ran the site's startup pipeline has the + # file, so ARR-affinity routing to any other worker returns 404. Retry + # per-instance until one worker responds with data. scm_url = _get_scm_url(cmd, resource_group_name, name, slot) headers = get_scm_site_headers(cmd.cli_ctx, name, resource_group_name, slot) config_url = '{}/api/troubleshoot/config'.format(scm_url) - built_in = {'siteName': None, 'instanceId': None, 'writtenAt': None, 'settings': []} - try: - config_response = requests.get(config_url, headers=headers, timeout=30) - except requests.RequestException as ex: - logger.warning("Failed to call '%s': %s", config_url, ex) - config_response = None + config_check = None + last_status = None + last_body_text = '' + tried_instances = [] + + def _try_config(cookies=None): + try: + return requests.get(config_url, headers=headers, cookies=cookies, + timeout=30, allow_redirects=False) + except requests.RequestException as ex: + logger.warning("Failed to call '%s': %s", config_url, ex) + return None - if config_response is not None: - if config_response.status_code == 200: + def _consume(resp): + nonlocal config_check, last_status, last_body_text + if resp is None: + return False + last_status = resp.status_code + last_body_text = (resp.text or '').strip() + if last_status == 200: try: - body = config_response.json() + body = resp.json() except ValueError: body = None if isinstance(body, dict): - built_in['siteName'] = body.get('SiteName') or body.get('siteName') - built_in['instanceId'] = body.get('InstanceId') or body.get('instanceId') - built_in['writtenAt'] = body.get('WrittenAt') or body.get('writtenAt') - settings = body.get('Settings') or body.get('settings') or [] - if isinstance(settings, list): - built_in['settings'] = [s for s in settings if isinstance(s, dict)] - elif config_response.status_code == 404: - # Endpoint not rolled out yet or app has never written a config snapshot. + config_check = body + return True + snippet = last_body_text[:200] logger.warning( - 'Built-in configuration checks are not available for this app. ' - "This feature requires a platform version that may not have rolled " - "out to your app's region yet.") - else: + "Built-in configuration checks endpoint '%s' returned 200 but the body " + "wasn't the expected JSON object. First 200 chars: %r", + config_url, snippet) + return False + + # First attempt: whichever worker ARR picks. + if not _consume(_try_config()): + # On 404, walk instances and retry with ARR affinity pinned to each. + if last_status == 404: + try: + client = web_client_factory(cmd.cli_ctx) + # pylint: disable=protected-access + api_version = client._config.api_version + # pylint: enable=protected-access + subscription_id = get_subscription_id(cmd.cli_ctx) + slot_segment = '/slots/{}'.format(slot) if slot else '' + instances_url = ( + '{rm}/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web' + '/sites/{name}{slot_seg}/instances?api-version={ver}' + ).format( + rm=cmd.cli_ctx.cloud.endpoints.resource_manager, + sub=subscription_id, rg=resource_group_name, name=name, + slot_seg=slot_segment, ver=api_version) + instances_payload = send_raw_request(cmd.cli_ctx, 'GET', instances_url).json() + instance_ids = [e.get('name') for e in (instances_payload.get('value') or []) + if isinstance(e, dict) and e.get('name')] + except Exception as ex: # pylint: disable=broad-except + logger.warning('Failed to enumerate site instances for retry: %s', ex) + instance_ids = [] + + for instance_id in instance_ids: + tried_instances.append(instance_id) + cookies = {'ARRAffinity': instance_id, 'ARRAffinitySameSite': instance_id} + if _consume(_try_config(cookies=cookies)): + break + + if config_check is None: + status = last_status + if status == 404: + if last_body_text: + logger.warning( + "Built-in configuration checks endpoint returned 404 on all " + "workers (tried %d instance(s)). Response body: %s " + "This usually means the site container has not started " + "successfully yet, so no configuration snapshot was written. " + "See the runtime-error section below or run 'az webapp log tail' " + "for startup details.", + max(1, len(tried_instances)), last_body_text) + else: + logger.warning( + "Built-in configuration checks are not available for this app " + "(SCM returned 404 for %s). This feature requires a platform " + "version that may not have rolled out to your app's region yet.", + config_url) + elif status in (401, 403): + logger.warning( + "Access to built-in configuration checks was denied by the SCM " + "endpoint (status %s). Make sure basic auth is enabled for SCM on " + "this site, or that your credentials have SCM access.", status) + elif status in (301, 302, 303, 307, 308): + logger.warning( + "Built-in configuration checks endpoint '%s' returned a redirect " + "(status %s). This usually means SCM authentication is " + "misconfigured for this app.", config_url, status) + elif status is not None: logger.warning( "Failed to retrieve built-in configuration checks from '%s' " - "(status %s %s).", - config_url, config_response.status_code, config_response.reason) + "(status %s).", + config_url, status) # ---- 2. Site runtime status from ARM /siteStatus ---- client = web_client_factory(cmd.cli_ctx) @@ -6644,10 +6717,7 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) payload = { 'name': name, 'resourceGroup': resource_group_name, - 'siteName': built_in['siteName'], - 'instanceId': built_in['instanceId'], - 'writtenAt': built_in['writtenAt'], - 'settings': built_in['settings'], + 'configCheck': config_check, 'runtimeError': runtime_error, } if report: @@ -6667,9 +6737,50 @@ def _out(*objs): def _row(*objs): _out(list(objs)) - settings = payload.get('settings') or [] + config_check = payload.get('configCheck') or {} + settings = config_check.get('Settings') or config_check.get('settings') or [] + if not isinstance(settings, list): + settings = [] + settings = [s for s in settings if isinstance(s, dict)] runtime_error = payload.get('runtimeError') + def _issue_detected(setting): + # KuduLite emits IssueDetected as a JSON string ("True"/"False") today, + # but tolerate a real bool too in case the contract firms up later. + raw = setting.get('IssueDetected') + if raw is None: + raw = setting.get('issueDetected') + if isinstance(raw, bool): + return raw + if isinstance(raw, str): + return raw.strip().lower() == 'true' + return False + + any_issue = any(_issue_detected(s) for s in settings) + + def _last_error_within(minutes): + # Runtime error is considered "fresh" if its timestamp is within the + # last N minutes (UTC). ARM emits lastErrorTimestamp as an ISO 8601 + # string; tolerate a trailing 'Z' and missing tzinfo (treated as UTC). + if not runtime_error: + return False + raw = runtime_error.get('lastErrorTimestamp') + if not raw: + return False + from datetime import datetime, timezone, timedelta + try: + ts = str(raw).strip() + if ts.endswith('Z'): + ts = ts[:-1] + '+00:00' + parsed = datetime.fromisoformat(ts) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + return False + return (datetime.now(timezone.utc) - parsed) <= timedelta(minutes=minutes) + + show_runtime = any_issue or _last_error_within(15) + # ---- Section 1: Built-in checks ---- _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) _out() @@ -6692,33 +6803,37 @@ def _row(*objs): details_v = str(setting.get('Details') or '') line = '{s:<{sw}}{v:<{vw}}{d}'.format( s=name_v, sw=setting_w, v=value_v, vw=value_w, d=details_v) - # Highlight problem findings (anything other than "No issues detected"). - is_ok = details_v.strip().lower().startswith('no issues') - style = Style.SUCCESS if is_ok else Style.WARNING + style = Style.WARNING if _issue_detected(setting) else Style.SUCCESS _row((style, line)) _out() # ---- Section 2: Site runtime error recommendation ---- + # Rendered when the built-in checks flagged at least one setting with + # IssueDetected == true, OR when the ARM lastErrorTimestamp is within the + # last 15 minutes (fresh failures are worth surfacing even if the SCM + # checks look clean). + if not show_runtime: + return + _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: _row((Style.SUCCESS, 'No runtime error reported.')) else: state = str(runtime_error.get('state') or '') - action = str(runtime_error.get('lastErrorAction') or runtime_error.get('action') or '') last_error = str(runtime_error.get('lastError') or '') details = str(runtime_error.get('lastErrorDetails') or runtime_error.get('details') or '') + timestamp = str(runtime_error.get('lastErrorTimestamp') or '') - _row((Style.PRIMARY, 'Last runtime error')) if state: - _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) - if action: - _row((Style.PRIMARY, ' Action: '), (Style.WARNING, action)) + _row((Style.PRIMARY, ' State: '), (Style.WARNING, state)) if last_error: - _row((Style.PRIMARY, ' Error: '), (Style.ERROR, last_error)) + _row((Style.PRIMARY, ' Last Error: '), (Style.ERROR, last_error)) if details: - _row((Style.PRIMARY, ' Details: '), (Style.WARNING, details)) + _row((Style.PRIMARY, ' Last Error Details: '), (Style.WARNING, details)) + if timestamp: + _row((Style.PRIMARY, ' Last Error Timestamp: '), (Style.WARNING, timestamp)) _out() _row((Style.PRIMARY, '─' * 75)) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py index 6f9589f642c..f9b027ecaab 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/tests/latest/test_webapp_commands_thru_mock.py @@ -1672,12 +1672,13 @@ def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', 'Details': 'No issues detected'}, {'Setting': 'alwaysOn', 'Value': 'false', 'Details': 'App may be unloaded when idle'}, ] - requests_get_mock.return_value = self._scm_response(200, json_data={ + scm_body = { 'SiteName': 'myApp', 'InstanceId': 'abc123', 'WrittenAt': '2026-07-07T18:12:29+00:00', 'Settings': settings, - }) + } + requests_get_mock.return_value = self._scm_response(200, json_data=scm_body) send_raw_request_mock.return_value = self._arm_response({'properties': [ {'state': 'Stopped', 'lastError': 'ContainerTimeout', 'lastErrorDetails': 'Container did not respond', 'lastErrorAction': 'WaitingForSiteToStart', @@ -1688,7 +1689,9 @@ def test_troubleshoot_config_success(self, requests_get_mock, _scm_url_mock, self.assertEqual(result['name'], 'myApp') self.assertEqual(result['resourceGroup'], 'myRG') - self.assertEqual(result['settings'], settings) + # configCheck is the verbatim SCM body — PascalCase keys preserved. + self.assertEqual(result['configCheck'], scm_body) + self.assertEqual(result['configCheck']['Settings'], settings) self.assertEqual(result['runtimeError']['lastError'], 'ContainerTimeout') @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') @@ -1706,7 +1709,7 @@ def test_troubleshoot_config_scm_404_returns_empty_settings(self, requests_get_m with mock.patch('azure.cli.command_modules.appservice.custom.logger') as logger_mock: result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') - self.assertEqual(result['settings'], []) + self.assertIsNone(result['configCheck']) self.assertIsNone(result['runtimeError']) logger_mock.warning.assert_called() @@ -1730,7 +1733,8 @@ def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_ result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') self.assertIsNone(result['runtimeError']) - self.assertEqual(len(result['settings']), 1) + self.assertEqual(len(result['configCheck']['Settings']), 1) + self.assertEqual(result['configCheck']['SiteName'], 'myApp') @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', @@ -1741,13 +1745,16 @@ def test_troubleshoot_config_no_runtime_error(self, requests_get_mock, _scm_url_ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # At least one setting has IssueDetected: True so the runtime-error + # section will render. requests_get_mock.return_value = self._scm_response(200, json_data={ 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', 'Settings': [ {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', - 'Details': 'No issues detected'}, - {'Setting': 'alwaysOn', 'Value': 'false', - 'Details': 'App may be unloaded when idle'}, + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + {'Setting': 'websitesPort', 'Value': '', + 'Details': "App doesn't respond on expected port", + 'IssueDetected': 'True'}, ], }) send_raw_request_mock.return_value = self._arm_response({'properties': [ @@ -1761,8 +1768,6 @@ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_m result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) self.assertIsNone(result) - # BUILT-IN CHECKS + SITE RUNTIME ERROR RECOMMENDATION headers should - # have been printed. printed_text = '' for call in print_mock.call_args_list: for arg in call.args: @@ -1775,6 +1780,98 @@ def test_troubleshoot_config_report_prints_and_returns_none(self, requests_get_m self.assertIn('BUILT-IN CHECKS', printed_text) self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) self.assertIn('ImagePullUnauthorizedFailure', printed_text) + self.assertIn('Last Error:', printed_text) + self.assertIn('Last Error Details:', printed_text) + self.assertIn('Last Error Timestamp:', printed_text) + # Removed labels should NOT appear. + self.assertNotIn('Last runtime error', printed_text) + self.assertNotIn('Action:', printed_text) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_suppresses_runtime_when_no_issues( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # Every setting has IssueDetected == "False" -> runtime section suppressed. + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + {'Setting': 'alwaysOn', 'Value': 'true', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + ], + }) + # Even if ARM reports a stale runtime error, we should not render the + # recommendation section when no setting has an issue. + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Started', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'old error', 'lastErrorAction': 'None', + 'lastErrorTimestamp': '2026-07-01T00:00:00Z'}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('BUILT-IN CHECKS', printed_text) + self.assertNotIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertNotIn('ContainerTimeout', printed_text) + + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_report_shows_runtime_when_error_is_fresh( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # All settings clean, but ARM reports a runtime error whose timestamp + # is within the last 15 minutes -> section should still render. + from datetime import datetime, timezone, timedelta + fresh_ts = (datetime.now(timezone.utc) - timedelta(minutes=2)).strftime( + '%Y-%m-%dT%H:%M:%SZ') + requests_get_mock.return_value = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'abc', 'WrittenAt': 't', + 'Settings': [ + {'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected', 'IssueDetected': 'False'}, + ], + }) + send_raw_request_mock.return_value = self._arm_response({'properties': [ + {'state': 'Stopped', 'lastError': 'ContainerTimeout', + 'lastErrorDetails': 'fresh error', 'lastErrorAction': 'None', + 'lastErrorTimestamp': fresh_ts}, + ]}) + + with mock.patch( + 'azure.cli.core.style.print_styled_text') as print_mock: + troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp', report=True) + + printed_text = '' + for call in print_mock.call_args_list: + for arg in call.args: + if isinstance(arg, list): + for tup in arg: + if isinstance(tup, tuple) and len(tup) > 1: + printed_text += str(tup[1]) + elif isinstance(arg, tuple) and len(arg) > 1: + printed_text += str(arg[1]) + self.assertIn('SITE RUNTIME ERROR RECOMMENDATION', printed_text) + self.assertIn('ContainerTimeout', printed_text) + self.assertIn('fresh error', printed_text) def test_troubleshoot_config_raises_on_windows(self): with mock.patch( @@ -1784,6 +1881,51 @@ def test_troubleshoot_config_raises_on_windows(self): troubleshoot_config(_get_test_cmd(), 'myRG', 'myWindowsApp') self.assertIn('Linux', str(cm.exception)) + @mock.patch('azure.cli.command_modules.appservice.custom.send_raw_request') + @mock.patch('azure.cli.command_modules.appservice.custom.get_scm_site_headers', + return_value={'Authorization': '******'}) + @mock.patch('azure.cli.command_modules.appservice.custom._get_scm_url', + return_value='https://myapp.scm.azurewebsites.net') + @mock.patch('requests.get') + def test_troubleshoot_config_retries_across_instances_on_404( + self, requests_get_mock, _scm_url_mock, _headers_mock, send_raw_request_mock): + # First (unpinned) call and the first instance retry return 404 with + # KuduLite's "not available" body; the second instance retry returns + # the snapshot. + settings = [{'Setting': 'linuxFxVersion', 'Value': 'NODE|20-lts', + 'Details': 'No issues detected'}] + good = self._scm_response(200, json_data={ + 'SiteName': 'myApp', 'InstanceId': 'inst2', 'WrittenAt': 't', + 'Settings': settings, + }) + bad = mock.MagicMock() + bad.status_code = 404 + bad.text = 'Config check information is not available.' + bad.json.side_effect = ValueError('not json') + + requests_get_mock.side_effect = [bad, bad, good] + + send_raw_request_mock.side_effect = [ + # ARM /instances response (retry lookup). + self._arm_response({'value': [{'name': 'inst1'}, {'name': 'inst2'}]}), + # ARM /siteStatus response. + self._arm_response({'properties': []}), + ] + + result = troubleshoot_config(_get_test_cmd(), 'myRG', 'myApp') + + self.assertIsNotNone(result['configCheck']) + self.assertEqual(result['configCheck']['InstanceId'], 'inst2') + # 3 SCM calls: unpinned, then per-instance retry twice. + self.assertEqual(requests_get_mock.call_count, 3) + # Second and third calls should carry ARR affinity cookies. + self.assertEqual( + requests_get_mock.call_args_list[1].kwargs['cookies'], + {'ARRAffinity': 'inst1', 'ARRAffinitySameSite': 'inst1'}) + self.assertEqual( + requests_get_mock.call_args_list[2].kwargs['cookies'], + {'ARRAffinity': 'inst2', 'ARRAffinitySameSite': 'inst2'}) + if __name__ == '__main__': unittest.main() From b27ee474e872521d680ae74d719acbe26b1bda78 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:35:07 -0700 Subject: [PATCH 03/17] troubleshoot config: friendly retry message on SCM failure in --report mode Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cli/command_modules/appservice/custom.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 222817d5e3b..fc9004bcc15 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6656,7 +6656,7 @@ def _consume(resp): if _consume(_try_config(cookies=cookies)): break - if config_check is None: + if config_check is None and not report: status = last_status if status == 404: if last_body_text: @@ -6738,6 +6738,7 @@ def _row(*objs): _out(list(objs)) config_check = payload.get('configCheck') or {} + config_check_failed = payload.get('configCheck') is None settings = config_check.get('Settings') or config_check.get('settings') or [] if not isinstance(settings, list): settings = [] @@ -6784,7 +6785,14 @@ def _last_error_within(minutes): # ---- Section 1: Built-in checks ---- _row((Style.HIGHLIGHT, '═══ BUILT-IN CHECKS ' + '═' * 55)) _out() - if not settings: + if config_check_failed: + _row((Style.WARNING, + 'Failed to retrieve built-in configuration checks. Restart the ' + 'application (\'az webapp restart\') and try again. If the issue ' + 'persists, confirm the app is running and reachable, verify you ' + 'have permission to call the SCM (Kudu) endpoint, and check for ' + 'an ongoing App Service incident in this region.')) + elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) else: setting_w = max(20, min(40, max(len(str(s.get('Setting') or '')) for s in settings) + 2)) From ee5dc83b4b477938d78e75f3dcc917e268fee559 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:42:39 -0700 Subject: [PATCH 04/17] troubleshoot config: shorter failure message Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cli/command_modules/appservice/custom.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index fc9004bcc15..8d57295bcac 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6788,10 +6788,8 @@ def _last_error_within(minutes): if config_check_failed: _row((Style.WARNING, 'Failed to retrieve built-in configuration checks. Restart the ' - 'application (\'az webapp restart\') and try again. If the issue ' - 'persists, confirm the app is running and reachable, verify you ' - 'have permission to call the SCM (Kudu) endpoint, and check for ' - 'an ongoing App Service incident in this region.')) + 'application (\'az webapp restart\') and confirm the SCM (Kudu) ' + 'is running and reachable.')) elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) else: From 27e1b3befa65560dd44d677551f4ef4511b15df7 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 09:55:09 -0700 Subject: [PATCH 05/17] troubleshoot config: drop trailing blank line when runtime section not rendered Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure-cli/azure/cli/command_modules/appservice/custom.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 8d57295bcac..0ed564e3efa 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6812,8 +6812,6 @@ def _last_error_within(minutes): style = Style.WARNING if _issue_detected(setting) else Style.SUCCESS _row((style, line)) - _out() - # ---- Section 2: Site runtime error recommendation ---- # Rendered when the built-in checks flagged at least one setting with # IssueDetected == true, OR when the ARM lastErrorTimestamp is within the @@ -6822,6 +6820,7 @@ def _last_error_within(minutes): if not show_runtime: return + _out() _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: From e8f3277975c4e4be22802bc8c0259df5b2cfda88 Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 10:24:54 -0700 Subject: [PATCH 06/17] troubleshoot config: report polish - Add extra blank line above SITE RUNTIME ERROR RECOMMENDATION. - Render 'No runtime error reported.' in primary (white) instead of green. - Reword SCM failure message to lead with retry before restart. - Drop trailing horizontal rule at end of report. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/cli/command_modules/appservice/custom.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 0ed564e3efa..1120265408e 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6787,8 +6787,8 @@ def _last_error_within(minutes): _out() if config_check_failed: _row((Style.WARNING, - 'Failed to retrieve built-in configuration checks. Restart the ' - 'application (\'az webapp restart\') and confirm the SCM (Kudu) ' + 'Failed to retrieve built-in configuration checks. Please try again. ' + 'If the issue persists, restart the application (\'az webapp restart\') and confirm the SCM (Kudu) ' 'is running and reachable.')) elif not settings: _row((Style.WARNING, 'No built-in configuration checks reported.')) @@ -6820,11 +6820,12 @@ def _last_error_within(minutes): if not show_runtime: return + _out() _out() _row((Style.HIGHLIGHT, '═══ SITE RUNTIME ERROR RECOMMENDATION ' + '═' * 37)) _out() if runtime_error is None: - _row((Style.SUCCESS, 'No runtime error reported.')) + _row((Style.PRIMARY, 'No runtime error reported.')) else: state = str(runtime_error.get('state') or '') last_error = str(runtime_error.get('lastError') or '') @@ -6841,7 +6842,6 @@ def _last_error_within(minutes): _row((Style.PRIMARY, ' Last Error Timestamp: '), (Style.WARNING, timestamp)) _out() - _row((Style.PRIMARY, '─' * 75)) def config_slot_auto_swap(cmd, resource_group_name, webapp, slot, auto_swap_slot=None, disable=None): From 176158ef24eab9622fafbc07ee4d9f279278182c Mon Sep 17 00:00:00 2001 From: Amber Amos Date: Fri, 10 Jul 2026 14:03:21 -0700 Subject: [PATCH 07/17] troubleshoot config: retries, DetailsLevel, wrapping, hints - Retry SCM config fetch on transient 5xx / connection errors (3 attempts, 1.5s backoff). - Read KuduLite's new DetailsLevel field (info/warning/error) with legacy IssueDetected fallback; map to SUCCESS/WARNING/ERROR styles. - Restyle the built-in checks table: HIGHLIGHT header, SECONDARY underlines, per-row Details colored by level. - Wrap Details column so long values hang-indent under the Details column instead of overflowing. - Runtime error section: hanging-indent wrapping for State / Last Error / Last Error Details / Last Error Timestamp, plus relative age (e.g. '2h 18m ago') next to the timestamp. - Gate runtime section on last-error-within-15-min OR config-check-failed (never on stale flagged settings alone). - Add 'Hint:' footer with az webapp config appsettings set / az webapp config set / az webapp log tail commands when any check is warning/error. - _help.py: drop KuduLite / ARM implementation details from the long-summary and reword for consistency. - Tests: add transient-5xx retry cases, config-check-failed keeps runtime section case; refresh stale-timestamp fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da04b3ec-d5e4-4ab8-aeea-58dd7010ac84 --- .../cli/command_modules/appservice/_help.py | 9 +- .../cli/command_modules/appservice/custom.py | 240 ++++++++++++++---- .../latest/test_webapp_commands_thru_mock.py | 100 +++++++- 3 files changed, 292 insertions(+), 57 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/appservice/_help.py b/src/azure-cli/azure/cli/command_modules/appservice/_help.py index 6c173d22213..562e7a97eb3 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/_help.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/_help.py @@ -2448,19 +2448,18 @@ long-summary: > Aggregates two data sources into a single report: - (1) Built-in configuration checks from KuduLite — a set of common + (1) Built-in configuration checks — a set of common Linux App Service settings (linuxFxVersion, port binding, startup command, alwaysOn, health check path, ...) evaluated against the running site's configuration snapshot. - (2) The most recent runtime error reported by Azure Resource Manager - (`/siteStatus`) — echoed verbatim from the platform. + (2) The most recent site runtime status error reported by App Service By default the command returns a structured payload so the standard - `-o json/yaml/tsv/table` formatters handle output. Pass `--report` to + `-o json/yaml/table` formatters handle output. Pass `--report` to print a human-readable two-section report to stdout instead. examples: - - name: Run the built-in configuration checks and show the runtime error, if any + - name: Run the built-in configuration checks and show the runtime error, if any (JSON by default) text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup - name: Print the human-readable report text: az webapp troubleshoot config --name MyWebApp --resource-group MyResourceGroup --report diff --git a/src/azure-cli/azure/cli/command_modules/appservice/custom.py b/src/azure-cli/azure/cli/command_modules/appservice/custom.py index 1120265408e..14a65e6ea61 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/custom.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/custom.py @@ -6596,6 +6596,16 @@ def troubleshoot_config(cmd, resource_group_name, name, slot=None, report=False) last_body_text = '' tried_instances = [] + # SCM (Kudu) is occasionally slow to respond — especially when the app has + # alwaysOn=false (so Kudu itself cold-starts) or the container is thrashing + # during startup. Transient 5xx / timeouts / connection errors resolve after + # a short wait, so retry a few times with backoff before treating the + # failure as terminal. 404 is NOT transient — that's handled separately by + # the per-instance ARR walk below. + _TRANSIENT_STATUSES = {408, 425, 429, 500, 502, 503, 504} + _MAX_ATTEMPTS = 3 + _BACKOFF_SECONDS = 1.5 + def _try_config(cookies=None): try: return requests.get(config_url, headers=headers, cookies=cookies, @@ -6604,6 +6614,18 @@ def _try_config(cookies=None): logger.warning("Failed to call '%s': %s", config_url, ex) return None + def _try_config_with_retry(cookies=None): + import time + resp = None + for attempt in range(1, _MAX_ATTEMPTS + 1): + resp = _try_config(cookies=cookies) + transient = (resp is None) or (resp.status_code in _TRANSIENT_STATUSES) + if not transient: + return resp + if attempt < _MAX_ATTEMPTS: + time.sleep(_BACKOFF_SECONDS * attempt) + return resp + def _consume(resp): nonlocal config_check, last_status, last_body_text if resp is None: @@ -6625,8 +6647,8 @@ def _consume(resp): config_url, snippet) return False - # First attempt: whichever worker ARR picks. - if not _consume(_try_config()): + # First attempt: whichever worker ARR picks (with transient-failure retry). + if not _consume(_try_config_with_retry()): # On 404, walk instances and retry with ARR affinity pinned to each. if last_status == 404: try: @@ -6653,7 +6675,7 @@ def _consume(resp): for instance_id in instance_ids: tried_instances.append(instance_id) cookies = {'ARRAffinity': instance_id, 'ARRAffinitySameSite': instance_id} - if _consume(_try_config(cookies=cookies)): + if _consume(_try_config_with_retry(cookies=cookies)): break if config_check is None and not report: @@ -6726,6 +6748,66 @@ def _consume(resp): return payload +def _troubleshoot_format_dt(value): + """Human-readable timestamp: 'YYYY-MM-DD HH:MM:SS UTC' (or best-effort).""" + if not value: + return None + if isinstance(value, str): + v = value.replace('T', ' ') + is_utc = v.endswith('Z') + if '.' in v: + v = v.split('.', 1)[0] + if is_utc: + if v.endswith('Z'): + v = v[:-1] + v = v + ' UTC' + elif '+' in v: + v = v.split('+', 1)[0] + return v + return str(value) + + +def _troubleshoot_relative_age(iso_value): + """Return a short 'Nh Mm ago' / 'Nm ago' / 'just now' / 'in the future' string + for an ISO-8601 UTC timestamp, or None if the input is unparseable/missing.""" + if not iso_value or not isinstance(iso_value, str): + return None + from datetime import datetime, timezone + v = iso_value + if '.' in v: + head, _, tail = v.partition('.') + tz = '' + for suffix in ('Z', '+', '-'): + if suffix in tail: + idx = tail.find(suffix) + tz = tail[idx:] + break + v = head + tz + v = v.replace('Z', '+00:00') + try: + dt = datetime.fromisoformat(v) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - dt + total_seconds = int(delta.total_seconds()) + if total_seconds < 0: + return 'in the future' + if total_seconds < 60: + return 'just now' + minutes = total_seconds // 60 + if minutes < 60: + return '{}m ago'.format(minutes) + hours = minutes // 60 + rem_min = minutes % 60 + if hours < 24: + return '{}h {}m ago'.format(hours, rem_min) if rem_min else '{}h ago'.format(hours) + days = hours // 24 + rem_hours = hours % 24 + return '{}d {}h ago'.format(days, rem_hours) if rem_hours else '{}d ago'.format(days) + + def _render_troubleshoot_config(payload): """Print the human-readable report: BUILT-IN CHECKS + SITE RUNTIME ERROR RECOMMENDATION. Invoked when ``--report`` is passed.""" @@ -6737,6 +6819,20 @@ def _out(*objs): def _row(*objs): _out(list(objs)) + def _labeled(label, value): + """Emit '