From 4cc21f6a3da05bbf41100bd73e18ad6409f9bdf5 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 12 Aug 2026 10:30:07 -0700 Subject: [PATCH 1/2] Fix PathNotFound error in msexports ETL ingestion on first run The "Get Existing Parquet Files" GetMetadata activity in both msexports_ETL_ingestion (Microsoft.CostManagement/Exports/app.bicep) and the analytics ingestion ETL pipeline (Microsoft.FinOpsHubs/Analytics/app.bicep) failed with PathNotFound when the destination folder had never been created, e.g. the first ingestion for a scope/dataset/month combination. Recommendation exports hit this most often because their path includes an extra exportName segment, but any export type can hit it on first run. Add 'exists' to the GetMetadata fieldList, which is ADF's documented way to make the activity return exists:false instead of failing when the path is missing. This mirrors the pattern already used by other GetMetadata activities in this codebase (Check Schema in Exports/app.bicep, and the two GetMetadata activities in IngestionQueries/app.bicep). Update the downstream Filter activity's items expression to check .output.exists before reading childItems, so a missing folder now resolves to an empty file list instead of failing the pipeline. Fixes #2088 --- .../modules/Microsoft.CostManagement/Exports/app.bicep | 3 ++- .../modules/Microsoft.FinOpsHubs/Analytics/app.bicep | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep index b372acac7..2b67b0cc8 100644 --- a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep @@ -1109,6 +1109,7 @@ resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { } } fieldList: [ + 'exists' 'childItems' ] storeSettings: { @@ -1135,7 +1136,7 @@ resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { userProperties: [] typeProperties: { items: { - value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))' + value: '@if(and(activity(\'Get Existing Parquet Files\').output.exists, contains(activity(\'Get Existing Parquet Files\').output, \'childItems\')), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))' type: 'Expression' } condition: { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 84a29d95c..b87b230a1 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -1694,6 +1694,7 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines } } fieldList: [ + 'exists' 'childItems' ] storeSettings: { @@ -1720,7 +1721,7 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines userProperties: [] typeProperties: { items: { - value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))' + value: '@if(and(activity(\'Get Existing Parquet Files\').output.exists, contains(activity(\'Get Existing Parquet Files\').output, \'childItems\')), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))' type: 'Expression' } condition: { From 73a0076193165d229861e45ba9b053b19bd694fc Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 13 Aug 2026 18:32:29 -0700 Subject: [PATCH 2/2] Fix dependency condition to fail loudly on genuine GetMetadata errors Change the 'Filter Out Current Exports' activity's dependency on 'Get Existing Parquet Files' from 'Completed' to 'Succeeded' in Exports/app.bicep. ADF's 'Completed' condition means succeeded OR failed. Now that 'exists' is in the fieldList (#2088 fix), a missing folder resolves as a clean 'Succeeded' result, so 'Completed' no longer serves a purpose for that case. Its only remaining effect was to let genuine failures (permissions errors, wrong storage account, throttling) silently flow through as if there were no existing files, which would prevent superseded parquet files from being deleted and cause duplicated data in reports with a green pipeline run. This matches the Analytics/app.bicep version of the same downstream filter activity, and the precedent in IngestionQueries/app.bicep, both of which already use 'Succeeded'. Co-Authored-By: Claude Sonnet 5 --- .../modules/Microsoft.CostManagement/Exports/app.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep index 2b67b0cc8..015eef74a 100644 --- a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep @@ -1129,7 +1129,7 @@ resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { { activity: 'Get Existing Parquet Files' dependencyConditions: [ - 'Completed' + 'Succeeded' ] } ]