diff --git a/operations/app/terraform/scripts/compare-records.shell b/operations/app/terraform/scripts/compare-records.shell new file mode 100755 index 00000000000..6733a7e6937 --- /dev/null +++ b/operations/app/terraform/scripts/compare-records.shell @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +## Usage: Validate generated records across reports +## > verify-records.shell ./python.csv ./powershell.csv + +oIFS="${IFS}" ; +IFS=$'\n' ; + +source="${1}" ; +target="${2}" ; + +declare -a lines=( $( cat "${source}" ) ) ; +# echo -e "${lines[3]}" ; + +count=1 ; +for line in "${lines[@]}"; do + echo -en "Line[${count}]: " ; + ## Escape double quotes inside the variable + record=$( echo $line | sed 's/^\(.*\)$"/'\2'/g' ) ; + echo -en $record ; + ## Use grep with escaped quotes + found=$( grep -Fxc "${record}" "${target}" ) ; + if [[ $found -gt 0 ]]; then echo " -> match" ; + else echo " -> missing" ; + fi ; + (( count++ )) ; +done ; + +IFS="${oIFS}" ; diff --git a/operations/app/terraform/scripts/export-resources.ps1 b/operations/app/terraform/scripts/export-resources.ps1 index 878ab3d7cba..521e3e4639c 100755 --- a/operations/app/terraform/scripts/export-resources.ps1 +++ b/operations/app/terraform/scripts/export-resources.ps1 @@ -1,17 +1,106 @@ #!/usr/bin/env pwsh -# Define parameters +<# +.SYNOPSIS + Fetches a list of Azure resources and exports them to a CSV file. + +.DESCRIPTION + This script retrieves Azure resources using the Azure CLI and formats the output as a CSV file. + It extracts specified attributes like location, name, and resource group. + +.PARAMETER OutputFile + The name of the output CSV file. Default is 'azure-resources.csv'. + +.PARAMETER OutputHeaders + Specifies which properties to extract from the Azure resources. + Default is `"Location":location,"Name":name,"Resource Group":resourceGroup`. + +.EXAMPLE + ./export-azure-resources.ps1 -OutputFile "my-resources.csv" + + Runs the script and saves Azure resources to "my-resources.csv". + +.NOTES + Requires: + - PowerShell 7+ + - Azure CLI (`az` command) + - Logged-in Azure account (`az login`) +#> + param ( - [string]$OutputFile = "azure-resources--powershell.csv" + [string]$OutputFile = "azure-resources.powershell.csv", + [string]$OutputHeaders = '"Location":location,"Name":name,"Resource Group":resourceGroup', + [switch]$Help ) -# Output header -"Location,Name,Resource Group" | Out-File -FilePath "azure-resources.csv" -Encoding utf8 ; +## Display help message if -Help or -? is used +if ( $Help ) { + Get-Help $PSCommandPath -Full ; + exit ; +} ; + +## Ensure Azure CLI is installed +Write-Host "`nChecking Azure CLI availability ..." ` + -ForegroundColor Cyan ; +if ( -not ( Get-Command az -ErrorAction SilentlyContinue ) ) { + Write-Host "Error: Azure CLI is not installed or not found in PATH." ` + -ForegroundColor Red ; + Write-Host "➡ Please install Azure CLI from https://aka.ms/installazurecli" ; + exit 1 ; +} ; + +## Ensure the user is logged in to Azure +Write-Host "Checking Azure authentication ..." ` + -ForegroundColor Cyan ; +if ( -not ( az account show 2>$null ) ) { + Write-Error "You are not logged in to Azure. Run 'az login' and try again." ; + exit 1 ; +} ; + +## Extract only column names and enforce double quotes for all headers +Write-Host "`nParsing output headers ..." ` + -ForegroundColor Cyan ; +$ColumnNames = ( $OutputHeaders -split ',' ) -replace '"([^"]+)":.*', '$1' ; +$QuotedHeaders = $ColumnNames | ForEach-Object { + if ($_ -match '^".*"$') { $_ } else { "`"$_`"" } +} ; + +## Convert array to CSV format +$HeaderLine = $QuotedHeaders -join ',' ; + +Write-Host "Writing headers to '${OutputFile}'..." ` + -ForegroundColor Cyan ; +## Write CSV header correctly +$HeaderLine | Set-Content -Path $OutputFile -Encoding utf8NoBOM; -# Fetch Azure resources and append to CSV -az resource list --query '[].{"Location":location,"Name":name,"Resource Group":resourceGroup}' --output tsv | - ForEach-Object { $_ -replace "`t", "," } | - Out-File -FilePath $OutputFile -Append -Encoding utf8 ; +Write-Host "`nFetching Azure resources from CLI ..." ` + -ForegroundColor Cyan ; +## Fetch Azure resources and append to CSV +try { + $sortedData = az resource list --query "[].{${OutputHeaders}}" ` + --output tsv ` + | ForEach-Object { + ( $_ -replace "`t", '","' ) -replace '^(.*)$', '"$1"' + } ` + | Sort-Object -CaseSensitive ; + # $sortedData | Out-File -FilePath $OutputFile ` + # -Append ` + # -Encoding utf8; + $sortedData -replace "\r","" -replace '\s+$', "" ` + | Set-Content -Path $OutputFile ` + -Append ` + -Encoding utf8NoBOM; + Write-Host "`nAzure resources successfully exported!" ` + -ForegroundColor Green; + Write-Host "Saved as: $OutputFile" ` + -ForegroundColor Green; +} catch { + Write-Error "`nFailed to fetch Azure resources. Please check your Azure configuration." ; + exit 1 ; +} ; -# Display the contents of the generated CSV -Get-Content -Path $OutputFile ; +## Display output +Write-Host "`nCSV Content (First 10 Lines):`n" ` + -ForegroundColor Yellow ; +Get-Content -Path $OutputFile ` +| Select-Object -First 10 ; diff --git a/operations/app/terraform/scripts/export-resources.py b/operations/app/terraform/scripts/export-resources.py index 84644db2727..c0219579a8f 100755 --- a/operations/app/terraform/scripts/export-resources.py +++ b/operations/app/terraform/scripts/export-resources.py @@ -1,39 +1,166 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +""" +Fetch Azure resources and export them to a CSV file. + +This script retrieves Azure resources using the Azure CLI and saves them to a CSV file. +Users can customize the fields they want to include using the --output-headers parameter. + +Requirements: + - Python 3.x + - Azure CLI (`az` command) + - Logged-in Azure account (`az login`) + +Usage: + python export_azure_resources.py --output-file my-resources.csv + python export_azure_resources.py --output-headers 'Location:location,Name:name,Resource Group:resourceGroup' + python export_azure_resources.py --suppress-output + python export_azure_resources.py --help + +Author: Your Name +""" import subprocess import csv import argparse +import shutil +import sys +import re + +# Default headers (matches PowerShell script) +DEFAULT_OUTPUT_HEADERS = '"Location":location,"Name":name,"Resource Group:resourceGroup"' + +def check_azure_cli(): + """Check if Azure CLI is installed and accessible.""" + if not shutil.which( "az" ): + print( "\nError: Azure CLI is not installed or not found in PATH." ) + print( "➡ Please install Azure CLI from https://aka.ms/installazurecli\n" ) + sys.exit( 1 ) + +def check_azure_login(): + """Check if the user is logged into Azure.""" + try: + subprocess.run( + ["az", "account", "show"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True + ) + except subprocess.CalledProcessError: + print( "\nError: You are not logged in to Azure." ) + print( "➡ Run 'az login' and try again.\n" ) + sys.exit( 1 ) + +def parse_headers( header_string ): + """Convert PowerShell-like headers to CSV format and Azure CLI query format.""" + ## 1. Remove all double quotes from the input + header_string = header_string.replace( '"', '' ) + ## 2. Split headers into (column name, Azure field ID) + header_pairs = [header.strip().split( ":" ) for header in header_string.split( "," )] + if not all( len( pair ) == 2 for pair in header_pairs ): + print( "\nError: Invalid --output-headers format." ) + print( "➡ Expected format: Column1:azureField1,Column2:azureField2" ) + sys.exit( 1 ) + ## 3. Ensure CSV headers are quoted only if they contain spaces + csv_headers = [f'"{pair[0]}"' if " " in pair[0] else pair[0] for pair in header_pairs] + # print( csv_headers ) + ## 4. Construct the valid Azure CLI JMESPath query format + azure_query = ", ".join( [f'"{pair[0]}":{pair[1]}' for pair in header_pairs] ) + # print( azure_query ) + return csv_headers, azure_query + +def fetch_azure_resources( query ): + """Run Azure CLI command to fetch resources based on dynamic query.""" + cmd = [ + "az", "resource", "list", + "--query", f"[].{{{query}}}", + "--output", "tsv" + ] + try: + result = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True + ) + ## Sort the full record instead of isolating fields + return sorted( result.stdout.splitlines(), key=str ) + + except subprocess.CalledProcessError as e: + print( "\nError: Failed to fetch Azure resources." ) + print( f"➡ Azure CLI error: {e.stderr.strip()}\n" ) + sys.exit( 1 ) + +def write_csv( output_file, csv_headers, data ): + """Write Azure resource data to a CSV file, ensuring all fields are quoted.""" + ## Ensure all headers are explicitly quoted + cleaned_headers = [header.strip('"') for header in csv_headers] + quoted_headers = [f'{header}' for header in cleaned_headers] + + with open( output_file, "w", newline="\n", encoding="utf-8" ) as csvfile: + ## Force quotes for all fields + writer = csv.writer( csvfile, quoting=csv.QUOTE_ALL ) + ## Write quoted headers + writer.writerow( quoted_headers ) + for line in data: + ## Trim spaces and ensure consistent line endings + values = [col.strip() for col in line.split( "\t" )] + if len( values ) == len( quoted_headers ): + writer.writerow( values ) + else: + print(f"Skipping malformed line: {line}") + +def main(): + """Main script execution.""" + parser = argparse.ArgumentParser( description="Fetch Azure resources and export them to a CSV file." ) + parser.add_argument( + "--output-file", + type=str, + default="azure-resources.python.csv", + help="Path to the output CSV file (default: azure-resources.csv)" + ) + parser.add_argument( + "--output-headers", + type=str, + default=DEFAULT_OUTPUT_HEADERS, + help="Comma-separated list of headers in 'ColumnName:AzureField' format." + ) + parser.add_argument( + "--suppress-output", + action="store_true", + help="Suppress printing the CSV content to the console" + ) + + args = parser.parse_args() + output_file = args.output_file + output_headers = args.output_headers + suppress_output = args.suppress_output + + print( "\nChecking Azure CLI availability..." ) + check_azure_cli() + + print( "Checking Azure authentication..." ) + check_azure_login() + + print( "\nParsing output headers..." ) + csv_headers, azure_query = parse_headers( output_headers ) + + print( "Fetching Azure resources..." ) + data = fetch_azure_resources( azure_query ) + + print( f"\nWriting data to '{output_file}'..." ) + write_csv( output_file, csv_headers, data ) + + print( "\nAzure resources successfully exported!" ) + print( f"Saved as: {output_file}\n" ) + + if not suppress_output: + print("CSV Content (First 10 Lines):\n" ) + with open( output_file, "r", encoding="utf-8" ) as csvfile: + for i, line in enumerate( csvfile ): + if i >= 10: break + print( line.strip() ) -# Set up argument parser -parser = argparse.ArgumentParser( description="Fetch Azure resources and export them to a CSV file." ) ; -parser.add_argument( - "--output-file", - type=str, - default="azure-resources--python.csv", - help="Path to the output CSV file (default: azure-resources--python.csv)" -) ; - -# Parse arguments -args = parser.parse_args() ; -output_file = args.output_file ; - -# Command to fetch Azure resources -cmd = [ - "az", "resource", "list", - "--query", "[].{\"Location\":location, \"Name\":name, \"Resource Group\":resourceGroup}", - "--output", "tsv" -] ; - -# Run the Azure CLI command -result = subprocess.run( cmd, stdout=subprocess.PIPE, text=True ) ; - -# Write header and data to CSV file -with open( output_file, "w", newline="" ) as csvfile: - writer = csv.writer( csvfile ) ; - writer.writerow( ["Location", "Name", "Resource Group"] ) ; - for line in result.stdout.splitlines(): - writer.writerow( line.split( "\t" ) ) ; - -# Print the contents of the generated CSV -with open( output_file, "r" ) as csvfile: - print( csvfile.read() ) ; +if __name__ == "__main__": + main() diff --git a/operations/app/terraform/scripts/export-resources.shell b/operations/app/terraform/scripts/export-resources.shell index a96e549e35a..ee9cf6a4370 100755 --- a/operations/app/terraform/scripts/export-resources.shell +++ b/operations/app/terraform/scripts/export-resources.shell @@ -1,13 +1,22 @@ #!/usr/bin/env bash +## Usage: Exporting Azure Resources listing +## > export-resources.shell "report.csv" ; + if [[ ${#1} -gt 0 ]]; then OutputFile="${1}" ; - else OutputFile="azure-resources--shell.csv" ; + else OutputFile="azure-resources.shell.csv" ; fi ; -echo "Location,Name,Resource Group" \ -| cat - <( az resource list --query "[].{\"Location\":location, \"Name\":name, \"Resource Group\":resourceGroup}" --output tsv \ -| sed 's/\t/,/g' ) \ -> ${OutputFile} ; +## Default (hardcoded) CSV headers +echo '"Location","Name","Resource Group"' > ${OutputFile} ; + +## Fetching and transforming records to CSV-format +az resource list --query "[].{\"Location\":location, \"Name\":name, \"Resource Group\":resourceGroup}" \ + --output tsv \ +| sed -e 's/\t/","/g' -e 's|^|"|g' -e 's|$|"|g' \ +| sort -u \ +>> ${OutputFile} ; -cat ${OutputFile} ; +## Listing exported content (10 records) +head -n10 ${OutputFile} ;