verify-major-cities.py

import pandas as pd

# Read the TSV file
df = pd.read_csv('generated_data/db.tsv', sep='\t')

# Function to check if a city is in the dataframe
def city_in_df(city, country, df):
    return ((df['name'] == city) & (df['country_code'] == country.lower())).any()

# Extended list of world capitals and major cities
world_cities = [
    ("Tokyo", "JP"),
    ("Delhi", "IN"),
    ("Shanghai", "CN"),
    ("São Paulo", "BR"),
    ("Mexico City", "MX"),
    ("Cairo", "EG"),
    ("Mumbai", "IN"),
    ("Beijing", "CN"),
    ("Dhaka", "BD"),
    ("Osaka", "JP"),
    ("New York", "US"),
    ("Karachi", "PK"),
    ("Buenos Aires", "AR"),
    ("Chongqing", "CN"),
    ("İstanbul", "TR"),
    ("Kolkata", "IN"),
    ("Manila", "PH"),
    ("Lagos", "NG"),
    ("Rio de Janeiro", "BR"),
    ("Tianjin", "CN"),
    ("London", "GB"),
    ("Paris", "FR"),
    ("Bangkok", "TH"),
    ("Seoul", "KR"),
    ("Moscow", "RU"),
    ("Jakarta", "ID"),
    ("Lima", "PE"),
    ("Ho Chi Minh City", "VN"),
    ("Guangzhou", "CN"),
    ("Shenzhen", "CN"),
    ("Bangalore Urban", "IN"),
    ("Kinshasa", "CD"),
    ("Lahore", "PK"),
    ("Bogotá D.C.", "CO"),
    ("Chennai", "IN"),
    ("Johannesburg", "ZA"),
    ("Taipei", "TW"),
    ("Hyderabad", "IN"),
    ("Dongguan", "CN"),
    ("Hangzhou", "CN"),
    ("Hong Kong", "HK"),
    ("Chengdu", "CN"),
    ("Nanjing", "CN"),
    ("Tehran", "IR"),
    ("Wuhan", "CN"),
    ("Onitsha", "NG"),
    ("Ahmedabad", "IN"),
    ("Kuala Lumpur", "MY"),
    ("Xi'an", "CN"),
    ("Harbin", "CN"),
    ("Surat", "IN"),
    ("Singapore", "SG"),
    ("Berlin", "DE"),
    ("Madrid", "ES"),
    ("Rome", "IT"),
    ("Kyiv", "UA"),
    ("Bucharest", "RO"),
    ("Vienna", "AT"),
    ("Hamburg", "DE"),
    ("Warsaw", "PL"),
    ("Budapest", "HU"),
    ("Amsterdam", "NL"),
    ("Brussels", "BE"),
    ("Stockholm", "SE"),
    ("Prague", "CZ"),
    ("Copenhagen", "DK"),
    ("Helsinki", "FI"),
    ("Athens", "GR"),
    ("Lisbon", "PT"),
    ("Dublin", "IE"),
    ("Oslo", "NO"),
    ("Washington D.C.", "US"),
    ("Los Angeles", "US"),
    ("Chicago", "US"),
    ("Houston", "US"),
    ("Toronto", "CA"),
    ("Montreal", "CA"),
    ("Vancouver", "CA"),
    ("Sydney", "AU"),
    ("Melbourne", "AU"),
    ("Brisbane", "AU"),
    ("Perth", "AU"),
    ("Auckland", "NZ"),
    ("Wellington", "NZ"),
    ("Cape Town", "ZA"),
    ("Nairobi", "KE"),
    ("Addis Ababa", "ET"),
    ("Khartoum", "SD"),
    ("Algiers", "DZ"),
    ("Casablanca", "MA"),
    ("Tunis", "TN"),
    ("Accra", "GH"),
    ("Dakar", "SN"),
    ("Riyadh", "SA"),
    ("Jeddah", "SA"),
    ("Dubai", "AE"),
    ("Abu Dhabi", "AE"),
    ("Doha", "QA"),
    ("Kuwait City", "KW"),
    ("Muscat", "OM"),
    ("Manama", "BH"),
    ("Baghdad", "IQ"),
    ("Amman", "JO"),
    ("Beirut", "LB"),
    ("Damascus", "SY"),
    ("Jerusalem", "IL"),
    ("Tel Aviv", "IL"),
    ("Ankara", "TR"),
    ("Tashkent", "UZ"),
    ("Almaty", "KZ"),
    ("Bishkek", "KG"),
    ("Dushanbe", "TJ"),
    ("Ashgabat", "TM"),
    ("Kabul", "AF"),
    ("Islamabad", "PK"),
    ("Kathmandu", "NP"),
    ("Colombo", "LK"),
    ("Dhaka", "BD"),
    ("Yangon", "MM"),
    ("Phnom Penh", "KH"),
    ("Vientiane", "LA"),
    ("Hanoi", "VN"),
    ("Ulaanbaatar", "MN"),
    ("Pyongyang", "KP"),
    ("Taipei", "TW"),
    ("Manila", "PH"),
    ("Kuala Lumpur", "MY"),
    ("Jakarta", "ID"),
    ("Surabaya", "ID"),
    ("Bandung", "ID"),
    ("Port Moresby", "PG"),
    ("Suva", "FJ"),
    ("Nouméa", "NC"),
    ("Honolulu", "US"),
    ("San Francisco", "US"),
    ("Seattle", "US"),
    ("Boston", "US"),
    ("Miami", "US"),
    ("Atlanta", "US"),
    ("Dallas", "US"),
    ("Phoenix", "US"),
    ("Philadelphia", "US"),
    ("San Diego", "US"),
    ("Detroit", "US"),
    ("San Jose", "US"),
    ("Havana", "CU"),
    ("Santo Domingo", "DO"),
    ("Port-au-Prince", "HT"),
    ("San Juan", "PR"),
    ("Kingston", "JM"),
    ("Guatemala City", "GT"),
    ("San Salvador", "SV"),
    ("Tegucigalpa", "HN"),
    ("Managua", "NI"),
    ("San Jose", "CR"),
    ("Panama", "PA"),
    ("Caracas", "VE"),
    ("Quito", "EC"),
    ("Guayaquil", "EC"),
    ("Cali", "CO"),
    ("Medellín", "CO"),
    ("Brasília", "BR"),
    ("Salvador", "BR"),
    ("Fortaleza", "BR"),
    ("Belo Horizonte", "BR"),
    ("Manaus", "BR"),
    ("Curitiba", "BR"),
    ("Recife", "BR"),
    ("Porto Alegre", "BR"),
    ("Montevideo", "UY"),
    ("Asunción", "PY"),
    ("La Paz", "BO"),
    ("Santiago", "CL"),
    ("Antofagasta", "CL"),
    ("Valparaíso", "CL"),
]

# Check each city
missing_cities = []
for city, country in world_cities:
    if not city_in_df(city, country, df):
        missing_cities.append(f"{city}, {country}")

# Print results
if missing_cities:
    print("The following major cities are missing from the dataset:")
    for city in missing_cities:
        print(city)
else:
    print("All major cities in the list are present in the dataset.")

# Print some statistics
print(f"\nTotal cities in the dataset: {len(df)}")
print(f"Number of countries represented: {df['country_code'].nunique()}")
print(f"Number of cities checked: {len(world_cities)}")
print(f"Number of missing cities: {len(missing_cities)}")