Код ниже работает:

import pandas as pd
from pandas.io.json import json_normalize
import warnings


def flatten_json(y):
    out = {}

    def flatten(x, name=''):
        if type(x) is dict:
            for a in x:
                flatten(x[a], name + a + '.')
        elif type(x) is list:
            i = 0
            for a in x:
                flatten(a, name + str(i) + '.')
                i += 1
        else:
            out[name[:-1]] = x

    flatten(y)
    return out


def check_common_columns(df1, df2):
    # convert nested JSON data to flat format
    with warnings.catch_warnings():
        warnings.simplefilter(action='ignore', category=FutureWarning)
        df1 = json_normalize(flatten_json(df1))
        df2 = json_normalize(flatten_json(df2))

    # get common columns
    common_cols = set(df1.columns) & set(df2.columns)

    # suppress FutureWarning for passing set as an indexer
    with warnings.catch_warnings():
        warnings.simplefilter(action='ignore', category=FutureWarning)
        df1_common = df1[list(common_cols)]
        df2_common = df2[list(common_cols)]

    # get rows with common columns that have different values
    diff_idx = (df1_common != df2_common).any(axis=1)
    df_diff = pd.concat([df1_common, df2_common])[diff_idx].drop_duplicates()

    # extract column names with different values
    col_names = list(df_diff.columns[df_diff.apply(pd.Series.nunique) > 1])

    return col_names, df_diff[col_names]


# Example JSONs
json1 = {
    "header": {
        "chef": "Gordon",
        "restaurant": "Hell's Kitchen"
    },
    "menu": [
        {
            "starters": {
                "Starter1": "Coastal herbs, lemon, olive oil",
                "Starter2": "Anjou pear, walnuts",
                "Starter3": "Oscietra caviar, celeriac, apple"
            },
            "main": {
                "main1": "Heirloom carrots, pomelo",
                "main2": "Beetroot, cranberry, red amaranth"
            }
        }
    ]
}

json2 = {
    "header": {
        "chef": "Bobby",
        "restaurant": "Bobby's Kitchen"
    },
    "menu": [
        {
            "starters": {
                "Starter1": "Coastal herbs, lemon, olive oil",
                "Starter2": "Anjou pear, walnuts",
                "Starter3": "Oscietra caviar, celeriac, apple"
            },
            "main": {
                "main1": "Heirloom carrots, pomelo",
                "main2": "Beetroot, cranberry, spinach"
            }
        }
    ]
}

col_names, diff_df = check_common_columns(json1, json2)

print("Columns with different values: ")
print(col_names)
print("\nDataframe with different values: ")
print(diff_df)

Выход

UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  df_diff = pd.concat([df1_common, df2_common])[diff_idx].drop_duplicates()
Columns with different values: 
['header.restaurant', 'menu.0.main.main2', 'header.chef']

Dataframe with different values: 
  header.restaurant                  menu.0.main.main2 header.chef
0    Hell's Kitchen  Beetroot, cranberry, red amaranth      Gordon
0   Bobby's Kitchen       Beetroot, cranberry, spinach       Bobby