Получить список элементов в отдельные массивы

Вот логика (используйте селен в приведенном ниже коде, но обновите его в соответствии с вашими потребностями).

Логика:

url = "https://scholar.google.com/scholar?hl=en&as_sdt=0%2C21&q=python&btnG = "
driver.get(url)
# get the authors and add to list
listBooks = []
books = driver.find_elements_by_xpath("//div[@class='gs_a']")
for bookNum in range(len(books)):
    auths = []
    authors = driver.find_elements_by_xpath("(//div[@class='gs_a'])[%s]/a|(//div[@class='gs_a'])[%s]/self::*[not(a)]"%(bookNum+1,bookNum+1))
    for author in authors:
        auths.append(author.text)
    listBooks.append(auths)

Выход:

[['F Pedregosa', 'G Varoquaux', 'A Gramfort'], ['PD Adams', 'PV Afonine'], ['TE Oliphant'], ['JW Peirce'], ['S Anders', 'PT Pyl', 'W Huber'], ['MF Sanner'], ['S Bird', 'E Klein'], ['M Lutz - 2001 - books.google.com'], ['G Rossum - 1995 - dl.acm.org'], ['W McKinney - … of the 9th Python in Science Conference, 2010 - pdfs.semanticscholar.org']]

Снимок экрана:

Чтобы сгруппировать по результату, вы можете создать пустой list, перебрать результаты и append извлечь данные в list как dict, а возвращенный результат можно сериализовать в строку JSON с использованием метода json_dumps(), например:

temp_list = []

for result in results:
    # extracting title, link, etc.

    temp_list.append({
         "title": title,
         # other extracted elements
     })

print(json.dumps(temp_list, indent=2))

"""
Returned results is a list of dictionaries:
[
  {
    "title": "A new biology for a new century",
    # other extracted elements..
  }
]

"""

Код и полный пример в онлайн-IDE:

from parsel import Selector
import requests, json, re

# https://docs.python-requests.org/en/master/user/quickstart/#passing-parameters-in-urls
params = {
    "q": "biology",  # search query
    "hl": "en"       # language
    }

# https://docs.python-requests.org/en/master/user/quickstart/#custom-headers
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.87 Safari/537.36",
    }

html = requests.get("https://scholar.google.com/scholar", params=params, headers=headers, timeout=30)
selector = Selector(html.text)

data = []

for result in selector.css(".gs_ri"):

    # xpath("normalize-space()") to get blank text nodes as well to get the full string output
    title = result.css(".gs_rt a").xpath("normalize-space()").get()

    # https://regex101.com/r/7bmx8h/1
    authors = re.search(r"^(.*?)-", result.css(".gs_a").xpath("normalize-space()").get()).group(1).strip()
    snippet = result.css(".gs_rs").xpath("normalize-space()").get()

    # https://regex101.com/r/47erNR/1
    year = re.search(r"\d+", result.css(".gs_a").xpath("normalize-space()").get()).group(0)

    # https://regex101.com/r/13468d/1
    publisher = re.search(r"\d+\s?-\s?(.*)", result.css(".gs_a").xpath("normalize-space()").get()).group(1)
    cited_by = int(re.search(r"\d+", result.css(".gs_or_btn.gs_nph+ a::text").get()).group(0))

    data.append({
        "title": title,
        "snippet": snippet,
        "authors": authors,
        "year": year,
        "publisher": publisher,
        "cited_by": cited_by
        })

print(json.dumps(data, indent=2, ensure_ascii=False))

Выход:

[
  {
    "title": "A new biology for a new century",
    "snippet": "… A society that permits biology to become an engineering discipline, that allows that science … science of biology that helps us to do this, shows the way. An engineering biology might still …",
    "authors": "CR Woese",
    "year": "2004",
    "publisher": "Am Soc Microbiol",
    "cited_by": 743
  }, ... other results
  {
    "title": "Campbell biology",
    "snippet": "… Now, Campbell series Biology texts are institutionalized. This is the standard biology text across colleges in the US To say the authors and editors know what they are doing at this point …",
    "authors": "JB Reece, LA Urry, ML Cain, SA Wasserman…",
    "year": "2014",
    "publisher": "fvsuol4ed.org",
    "cited_by": 1184
  }
]

Примечание: в приведенном выше примере я использую библиотеку parsel, которая очень похожа на beautifulsoup и selenium с точки зрения извлечения данных.

Кроме того, вы можете добиться того же, используя API органических результатов Google Scholar из SerpApi. Это платный API с бесплатным планом.

Разница в том, что вам не нужно создавать синтаксический анализатор с нуля, поддерживать его, придумывать, как его масштабировать без блокировки.

Пример кода для интеграции:

from serpapi import GoogleSearch
import os, json

params = {
  "api_key": os.getenv("API_KEY"),  # SerpApi API key
  "engine": "google_scholar",       # parsing engine
  "q": "biology",                   # search query 
  "hl": "en"                        # language
}

search = GoogleSearch(params)       # where data extraction happens
results = search.get_dict()         # JSON -> Python dictionary

for result in results["organic_results"]:
    print(json.dumps(result, indent=2))

Выход:

{
  "position": 0,
  "title": "A new biology for a new century",
  "result_id": "KNJ0p4CbwgoJ",
  "link": "https://journals.asm.org/doi/abs/10.1128/MMBR.68.2.173-186.2004",
  "snippet": "\u2026 A society that permits biology to become an engineering discipline, that allows that science \u2026 science of biology that helps us to do this, shows the way. An engineering biology might still \u2026",
  "publication_info": {
    "summary": "CR Woese - Microbiology and molecular biology reviews, 2004 - Am Soc Microbiol"
  },
  "resources": [
    {
      "title": "nih.gov",
      "file_format": "HTML",
      "link": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC419918/"
    },
    {
      "title": "View it @ CTU",
      "link": "https://scholar.google.com/scholar?output=instlink&q=info:KNJ0p4CbwgoJ:scholar.google.com/&hl=en&as_sdt=0,11&scillfp=15047057806408271473&oi=lle"
    }
  ],
  "inline_links": {
    "serpapi_cite_link": "https://serpapi.com/search.json?engine=google_scholar_cite&q=KNJ0p4CbwgoJ",
    "html_version": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC419918/",
    "cited_by": {
      "total": 743,
      "link": "https://scholar.google.com/scholar?cites=775353062728716840&as_sdt=80005&sciodt=0,11&hl=en",
      "cites_id": "775353062728716840",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=80005&cites=775353062728716840&engine=google_scholar&hl=en"
    },
    "related_pages_link": "https://scholar.google.com/scholar?q=related:KNJ0p4CbwgoJ:scholar.google.com/&scioq=biology&hl=en&as_sdt=0,11",
    "versions": {
      "total": 20,
      "link": "https://scholar.google.com/scholar?cluster=775353062728716840&hl=en&as_sdt=0,11",
      "cluster_id": "775353062728716840",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=0%2C11&cluster=775353062728716840&engine=google_scholar&hl=en"
    }
  }
}
{
  "position": 9,
  "title": "Campbell biology",
  "result_id": "YnWp49O_RTMJ",
  "type": "Book",
  "link": "http://www.fvsuol4ed.org/reviews/Biology%20Organismal%20Template_Campbell%20Biology_Moran.pdf",
  "snippet": "\u2026 Now, Campbell series Biology texts are institutionalized. This is the standard biology text across colleges in the US To say the authors and editors know what they are doing at this point \u2026",
  "publication_info": {
    "summary": "JB Reece, LA Urry, ML Cain, SA Wasserman\u2026 - 2014 - fvsuol4ed.org"
  },
  "resources": [
    {
      "title": "fvsuol4ed.org",
      "file_format": "PDF",
      "link": "http://www.fvsuol4ed.org/reviews/Biology%20Organismal%20Template_Campbell%20Biology_Moran.pdf"
    }
  ],
  "inline_links": {
    "serpapi_cite_link": "https://serpapi.com/search.json?engine=google_scholar_cite&q=YnWp49O_RTMJ",
    "cited_by": {
      "total": 1184,
      "link": "https://scholar.google.com/scholar?cites=3694569986105898338&as_sdt=80005&sciodt=0,11&hl=en",
      "cites_id": "3694569986105898338",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=80005&cites=3694569986105898338&engine=google_scholar&hl=en"
    },
    "related_pages_link": "https://scholar.google.com/scholar?q=related:YnWp49O_RTMJ:scholar.google.com/&scioq=biology&hl=en&as_sdt=0,11",
    "versions": {
      "total": 33,
      "link": "https://scholar.google.com/scholar?cluster=3694569986105898338&hl=en&as_sdt=0,11",
      "cluster_id": "3694569986105898338",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=0%2C11&cluster=3694569986105898338&engine=google_scholar&hl=en"
    },
    "cached_page_link": "http://scholar.googleusercontent.com/scholar?q=cache:YnWp49O_RTMJ:scholar.google.com/+biology&hl=en&as_sdt=0,11"
  }
}

Если вам нужно проанализировать данные из всех результатов Google Scholar Organic, в моем блоге Собрать исторические органические данные за 2017-2021 гг., цитировать результаты Google Scholar в CSV, SQLite на SerpApi есть специальная запись, в которой показано, как это сделать с помощью API.

Disclaimer, I work for SerpApi.