Веб-скрапинг Python с данными, полученными из javascripts

Вот один из способов получить эту информацию с помощью Selenium. Это не быстро, но надежно и возвращает всех игроков (725). Настройка Selenium — это chromedriver/linux, вы можете адаптировать ее к своей собственной настройке, просто наблюдайте за импортом и кодом после определения драйвера.

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t
import pandas as pd

pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)

chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")

webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 25)
url = 'https://nextgenftl.com/leagues/ftl-main-2022/game-weeks/week-30/players'
big_list = []
driver.get(url)

for x in range(10):
    players = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//ion-list[not(@id="menu-list")]//ion-item')))
    for p in players:
        p.location_once_scrolled_into_view
    wait.until(EC.presence_of_element_located((By.TAG_NAME, 'ion-infinite-scroll'))).location_once_scrolled_into_view
    
    t.sleep(1)
players = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//ion-list[not(@id="menu-list")]//ion-item')))
for p in players:
    try:
        p.location_once_scrolled_into_view
        badge = p.find_element(By.XPATH, './/ion-badge').text
        name = p.find_element(By.XPATH, './/ion-label').text
        current_price = p.find_element(By.XPATH, './/div[@title="Current Price"]').text
        price_change = p.find_element(By.XPATH, './/div[@title="Price Change"]').text
        average_points = p.find_element(By.XPATH, './/div[@title="3-Week Average Points"]').text
        events_played = p.find_element(By.XPATH, './/div[@title="Events Played"]').text
        
        big_list.append((badge, name, current_price, price_change, average_points, events_played))
    except Exception as e:
        print('error')
        continue
t.sleep(2)
print(len(big_list))
df = pd.DataFrame(big_list, columns = ['badge', 'name', 'current_price', 'price_change', 'average_points', 'events_played'])
print(df)
df.to_csv('fantasy_tennis.csv')

Это отобразит фрейм/таблицу данных в терминале, а также сохранит его как csv:

725
badge   name    current_price   price_change    average_points  events_played
0   ATP Novak Djokovic  $19.864m    --  116.97  7
1   ATP Rafael Nadal    $19.295m    ↓ 1.137 53.92   9
2   WTA Iga Swiatek $17.835m    ↓ 0.074 72.70   13
3   WTA Ashleigh Barty  $16.800m    --  169.50  1
4   ATP Carlos Alcaraz  $15.587m    ↑ 0.494 74.14   14
... ... ... ... ... ... ...
720 WTA Dayana Yastremska   $1.450m ↓ 0.068 3.75    14
721 WTA Xiaodi You  $1.450m --  3.77    1
722 WTA Eleana Yu   $1.450m --  2.90    1
723 WTA Anastasia Zakharova $1.450m --  1.77    1
724 ATP Kacper Zuk  $1.450m --  4.16    1

См. документацию по Selenium на https://www.selenium.dev/documentation/