from time import sleep import selenium from selenium import webdriver from selenium.webdriver.common.by import By url = 'https://gwent.one/en/cards/?type=30&v=11.10.5' driver = webdriver.Chrome() driver.get(url) sleep(1) # Get all the cards cards=[] cards_ids=[] new_cards = driver.find_elements(By.XPATH, "//div[contains(@class,'card-data')]") i=1 while len(new_cards) > 0: i+=1 cards += new_cards cards_ids += [card.get_attribute("data-id") for card in new_cards] driver.execute_script(f"ajaxSearch({i})") sleep(3) new_cards = driver.find_elements(By.XPATH, "//div[contains(@class,'card-data')]") processed_cards = [] for id in cards_ids: driver.get(f'https://gwent.one/en/card/{id}') sleep(0.1) card_data = driver.find_elements(By.XPATH, "//div[contains(@class,'card-data')]") card_data = card_data[0] name = card_data.find_element(By.XPATH, "//div[contains(@class,'card-name')]").text faction = card_data.get_attribute("data-faction") description = card_data.find_element(By.XPATH, "//div[contains(@class,'card-body-flavor')]").text action = card_data.find_element(By.XPATH, "//div[contains(@class,'card-body-ability')]").text image = card_data.find_element(By.XPATH, "//div[contains(@class,'card_asset-img')]/img").get_attribute("src") provision = card_data.get_attribute("data-provision") power = card_data.get_attribute("data-power") color = card_data.get_attribute("data-color") type = card_data.get_attribute("data-type") cardSet = card_data.get_attribute("data-set") processed_cards.append({ "name": name, "faction": faction, "description": description, "action": action, "image": image, "provision": provision, "power": power, "color": color, "type": type, "cardSet": cardSet }) print(len(processed_cards)) # Save to file to an Excel file import pandas as pd df = pd.DataFrame(processed_cards) df.to_excel('cards.xlsx', index=False) driver.close()