python - selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element : what should i do

Question

I am web crwaling but I keep getting troubles...

selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element

there is a error in

    click = '//*[@id="pagerTagAnchor' + str(j) + '"]'
    driver.find_element_by_xpath(click).click()

what should i do?

import requests
from bs4 import BeautifulSoup
from selenium import webdriver

f = open('movie.txt', 'w', encoding='utf-8')

options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('window-size=1920x1080')
options.add_argument("disable-gpu")

driver = webdriver.Chrome('C:/Users/Administrator/Downloads/chromedriver', chrome_options=options)

for i in range(1, 101):
for j in range(1, 11):
    driver.get('https://movie.naver.com/movie/bi/mi/point.nhn?code=167638')
    driver.implicitly_wait(20)
    click = '//*[@id="pagerTagAnchor' + str(j) + '"]'
    driver.find_element_by_xpath(click).click()
    response = requests.get(str(driver.current_url))
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    notices1 = soup.select(
        'body > div > div > div.score_result > ul > li:nth-of-type(' + str(j) + ') > div.score_reple > p')

    for n in notices1:
        a = n.text.strip()
        a = str(a).replace(",", "")
    f.write(str(a) + ',')

f.close()

this is the full chord.

Answer 1

We have to wait for locate the element. Here is function that find element by xpath

import logging
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException

def findByXpath(locator, timeout = None):
    ''' Get one item by xpath'''
    if not timeout:
        timeout = 10
    try:
        element = WebDriverWait(driver, timeout).until(
            EC.presence_of_element_located((By.XPATH, locator))
        )
        return element
    except TimeoutException:
        logging.info(' Find by xpath not found : %s', locator)
        logging.debug('%s', TimeoutException)
        return None