All geek questions in one place

Time.sleep gives the desired scratch, but will not wait until

Why, when I add time.sleep (2), I get my desired result, but if I add, wait until the specific xpath gives less results?

Output using time.sleep (2) (also desirable):

Adelaide Utd
Tottenham
Dundee Fc
 ...

Quantity: 145 names

Delete time.sleep

Adelaide Utd
Tottenham
Dundee Fc
 ...

Quantity: 119 names

I added:

clickMe = wait(driver,    13).until(EC.element_to_be_clickable((By.CSS_SELECTOR,    ("#page-container > div:nth-child(4) > div >    div.ubet-sports-section-page > div > div:nth-child(2) > div > div >    div:nth-child(1) > div > div > div.page-title-new > h1"))))

Since this element is present on all pages.

Seems to be much smaller. How can I get around this problem?

Script:

import csv
import os

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import WebDriverWait as wait


driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()

driver.get('https://ubet.com/sports/soccer')



clickMe = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, ('//select[./option="Soccer"]/option'))))

options = driver.find_elements_by_xpath('//select[./option="Soccer"]/option')


indexes = [index for index in range(len(options))]
for index in indexes:


    try:
        try:
            zz = wait(driver, 10).until(
                EC.element_to_be_clickable((By.XPATH, '(//select/optgroup/option)[%s]' % str(index + 1))))
            zz.click()
        except StaleElementReferenceException:
            pass

        from selenium.webdriver.support.ui import WebDriverWait
        def find(driver):
            pass

        from selenium.common.exceptions import StaleElementReferenceException, NoSuchElementException
        import time
        clickMe = wait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ("#page-container > div:nth-child(4) > div > div.ubet-sports-section-page > div > div:nth-child(2) > div > div > div:nth-child(1) > div > div > div.page-title-new > h1"))))

        langs0 = driver.find_elements_by_css_selector(
            "div > div > div > div > div > div > div > div > div.row.collapse > div > div > div:nth-child(2) > div > div > div > div > div > div.row.small-collapse.medium-collapse > div:nth-child(1) > div > div > div > div.lbl-offer > span")
        langs0_text = []

        for lang in langs0:
            try:
                langs0_text.append(lang.text)
            except StaleElementReferenceException:
                pass


        directory = 'C:\\A.csv' #####################################
        with open(directory, 'a', newline='', encoding="utf-8") as outfile:
            writer = csv.writer(outfile)
            for row in zip(langs0_text):
                writer.writerow(row)
    except StaleElementReferenceException:
        pass

If you cannot access the page, you will need vpn.

Updating ...

This item may be loading in front of others. Therefore, if we changed it to datascraped (not all pages have data that needs to be cleared).

Add

to try:

    clickMe = wait(driver, 13).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ("div > div > div > div > div > div > div > div > div.row.collapse > div > div > div:nth-child(2) > div > div > div > div > div > div.row.small-collapse.medium-collapse > div:nth-child(3) > div > div > div > div.lbl-offer > span"))))
except TimeoutException as ex:
    pass

The same question is still present.

Manual steps:

#Load driver.get('https://ubet.com/sports/soccer')
#Click drop down (//select/optgroup/option
#Wait for page elements so can scrape
Scrape:

    div > div > div > div > div > div > div > div > div.row.collapse > div > div > div:nth-child(2) > div > div > div > div > div > div.row.small-collapse.medium-collapse > div:nth-child(1) > div > div > div > div.lbl-offer > span
Loop repeat.
+4
2

- angularjs, , angular AJAX ( , ). :

class NgReady:

    js = ('return (window.angular !== undefined) && '
          '(angular.element(document).injector() !== undefined) && '
          '(angular.element(document).injector().get("$http").pendingRequests.length === 0)')

    def __call__(self, driver):
        return driver.execute_script(self.js)

# NgReady does not have any internal state, so one instance 
# can be reused for waiting multiple times
ng_ready = NgReady()

zz.click():

zz.click()
wait(driver, 10).until(ng_ready)

  • , ( ng_ready):

    $ python so-47954604.py && wc -l out.csv && rm out.csv
86 out.csv

  • time.sleep(10) zz.click():

    $ python so-47954604.py && wc -l out.csv && rm out.csv
101 out.csv

  • wait(driver, 10).until(ng_ready) zz.click():

    $ python so-47954604.py && wc -l out.csv && rm out.csv
101 out.csv

NgReady , python , Java, , .

+6

@hoefling , "wait for Angular".

, NgReady, angular . , -, , angular .

Protractor - angular - "" Angular, "Testability" API, Angular.

pytractor package, - WebDriverMixin, angular .

pytractor ( ). , , , webdriver Angular. waitForAngular.js script ( angular 1 2 - , Protractor script):

try { return (function (rootSelector, callback) {
  var el = document.querySelector(rootSelector);
  try {
    if (!window.angular) {
      throw new Error('angular could not be found on the window');
    }

    if (angular.getTestability) {
      angular.getTestability(el).whenStable(callback);
    } else {
      if (!angular.element(el).injector()) {
        throw new Error('root element (' + rootSelector + ') has no injector.' +
           ' this may mean it is not inside ng-app.');
      }
      angular.element(el).injector().get('$browser').
          notifyWhenNoOutstandingRequests(callback);
    }
  } catch (err) {
    callback(err.message);
  }
}).apply(this, arguments); }
catch(e) { throw (e instanceof Error) ? e : new Error(e); }

webdriver.Chrome execute(), , , , angular :

import csv

from selenium import webdriver
from selenium.webdriver.remote.command import Command
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.common.by import By
from selenium.common.exceptions import StaleElementReferenceException
from selenium.webdriver.support import expected_conditions as EC


COMMANDS_NEEDING_WAIT = [
    Command.CLICK_ELEMENT,
    Command.SEND_KEYS_TO_ELEMENT,
    Command.GET_ELEMENT_TAG_NAME,
    Command.GET_ELEMENT_VALUE_OF_CSS_PROPERTY,
    Command.GET_ELEMENT_ATTRIBUTE,
    Command.GET_ELEMENT_TEXT,
    Command.GET_ELEMENT_SIZE,
    Command.GET_ELEMENT_LOCATION,
    Command.IS_ELEMENT_ENABLED,
    Command.IS_ELEMENT_SELECTED,
    Command.IS_ELEMENT_DISPLAYED,
    Command.SUBMIT_ELEMENT,
    Command.CLEAR_ELEMENT
]


class ChromeWithAngular(webdriver.Chrome):
    def __init__(self, root_element, *args, **kwargs):
        self.root_element = root_element

        with open("waitForAngular.js") as f:
            self.script = f.read()

        super(ChromeWithAngular, self).__init__(*args, **kwargs)

    def wait_for_angular(self):
        self.execute_async_script(self.script, self.root_element)

    def execute(self, driver_command, params=None):
        if driver_command in COMMANDS_NEEDING_WAIT:
            self.wait_for_angular()
        return super(ChromeWithAngular, self).execute(driver_command, params=params)


driver = ChromeWithAngular(root_element='body')

# the rest of the code as is with what you had

, pytractor Protractor.

+4

Source: https://habr.com/ru/post/1691279/

More articles:


All Articles