labs_python/utils.py at main · Drowchik/labs_python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
import os
from time import sleep

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By


def make_dir(name: str) -> None:
    dirs = [f'{name}/{i}' for i in range(1,6)]
    if not os.path.isdir(name):
        os.mkdir(name)
    for dir_name in dirs:
        if not os.path.isdir(dir_name):
            os.mkdir(dir_name)


def get_link_review(driver: webdriver.edge.webdriver.WebDriver, links: str) -> str:
    driver.get("https://www.livelib.ru" + links.get("href"))
    sleep(2)
    soup_review = BeautifulSoup(driver.page_source, "lxml")
    return soup_review.find("div", {"id": "lenta-card__text-review-full"}).text.strip()


def write_review(number: int, title: str, links: str, rating: int, driver: webdriver.edge.webdriver.WebDriver, name: str) -> None:
    name_file = f'{name}/{rating}/{str(number).zfill(4)}.txt'
    with open(name_file, 'w', encoding="utf-8") as f:
        f.write(title.text.strip() + '\n' + get_link_review(driver, links))


def download_reviews(count: int, name: str) -> None:
    number_list = 1
    rating_review = [0] * 5
    driver = webdriver.Edge()
    driver.maximize_window()
    while rating_review[4] < count or rating_review[3] < count or rating_review[2] < count or rating_review[1] < count or rating_review[0] < count:
        URL = f"https://www.livelib.ru/reviews/~{number_list}#reviews"
        driver.get(URL)
        sleep(2)
        number_list += 1
        soup = BeautifulSoup(driver.page_source, "lxml")

        rating = soup.find_all("span", {"class": "lenta-card__mymark"})
        title = soup.find_all("a", "lenta-card__book-title")
        links = soup.find_all("a", {"class": "footer-card__link"})

        for i in range(len(rating) - 1):
            if float(rating[i].text) == 5.0 and rating_review[4] < count:
                rating_review[4] += 1
                write_review(rating_review[4], title[i], links[i], 5, driver, name)
            elif float(rating[i].text) == 4.0 and rating_review[3] < count:
                rating_review[3] += 1
                write_review(rating_review[3], title[i], links[i], 4, driver, name)
            elif float(rating[i].text) == 3.0 and rating_review[2] < count:
                rating_review[2] += 1
                write_review(rating_review[2], title[i], links[i], 3, driver, name)
            elif float(rating[i].text) == 2.0 and rating_review[1] < count:
                rating_review[1] += 1
                write_review(rating_review[1], title[i], links[i], 2, driver, name)
            elif float(rating[i].text) == 1.0 and rating_review[0] < count:
                rating_review[0] += 1
                write_review(rating_review[0], title[i], links[i], 1, driver, name)

    driver.close()
    driver.quit()

def main() -> None:
    make_dir('data')
    download_reviews(10, 'data')