Пишем парсер на Python - грабим Proxy ч.2

import requests
from bs4 import BeautifulSoup


def get_html(site):
    r = requests.get(site)
    return r.text


def get_page_data(html):
    soup = BeautifulSoup(html, 'lxml')
    head = soup.find('thead').find_all('tr')
    line = soup.find('table', id='theProxyList').find('tbody').find_all('tr')

    for tr in head:
        th = tr.find_all('th')
        ip1 = th[1].text
        port1 = th[2].text
        country1 = th[3].text.replace('\xa0', '')
        anonym1 = th[4].text.replace('\r\n        ', '')
        types1 = th[5].text.replace('\r\n\t\t\t\t\t', '').replace('\r\n        ', '')
        time1 = th[6].text

        data1 = (ip1, port1, country1, anonym1, types1, time1)
        print(", ".join([str(s) for s in data1]))

        for tr in line:
            td = tr.find_all('td')
            ip = td[1].text
            port = td[2].text
            country = td[3].text.replace('\xa0', '')
            anonym = td[4].text.replace('\r\n        ', '')
            types = td[5].text.replace('\r\n\t\t\t\t\t', '').replace('\r\n        ', '')
            time = td[6].text

            data = (ip, port, country, anonym, types, time)
            print(", ".join([str(s) for s in data]))


def main():
    url = 'http://foxtools.ru/Proxy'
    get_page_data(get_html(url))


if __name__ == '__main__':
    main()

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import argparse
import requests
import csv
import time

from bs4 import BeautifulSoup

URL = "http://foxtools.ru/Proxy"


class DownloadHTMLError(Exception):
    pass


def read_args():
    parser = argparse.ArgumentParser(description="""Parse proxy servers from foxtools.ru""",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-p", "--page-numbers", default=1, type=int, help="Max pages")
    parser.add_argument("-o", "--output", default="proxy", help="Output csv path")
    parser.add_argument("-d", "--delay", default=2, type=int, help="Delay between requests in seconds")
    return parser.parse_args()


def make_http_req(url, page_num, delay):
    """
    Функция отвечает за скачивание HTML
    :param url: строка
    :param page_num: целое число обозначающее номер страницы
    :param delay: целое число обозначающее задержку (в секундах) между запросами, если запрашивается несколько страниц,
    чтобы избежать возможного бана.
    :return: html
    """
    params = {"page": page_num}
    try:
        resp = requests.get(url, params=params)
        print("Fetch url {} - {}".format(resp.url, "OK" if resp.ok else resp.status_code))
        resp.raise_for_status()
        if delay:
            time.sleep(delay)
        return resp.text
    except requests.RequestException as e:
        print(e)


def parse_content(table):
    """
    Функия принимает объект BeautifulSoup, который содержит в себе только таблицу
    :param table: BeautifulSoup объект
    :return: список словарей, где каждый словарь представляет собой описание одного прокси сервера
    """
    tr = table.find("tbody").find_all("tr")
    content = []
    for t in tr:
        td = t.find_all("td")
        data = {
            "ip": td[1].find(text=True).strip().encode("utf-8"),
            "port": td[2].find(text=True).strip().encode("utf-8"),
            "country": td[3].find(text=True).strip().encode("utf-8"),
            "anonymous": td[4].find(text=True).strip().encode("utf-8"),
            "type": td[5].find(text=True).strip().encode("utf-8"),
            "response_time": td[6].find(text=True).strip().encode("utf-8"),
            "check_datetime": td[7].find(text=True).strip().encode("utf-8"),
        }

        content.append(data)
    return content


def main():
    cli_args = read_args()

    result = []
    for page_num in range(1, cli_args.page_numbers + 1):
        html = make_http_req(URL, page_num, cli_args.delay)
        if not html:
            raise DownloadHTMLError("Bad page number or some server error")

        soup = BeautifulSoup(html, "lxml")
        table = soup.find("table", {"id": "theProxyList"})

        result.extend(parse_content(table))

    with open(cli_args.output, "w") as f:
        fieldnames = ["ip", "port", "country", "anonymous", "type", "response_time", "check_datetime"]
        writer = csv.DictWriter(f, fieldnames=fieldnames)

        writer.writeheader()
        writer.writerows(result)


if __name__ == "__main__":
    main()

import requests
from bs4 import BeautifulSoup
import socket
from colorama import init
init()

def get_html(site):
    r = requests.get(site)
    return r.text


def get_page_data(html):
    soup = BeautifulSoup(html, 'lxml')
    head = soup.find('thead').find_all('tr')
    line = soup.find('table', id='theProxyList').find('tbody').find_all('tr')

    for tr in head:
        th = tr.find_all('th')
        ip1 = th[1].text
        port1 = th[2].text
        country1 = th[3].text.replace('\xa0', '')
        anonym1 = th[4].text.replace('\r\n        ', '')
        types1 = th[5].text.replace('\r\n\t\t\t\t\t', '').replace('\r\n        ', '')
        time1 = th[6].text

        data1 = (ip1, port1, country1, anonym1, types1, time1)
        print(", ".join([str(s) for s in data1]))
        
        for tr in line:
            td = tr.find_all('td')
            ip = td[1].text
            port = td[2].text
            country = td[3].text.replace('\xa0', '')
            anonym = td[4].text.replace('\r\n        ', '')
            types = td[5].text.replace('\r\n\t\t\t\t\t', '').replace('\r\n        ', '')
            time = td[6].text

            data = (ip, port, country, anonym, types, time)
            print(", ".join([str(s) for s in data]))
            
            ### проверка порта прокси   
            s = socket.socket()
            s.settimeout(4) ### сколько ждать соеденения с портом прокси
            print('try proxy\033[93m',ip+':'+port,'\033[00m')
            try:
                s.connect((ip,int(port)))
            except socket.error:
                s.close()
                print('port closed! proxy \033[31m'+ip+':'+port,'!UNAVAILABLE!\033[00m')
            else:
                s.close()
                with open ('proxy_list_checked0.txt','a') as fa:
                    print('proxy:\033[32m',ip+':'+port,'AVAILABLE\033[00m and written to file')
                    fa.write(ip+':'+port+'\n')


def main():
    url = 'http://foxtools.ru/Proxy'
    get_page_data(get_html(url))


if __name__ == '__main__':
    main()