python:parsers
https://dokuwiki.webmastermsk.ru/
2026-08-12T13:12:41+0000
https://dokuwiki.webmastermsk.ru/
https://dokuwiki.webmastermsk.ru/lib/tpl/dokuwiki/images/favicon.ico
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:json_from_ajax
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:json_from_ajax&rev=1673515089&do=diff
json from ajax
import requests
from bs4 import BeautifulSoup
import csv
def get_html(url):
r = requests.get(url)
return r
def write_csv(data):
with open('videos.csv', 'a', encoding='utf-8') as f:
order = ['name', 'url']
writer = csv.DictWriter(f, fieldnames=order)
writer.writerow(data)
def get_page_data(response):
if 'html' in response.headers['Content-Type']:
html = response.text
else:
html = response.json()['content_html']
s…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:multiple_data
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:multiple_data&rev=1673515089&do=diff
multiple data
import requests
from bs4 import BeautifulSoup
import csv
def get_html(url):
r = requests.get(url)
return r.text
def refined(s):
# 1,470 total ratings
r = s.split(' ')[0] # 1,470
return r.replace(',', '') # 1470
def write_csv(data):
with open('plugins.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow( [data['name'],
data['url'],
data['reviews']] )
def get_data(html):
sou…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:multiprocessing
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:multiprocessing&rev=1673515089&do=diff
Multiprocessing
Многопоточность в python с помощью модуля multiprocessing
import requests
import csv
# Многопоточность
from multiprocessing import Pool
from time import sleep
def get_html(url):
# sleep(1)
r = requests.get(url)
return r.text
def write_csv(data):
with open('websites.csv', 'a', encoding='utf-8') as file:
order = ['name', 'url', 'description', 'traffic', 'percent']
writer = csv.DictWriter(file, fieldnames=order)
writer.writerow(data)
…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:pagination
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:pagination&rev=1673515089&do=diff
Парсер с использованием пагинации
example 1
Фломатируя url пробежимся по нескольким страницам
import requests
from bs4 import BeautifulSoup
import csv
def get_html(url):
r = requests.get(url)
if r.ok: # 200 ## 403 404
return r.text
print(r.status_code)
def refine_cy(s):
# ТИЦ: 500000 -> ['ТИЦ:', '500000']
return s.split(' ')[-1]
def write_csv(data):
with open('yaca.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow((data['name'],
…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:plain_text_from_ajax
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:plain_text_from_ajax&rev=1673515089&do=diff
plain/text from ajax
import requests
import csv
def get_html(url):
r = requests.get(url)
return r.text
def write_csv(data):
with open('websites.csv', 'a', encoding='utf-8') as f:
order = ['name', 'url', 'description', 'traffic', 'percent']
writer = csv.DictWriter(f, fieldnames=order)
writer.writerow(data)
def main():
for i in range(0, 6428):
url = 'https://www.liveinternet.ru/rating/ru//today.tsv?page={}'.format(str(i))
response = …
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:postgres_peewee
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:postgres_peewee&rev=1673515089&do=diff
postgres peewee
import csv
#ORM для бд
from peewee import *
#Установка соединения
db = PostgresqlDatabase(database='test', user='postgres', password='2619192', host='db')
# Описание таблички coin
class Coin(Model):
# Поля таблицы
name = CharField()
url = TextField()
price = CharField()
def main():
# соединение с бд
db.connect()
# Создаем табличку, внутрь передается список таблиц которые надо создать
db.create_tables([Coin])
# Открываем csv файл на чтен…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:proxy
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:proxy&rev=1673515089&do=diff
proxy
import requests
from bs4 import BeautifulSoup
from random import choice
def get_proxy():
html = requests.get('https://free-proxy-list.net/').text
soup = BeautifulSoup(html, 'lxml')
trs = soup.find('table', id='proxylisttable').find_all('tr')[1:13]
proxies = []
for tr in trs:
tds = tr.find_all('td')
ip = tds[0].text.strip()
port = tds[1].text.strip()
schema = 'https' if 'yes' in tds[6].text.strip() else 'http'
proxy = {'sche…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:table_data
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:table_data&rev=1673515089&do=diff
table data
import requests
from bs4 import BeautifulSoup
import csv
def get_html(url):
r = requests.get(url)
return r.text
def write_csv(data):
with open('cmc.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow([data['name'],
data['symbol'],
data['url'],
data['price']])
def get_page_data(html):
soup = BeautifulSoup(html, 'lxml')
trs = soup.find('table', id='currencies').fi…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:ua-footbal
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:ua-footbal&rev=1673515089&do=diff
ua-football.com
Парсер забирает новости со страницы ua-football.com/sport
# https://pypi.org/project/beautifulsoup4/ - установить сторонний пакет с помощью пакетного менеджера pip
# pip freeze - список установленных пакетов
# pip install beautifulsoup4 установить пакет
from bs4 import BeautifulSoup
import urllib.request
# Создаем запрос к сайту
req = urllib.request.urlopen('https://www.ua-football.com/sport')
# Читаем ответ
html = req.read()
# Парсим страницу, features='html.parser' - мо…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:user-agent
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:user-agent&rev=1673515089&do=diff
User-Agent
import requests
from bs4 import BeautifulSoup
import csv
def get_html(url):
header = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0',
'From': 'youremail@domain.com'
}
r = requests.get(url, headers=header)
return r.text
def get_articles(html):
soup = BeautifulSoup(html, 'lxml')
ts = soup.find('div', class_='testimonial-container').find_all('article')
return ts
def get_data(ts):
for item…
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:wordpress.org
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:wordpress.org&rev=1673515089&do=diff
wordpress.org
Будет Распарсен заголовок страницы:
import requests
from bs4 import BeautifulSoup
def get_html(url):
r = requests.get(url)
return r.text
def get_data(html):
soup = BeautifulSoup(html, 'lxml')
h1 = soup.find('div', id='home-welcome').find('header').find('h1').text
return h1
def main():
url = 'https://wordpress.org/'
print(get_data(get_html(url)))
if __name__ == '__main__':
main()
-
text/html
2023-01-12T09:18:09+0000
Anonymous (anonymous@undisclosed.example.com)
python:parsers:write_reade_csv
https://dokuwiki.webmastermsk.ru/doku.php?id=python:parsers:write_reade_csv&rev=1673515089&do=diff
чтение запись csv
import csv
def write_csv(data):
with open('names.csv', 'a') as file:
writer = csv.writer(file)
writer.writerow((data['name'], data['surname'], data['age']))
def write_csv2(data):
with open('names.csv', 'a') as file:
order = ['name', 'surname', 'age']
writer = csv.DictWriter(file, fieldnames=order)
writer.writerow(data)
def main():
d = {'name': 'Petr', 'surname': 'Ivanov', 'age': 21}
d1 = {'name': 'Ivan', 'surnam…