WebMasterMsk Wiki

json from ajax

json from ajax

import requests
from bs4 import BeautifulSoup
import csv

def get_html(url):
    r = requests.get(url)
    return r

def write_csv(data):
    with open('videos.csv', 'a', encoding='utf-8') as f:
        order = ['name', 'url']
        writer = csv.DictWriter(f, fieldnames=order)
        writer.writerow(data)

def get_page_data(response):
    if 'html' in response.headers['Content-Type']:
        html = response.text
    else:
        html = response.json()['content_html']

    soup = BeautifulSoup(html, 'lxml')
    items = soup.find_all('h3', class_='yt-lockup-title')

    for item in items:
        name = item.text.strip()
        url = item.find('a').get('href')
        data = {'name': name, 'url': url}
        write_csv(data)

def get_next(response):
    if 'html' in response.headers['Content-Type']:
        html = response.text
    else:
        html = response.json()['load_more_widget_html']
    soup = BeautifulSoup(html, 'lxml')
    try:
        url = 'https://youtube.com' + soup.find('button', class_='load-more-button').get('data-uix-load-more-href')
    except:
        url = ''
    return url

def main():
    # url = 'https://www.youtube.com/browse_ajax?action_continuation=1&direct_render=1&continuation=4qmFsgJAEhhVQ09tNF9BbkxQTEVCVnJiby0tTjdrUEEaJEVnWjJhV1JsYjNNZ0FEZ0JZQUZxQUhvQk1yZ0JBQSUzRCUzRA%253D%253D'
    url = 'https://www.youtube.com/user/coolpropaganda/videos'
    # get_page_data(get_html(url))

    while True:
        response = get_html(url)
        get_page_data(response)
        url = get_next(response)
        if url:
            continue
        else:
            break

if __name__ == '__main__':
    main()