fandom-pyのメモ

!pip install fandom-py

import fandom

fandom.set_wiki("friends")

season_1 = fandom.page("Season 1")

dir(season_1)

['_FandomPage__continued_query',
 '_FandomPage__load',
 '_FandomPage__title_query_param',
 '__class__',
 '__delattr__',
 '__dict__',
 '__dir__',
 '__doc__',
 '__eq__',
 '__format__',
 '__ge__',
 '__getattribute__',
 '__gt__',
 '__hash__',
 '__init__',
 '__init_subclass__',
 '__le__',
 '__lt__',
 '__module__',
 '__ne__',
 '__new__',
 '__reduce__',
 '__reduce_ex__',
 '__repr__',
 '__setattr__',
 '__sizeof__',
 '__str__',
 '__subclasshook__',
 '__weakref__',
 'content',
 'html',
 'images',
 'language',
 'pageid',
 'plain_text',
 'revision_id',
 'section',
 'sections',
 'summary',
 'title',
 'url',
 'wiki']

from pprint import pprint

pprint(season_1.content)

{'content': 'Season 1 of Friends aired from September 22, 1994 to May 18, 1995 '
            'on NBC in the US.',
 'infobox': 'Season\n'
            'Season 1\n'
            'Season\n'
            '1\n'
            'Episodes\n'
            '23\n'
            'Aired from\n'
            'September 22, 1994 - May 18, 1995\n'
            'Premiere\n'
            'The Pilot\n'
            'Finale\n'
            'The One Where Rachel Finds Out\n'
            'Other seasons\n'
            '← Previous\n'
            'Next →\n'
            '–\n'
            '"Season 2"',
 'sections': [{'content': 'This season introduced the six main characters: '
                          'Chandler Bing, Phoebe Buffay, Monica Geller, Ross '
                          'Geller, Rachel Greene, and Joey Tribbiani.\n'
                          'Rachel, who left her fiancé at the altar on her '
                          'wedding day, has come to New York and ends up '
                          'living with Monica as her roommate when Phoebe had '
                          'just moved out. It establishes early on in the '
                          'season that Ross has been infatuated with Rachel '
                          'since the two characters attended high school. '
                          'Several episodes revolve around his attempts to '
                          'tell her how he feels. Ross leaves for a fossil dig '
                          'in China at the end of the season, missing out on '
                          "Rachel's birthday party but staying long enough to "
                          'give her a meaningful present: a cameo that was '
                          "just like the one that belonged to Rachel's "
                          'grandmother. Chandler accidentally reveals that '
                          'Ross is in love with Rachel, much to her shock. '
                          'Rachel realizes that she has feelings for Ross and '
                          'rushes off to the airport to tell him that she '
                          'wants to pursue a relationship with him. As she '
                          'waits at the gate for Ross to arrive, the season '
                          'ends on a cliffhanger as he gets off the plane but '
                          'with a new girlfriend: Julie.\n'
                          "Ross' estranged lesbian wife, Carol Willick, is "
                          "pregnant with his baby. This puts him and Carol's "
                          'lesbian life partner, Susan Bunch, in an awkward '
                          'position. When the baby is born at the end of the '
                          'season, Ross, Carol, and Susan agree to name him '
                          "Ben: after a name tag on a janitor's uniform worn "
                          'by Phoebe.\n'
                          'The episodic nature of the season sees the other '
                          'characters having multiple dates, many of which go '
                          'wrong (Monica dates a minor in one episode). The '
                          'recurring character of Janice is introduced as a '
                          'girlfriend Chandler breaks up with in an early '
                          'episode but frequently returns to through the '
                          'ensuing ten seasons.',
               'title': 'Season summary'},
              {'content': 'Jennifer Aniston as Rachel Greene\n'
                          'Courteney Cox as Monica Geller\n'
                          'Lisa Kudrow as Phoebe Buffay/Ursula Buffay ("The '
                          'One With Two Parts, Part 1"/"The One With Two '
                          'Parts, Part 2")\n'
                          'Matt LeBlanc as Joey Tribbiani\n'
                          'Matthew Perry as Chandler Bing\n'
                          'David Schwimmer as Ross Geller/Russ',
               'sections': [{'content': 'Anita Barone as Carol Willick ("The '
                                        'One With The Sonogram At The End")\n'
                                        'Jane Sibbett as Carol Willick\n'
                                        'Maggie Wheeler as Janice\n'
                                        'Mitchell Whitfield as Barry Farber\n'
                                        'Cosimo Fusco as Paolo\n'
                                        'Jessica Hecht as Susan Bunch\n'
                                        ' \n'
                                        ' ',
                             'title': 'Recurring Cast'}],
               'title': 'Starring'},

import pandas as pd
import fandom
import re
from tqdm.notebook import tqdm

df = pd.DataFrame(columns=["season", "#", "total#", "title", "writer", "director", "airdate", "short plot"])

def object_to_markdown(data, depth=1):
    markdown = []

    # title
    markdown.append(f"{'#' * depth} {data['title']}\n")
    
    # content
    markdown.append(f"{data['content']}\n")
    
    # sections
    for section in data.get('sections', []):
        if section['title'] not in ['External links', 'Episode Navigation', 'Photos']:
            markdown.append(object_to_markdown(section, depth + 1))
    
    return "\n".join(markdown)

fandom.set_wiki("friends")

regexp_eps_no = r'^([0-9]{1,2}|-)$'

for season in range(1,11):
    episode_list = fandom.page(f"Season_{season}")
    eps = episode_list.content["sections"][-1]["content"].split("\n")[7:]
    i = 0
    while i < len(eps):
        if re.fullmatch(regexp_eps_no, eps[i]) and re.fullmatch(regexp_eps_no, eps[i+7]):
            chunk = eps[i:i+7]
            df.loc[len(df)] = [season] + chunk
            i += 7
        elif re.fullmatch(regexp_eps_no, eps[i]) and not re.fullmatch(regexp_eps_no, eps[i+7]):
            chunk = eps[i:i+8]
            chunk[-2:] = ["".join(chunk[-2:])]
            df.loc[len(df)] = [season] + chunk
            i += 8
        else:
            i += 7

df = df[df["#"] != "-"]

for index, row in tqdm(df.iterrows(), total=df.shape[0]):
    epnum = str(row["total#"]).zfill(3)
    title = row["title"].replace('"', '')
    f_title = title.replace(" ","_")
    path = f"data/{epnum}_{f_title}.txt"

    p = fandom.page(title)
       
    with open(path, mode="w") as f:
        f.write(object_to_markdown(p.content))