Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ Scrapes data from [ao3.org](https://ao3.org) and beyond. Now with Types™.
| [**`getWorkWithChapters`**](./src/works/index.ts#L53) | Fetches a work and its chapter list. | `{ workId: string }` - The ID of the work. | [`Promise<{ title: string; authors: Author[] \| Anonymous; workId: string; chapters: Chapter[] }>`](./types/entities.ts) |
| [**`getSeries`**](./src/series/index.ts#L17) | Retrieves details for a specific series. | `{ seriesId: string }` - The ID of the series. | [`Promise<Series>`](./types/entities.ts) |
| [**`getUser`**](./src/users/index.ts#L15) | Fetches profile information for a user. | `{ username: string }` - Username of the user to fetch. | [`Promise<User>`](./types/entities.ts) |
| [**`getUserWorks`**](./src/users/index.ts#L63) | Lists one page of a user's public works. | `{ username: string, pseud?: string, page?: number, sortColumn?, sortDirection? }` | [`Promise<UserWorks>`](./types/entities.ts) |
| [**`setFetcher`**](./src/fetcher.ts#L5) | Sets a custom fetch function for requests. | `{ fetcher: typeof fetch }` - Custom fetch function. | `void` |

#### Why Override Fetch?
Expand All @@ -56,6 +57,7 @@ Using `setFetcher`, you can override the default `fetch` method used by the libr
- **[`WorkSummary`](./types/entities.ts#L20)** / **[`LockedWorkSummary`](./types/entities.ts#L45)**: Summarizes a work, including title, authors, tags, and statistics.
- **[`Series`](./types/entities.ts)**: Information on a series, such as title, authors, works, and publication details.
- **[`User`](./types/entities.ts)**: Profile information for an AO3 user, including pseudonyms, works, bookmarks, and more.
- **[`UserWorks`](./types/entities.ts)**: One page of a user's public works, with pagination.
- **[`Chapter`](./types/entities.ts)**: Details about individual chapters within a work.

## Sample usage
Expand Down
1 change: 1 addition & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ export * from "./works";
export * from "./series";
export { getAsShortUrl } from './urls';
export { setFetcher } from "./fetcher";
export { ArchivePageRequestError, InvalidIDError } from "./utils";
42 changes: 32 additions & 10 deletions src/page-loaders.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,20 @@ import {
getTagWorksFeedAtomUrl,
getTagWorksFeedUrl,
getUserProfileUrl,
getUserWorksUrl,
getWorkIndexUrl,
getWorkUrl,
} from "./urls";

import { CheerioAPI } from "cheerio";
import { load } from "cheerio/slim";
import { getFetcher } from "./fetcher";
import { ArchiveId, TagSearchFilters } from "types/entities";
import { ArchivePageRequestError } from "src/utils";
import {
ArchiveId,
TagSearchFilters,
UserWorksFilters,
} from "types/entities";

// This is a wrapper around the fetch function that loads the page into a CheerioAPI
// instance and returns the type of the page.
Expand All @@ -25,15 +31,17 @@ const fetchPage = async <ReturnType>({
url: string;
skipAdultBanner?: boolean;
}) => {
return (await load(
await (
await getFetcher()(url, {
headers: {
Cookie: skipAdultBanner ? "view_adult=true;" : "",
},
})
).text()
)) as ReturnType;
const response = await getFetcher()(url, {
headers: {
Cookie: skipAdultBanner ? "view_adult=true;" : "",
},
});

if (!response.ok) {
throw new ArchivePageRequestError({ url, status: response.status });
}

return (await load(await response.text())) as ReturnType;
};

// We create separate interfaces for each page type to make sure that the
Expand Down Expand Up @@ -118,6 +126,20 @@ export const loadUserProfilePage = async ({
});
};

// A page listing a user's works, optionally only the ones under one pseud.
// Sample: https://archiveofourown.org/users/astolat/works
// Sample: https://archiveofourown.org/users/astolat/pseuds/astolat/works
export interface UserWorksPage extends CheerioAPI {
kind: "UserWorksPage";
}
export const loadUserWorksPage = async (
options: Partial<UserWorksFilters> & { username: string; pseud?: string },
) => {
return await fetchPage<UserWorksPage>({
url: getUserWorksUrl(options),
});
};

export interface ChapterIndexPage extends CheerioAPI {
kind: "ChapterIndexPage";
}
Expand Down
173 changes: 8 additions & 165 deletions src/series/getters.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,7 @@
import type { Author, Series, SeriesWorkSummary } from "types/entities";
import { type CheerioAPI, load } from "cheerio/slim";
import type { SeriesPage, WorkPage } from "src/page-loaders";
import {
getWorkBookmarkCount,
getWorkHits,
getWorkKudosCount,
getWorkLanguage,
getWorkPublishedChapters,
getWorkTotalChapters,
getWorkWordCount,
} from "src/works/work-getters";
import { getAsShortUrl, getWorkDetailsFromUrl, getWorkUrl } from "src/urls";
import { parseArchiveId } from "src/utils";

const monthMap: { [month: string]: string } = {
Jan: "01",
Feb: "02",
Mar: "03",
Apr: "04",
May: "05",
Jun: "06",
Jul: "07",
Aug: "08",
Sep: "09",
Oct: "10",
Nov: "11",
Dec: "12",
};
import type { Author, Series, WorkBlurbSummary } from "types/entities";
import type { SeriesPage } from "src/page-loaders";
import { getAuthorFromUrl } from "src/urls";
import { getWorkBlurb } from "src/works/blurb-getters";

export const getSeriesTitle = ($seriesPage: SeriesPage): string => {
return $seriesPage("h2.heading").text().trim();
Expand All @@ -46,14 +21,7 @@ export const getSeriesAuthors = (

if (authorLinks.length !== 0) {
authorLinks.each((i, element) => {
const url = element.attribs.href;
const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;

authors.push({
username: username,
pseud: decodeURI(pseud),
anonymous: false,
});
authors.push(getAuthorFromUrl({ url: element.attribs.href }));
});
}

Expand Down Expand Up @@ -117,137 +85,12 @@ export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => {

export const getSeriesWorks = (
$seriesPage: SeriesPage
): SeriesWorkSummary[] => {
const works: SeriesWorkSummary[] = [];
): WorkBlurbSummary[] => {
const works: WorkBlurbSummary[] = [];

$seriesPage("ul.index > li.work").each((index, element) => {
works[index] = getSeriesWork($seriesPage(element).html() as string);
works[index] = getWorkBlurb($seriesPage(element).html() as string);
});

return works;
};

// Helpers for series' works
interface SeriesWork extends CheerioAPI {
kind: "SeriesWork";
}

const getSeriesWork = (workHtml: string): SeriesWorkSummary => {
const work = load(workHtml);
const $work = work as SeriesWork,
$$work = work as WorkPage;

const totalChapters = getWorkTotalChapters($$work);
const publishedChapters = getWorkPublishedChapters($$work);

const url = $work("a[href*='/works/']").attr("href") as string;
const id = getWorkDetailsFromUrl({ url }).workId;
const workUrl = getWorkUrl({ workId: id });
const shortUrl = getAsShortUrl({ url: workUrl });

return {
id: parseArchiveId(id),
url: workUrl,
shortUrl,
title: getSeriesWorkTitle($work),
updatedAt: getSeriesWorkUpdateDate($work),

summary: getSeriesWorkSummary($work),
adult: false,
fandoms: getSeriesWorkFandoms($work),
tags: {
characters: getSeriesWorkCharacters($work),
relationships: getSeriesWorkRelationships($work),
additional: getSeriesWorkAdditionalTags($work),
},
authors: getSeriesWorkAuthors($work),
language: getWorkLanguage($$work),
words: getWorkWordCount($$work),
chapters: {
published: publishedChapters,
total: totalChapters,
},
complete: totalChapters !== null && totalChapters === publishedChapters,
stats: {
bookmarks: getWorkBookmarkCount($$work),
kudos: getWorkKudosCount($$work),
hits: getWorkHits($$work),
},
};
};

const getSeriesWorkTitle = ($work: SeriesWork) => {
return $work("h4.heading a[href*='/works/']").text().trim();
};

const getSeriesWorkUpdateDate = ($work: SeriesWork) => {
const [day, month, year] = $work("p.datetime").text().trim().split(" ");
return `${year}-${monthMap[month]}-${day}`;
};

const getSeriesWorkSummary = ($work: SeriesWork) => {
const summary = $work("blockquote.summary").html();
return summary ? summary.trim() : null;
};

const getSeriesWorkFandoms = ($work: SeriesWork): string[] => {
const fandoms: string[] = [];

$work("h5.fandoms a.tag").each(function (i, element) {
fandoms[i] = $work(element).text().trim();
});
return fandoms;
};

const getSeriesWorkCharacters = ($work: SeriesWork): string[] => {
const characters: string[] = [];

$work("li.characters a.tag").each(function (i, character) {
characters[i] = $work(character).text().trim();
});
return characters;
};

const getSeriesWorkRelationships = ($work: SeriesWork): string[] => {
const ships: string[] = [];

$work("li.relationships a.tag").each(function (i, ship) {
ships[i] = $work(ship).text().trim();
});
return ships;
};

const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => {
const tags: string[] = [];

$work("li.freeforms a.tag").each(function (i) {
tags[i] = $work(this).text().trim();
});
return tags;
};

const getSeriesWorkAuthors = (
$work: SeriesWork
): SeriesWorkSummary["authors"] => {
const authorLinks = $work("h4.heading a[rel='author']");
const authors: Author[] = [];

if ($work("h4.heading").text().split("by")[1].trim() === "Anonymous") {
return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }];
}

if (authorLinks.length !== 0) {
authorLinks.each((i, element) => {
const url = element.attribs.href;
const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;

authors.push({
username: username,
pseud: decodeURI(pseud),
anonymous: false,
});
});
}

return authors;
};
4 changes: 2 additions & 2 deletions src/tags/search-getters.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { TagSearchPage } from "src/page-loaders";
import { TagSearchPage, UserWorksPage } from "src/page-loaders";
import { TagSearchResultSummary } from "types/entities";

const parseIntOrThrow = (text: string) => {
Expand All @@ -17,7 +17,7 @@ export const getTotalResults = (page: TagSearchPage) => {
return totalResultsMatch ? parseIntOrThrow(totalResultsMatch[1]) : 0;
};

export const getPagesCount = (page: TagSearchPage) => {
export const getPagesCount = (page: TagSearchPage | UserWorksPage) => {
const lastPageMatch = page(".pagination.actions li:not(.next, .previous)")
.last()
.text();
Expand Down
Loading
Loading