Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 113 additions & 0 deletions src/services/__tests__/__snapshots__/chrome-crawler.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -223,3 +223,116 @@ Absolutely! Blync is 100% secure.
"weeklyActiveUsers": 40,
}
`;

exports[`Chrome Web Store Crawler should extract extension details from 2026-04-17-mgmdkjcljneegjfajchedjpdhbadklcf.html 1`] = `
{
"iconUrl": "https://lh3.googleusercontent.com/lTplXRdnpEB-7DGRa_1nCKao_3aJ3C_e-GNVs9tQV9hDUXgupc1SsW6OrruxgrkSdFBSOqeia56YWgJI2IgpV1MK49s=s256",
"id": "mgmdkjcljneegjfajchedjpdhbadklcf",
"lastUpdated": "January 27, 2024",
"longDescription":
"Watch anime faster than ever! The Anime Skip Player lets you automatically skip intros, outros, or anything else that you don't want to watch.
Anime Skip replaces the video player of streaming services with it's own, standardizing the viewing experience.

✨ Features
• Customize what you want to watch and skip the rest
• Change the playback speed
• Customize your keyboard shortcuts
• Multiple color themes
• Screenshots
• Contribute timestamps and episode info for the community

🌐 Streaming Services
• Crunchyroll
• Aniwave
• Aniwatch
With more coming soon!

☔ Safe
The extension is open sourced!
https://github.com/anime-skip/web-extension

💕 Enjoying Anime Skip?
Leave a 5 star review! Reviews help build trust and attract new users, which means more timestamps!"
,
"name": "Anime Skip Player",
"rating": 4.4,
"reviewCount": 42,
"screenshots": [
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/tIwJ7CWPxY411qKw2_TEkdGAorjCnP8OQJMtLZ1t_4Njx62uP4a58xE8BUzvKpghVoGn4ukFYrw4H5gB-I5WQiAx=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/d9aFrUwRB5QZTK8HNxJJgZ1Mb4FGIdLlTqbTeO7RfIl9YlZvO4_gRchpnjX5g9OTEocfkiS3ge1e257UBGt17iGKACI=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/5PBHI8N5FXG2_kJY69KgVUotwRLdxvkzjUnkkXIyFMS8F3y1mtqfIWwkfRodqDH-wcCw9iEObpzXH6vfJ2tiqugsWA=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/OZ2XFtw0RIGPxeVrBubaZWrK9KJSKz51bwFI6z8m3ZLow9RPN_-zEmxuz48HUfTDG51J97LclUd2ydtFfyRbKm0xMA=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/lv8T04EmvcMaRAVJ6EiqG5yLa-EUE4l0sTtPzKWLXflkbkkV6dGd1OIhSCJUFj3DlRLc5SWFpuboCAw9LcY73sH7x1M=s1280",
},
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/tIwJ7CWPxY411qKw2_TEkdGAorjCnP8OQJMtLZ1t_4Njx62uP4a58xE8BUzvKpghVoGn4ukFYrw4H5gB-I5WQiAx=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/d9aFrUwRB5QZTK8HNxJJgZ1Mb4FGIdLlTqbTeO7RfIl9YlZvO4_gRchpnjX5g9OTEocfkiS3ge1e257UBGt17iGKACI=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/5PBHI8N5FXG2_kJY69KgVUotwRLdxvkzjUnkkXIyFMS8F3y1mtqfIWwkfRodqDH-wcCw9iEObpzXH6vfJ2tiqugsWA=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/OZ2XFtw0RIGPxeVrBubaZWrK9KJSKz51bwFI6z8m3ZLow9RPN_-zEmxuz48HUfTDG51J97LclUd2ydtFfyRbKm0xMA=s1280",
},
{
"index": 0,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/0",
"rawUrl": "https://lh3.googleusercontent.com/5PBHI8N5FXG2_kJY69KgVUotwRLdxvkzjUnkkXIyFMS8F3y1mtqfIWwkfRodqDH-wcCw9iEObpzXH6vfJ2tiqugsWA=s1280",
},
{
"index": 1,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/1",
"rawUrl": "https://lh3.googleusercontent.com/OZ2XFtw0RIGPxeVrBubaZWrK9KJSKz51bwFI6z8m3ZLow9RPN_-zEmxuz48HUfTDG51J97LclUd2ydtFfyRbKm0xMA=s1280",
},
{
"index": 2,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/2",
"rawUrl": "https://lh3.googleusercontent.com/lv8T04EmvcMaRAVJ6EiqG5yLa-EUE4l0sTtPzKWLXflkbkkV6dGd1OIhSCJUFj3DlRLc5SWFpuboCAw9LcY73sH7x1M=s1280",
},
{
"index": 3,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/3",
"rawUrl": "https://lh3.googleusercontent.com/tIwJ7CWPxY411qKw2_TEkdGAorjCnP8OQJMtLZ1t_4Njx62uP4a58xE8BUzvKpghVoGn4ukFYrw4H5gB-I5WQiAx=s1280",
},
{
"index": 4,
"indexUrl": "http://localhost:3000/api/rest/chrome-web-store/mgmdkjcljneegjfajchedjpdhbadklcf/screenshots/4",
"rawUrl": "https://lh3.googleusercontent.com/d9aFrUwRB5QZTK8HNxJJgZ1Mb4FGIdLlTqbTeO7RfIl9YlZvO4_gRchpnjX5g9OTEocfkiS3ge1e257UBGt17iGKACI=s1280",
},
],
"shortDescription": "Custom video player for anime streaming websites. Skip intros, outros, and more.",
"storeUrl": "https://chromewebstore.google.com/detail/anime-skip-player/mgmdkjcljneegjfajchedjpdhbadklcf",
"users": 2000,
"version": "2.0.3",
"weeklyActiveUsers": 2000,
}
`;

Large diffs are not rendered by default.

39 changes: 23 additions & 16 deletions src/services/chrome-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export async function crawlExtension(
// const date = new Date();
// const dateString = `${date.getFullYear()}-${(date.getMonth() + 1).toString().padStart(2, "0")}-${date.getDate().toString().padStart(2, "0")}`;
// Bun.write(
// `src/crawlers/__tests__/fixtures/chrome-web-store/.new/${dateString}-${id}.html`,
// `src/services/__tests__/fixtures/chrome-web-store/.new/${dateString}-${id}.html`,
// html,
// );
}
Expand Down Expand Up @@ -60,21 +60,28 @@ export async function crawlExtension(

// Header

const weeklyActiveUsersText = tryExtract("weeklyActiveUsers", validateInt, [
() => {
const userCountRow = document.querySelector(
"main > * > section:first-child > section > div > div:last-child",
) as HTMLElement | null;
removeAnchorChildren(userCountRow);
return (
userCountRow?.textContent
// "W,XYZ+ users"
?.replace(" users", "")
.replaceAll(",", "")
.replace("+", "")
);
},
]);
const cleanupUserCount = (str: string | undefined) =>
str
// "W,XYZ+ users"
?.replace(" users", "")
.replaceAll(",", "")
.replace("+", "");
const userCountSelectors = [
"main > * > section:first-child > section > div > div:nth-last-child(2)",
"main > * > section:first-child > section > div > div:last-child",
];
const weeklyActiveUsersText = tryExtract(
"weeklyActiveUsers",
validateInt,
userCountSelectors.map((selector) => () => {
const row = document.querySelector(selector) as HTMLElement | null;
if (!row?.textContent?.includes("users"))
throw Error("Row did not include users");

removeAnchorChildren(row);
return cleanupUserCount(row?.textContent);
}),
);
const weeklyActiveUsers = Number(weeklyActiveUsersText);

const rating = tryExtract("rating", validateFloat, [
Expand Down