|
1 | 1 | import axios from 'axios' |
2 | | -import { JSDOM } from 'jsdom' |
| 2 | +import { parseHTML } from 'linkedom' |
3 | 3 | import * as dns from 'dns' |
4 | 4 | import * as http from 'http' |
5 | 5 | import * as https from 'https' |
6 | 6 | import { Readability, isProbablyReaderable } from '@mozilla/readability' |
7 | | -import TurndownServiceJoplin from '@joplin/turndown' |
8 | | -import * as turndownPluginGfm from '@joplin/turndown-plugin-gfm' |
9 | | -import type TurndownService from 'turndown' |
| 7 | +import TurndownService from 'turndown' |
10 | 8 |
|
11 | 9 | const remove_markdown_images = (text: string) => { |
12 | 10 | const without_images = text.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, '') |
13 | 11 | return without_images.replace(/\n{3,}/g, '\n\n') |
14 | 12 | } |
15 | 13 |
|
16 | 14 | const create_turndown_service = () => { |
17 | | - const turndown_service: TurndownService = new TurndownServiceJoplin({ |
| 15 | + const turndown_service = new TurndownService({ |
18 | 16 | codeBlockStyle: 'fenced' |
19 | 17 | }) |
20 | | - turndown_service.use(turndownPluginGfm.gfm) |
21 | 18 | turndown_service.addRule('fencedCodeBlock', { |
22 | 19 | filter: (node: any, options: any) => { |
23 | 20 | return ( |
@@ -189,8 +186,10 @@ export const fetch_website = async ( |
189 | 186 |
|
190 | 187 | if (response.status === 200 && typeof response.data === 'string') { |
191 | 188 | const html = response.data |
192 | | - const dom = new JSDOM(html, { url }) |
193 | | - const doc = dom.window.document |
| 189 | + const { document: doc } = parseHTML(html) |
| 190 | + try { |
| 191 | + Object.defineProperty(doc, 'documentURI', { value: url }) |
| 192 | + } catch {} |
194 | 193 |
|
195 | 194 | doc.querySelectorAll('pre').forEach((pre) => { |
196 | 195 | const code = pre.querySelector('code') |
@@ -229,15 +228,17 @@ export const fetch_website = async ( |
229 | 228 | } |
230 | 229 | }) |
231 | 230 |
|
232 | | - if (isProbablyReaderable(doc)) { |
233 | | - const reader = new Readability(doc, { keepClasses: true }) |
| 231 | + if (isProbablyReaderable(doc as any)) { |
| 232 | + const reader = new Readability(doc as any, { keepClasses: true }) |
234 | 233 | const article = reader.parse() |
235 | 234 |
|
236 | 235 | if (article && article.content) { |
237 | 236 | const turndown_service = create_turndown_service() |
238 | | - const article_dom = new JSDOM(article.content) |
| 237 | + const { document: article_doc } = parseHTML( |
| 238 | + `<!DOCTYPE html><html><body>${article.content}</body></html>` |
| 239 | + ) |
239 | 240 | let content = turndown_service.turndown( |
240 | | - article_dom.window.document.body |
| 241 | + (article_doc.body || article_doc.documentElement) as any |
241 | 242 | ) |
242 | 243 | content = remove_markdown_images(content) |
243 | 244 |
|
|
0 commit comments