Skip to content

Commit 42e7cd9

Browse files
committed
Extend website fetcher to support JSON and plain text responses
- allow pasting url that has plain text only, then parse into code block e.g. `https://api.github.com/repos/robertpiosik/CodeWebChat/tags` (json) ├── apps │ └── editor │ └── src │ ├── utils │ │ └── get-markdown-backticks.ts │ └── views │ └── prompt │ ├── backend │ │ ├── message-handlers │ │ │ ├── handle-paste-url.ts │ │ │ └── index.ts │ │ ├── prompt-view-provider.ts │ │ └── utils │ │ ├── persistent-prompt-storage.ts │ │ ├── symbols │ │ │ ├── replace-symbols.ts │ │ │ └── website │ │ │ └── replace-website-symbol.ts │ │ └── website-fetcher.ts │ ├── frontend │ │ ├── Main │ │ │ ├── Main.tsx │ │ │ └── MainView │ │ │ └── MainView.tsx │ │ ├── Prompt.tsx │ │ └── hooks │ │ └── use-prompt │ │ └── use-prompt.ts │ └── types │ └── messages.ts └── packages └── ui └── src └── components └── editor └── common └── prompts └── PromptField ├── PromptField.tsx └── hooks └── use-handlers └── handle-paste.ts
1 parent a8f2e5e commit 42e7cd9

1 file changed

Lines changed: 91 additions & 52 deletions

File tree

‎apps/editor/src/views/prompt/backend/utils/website-fetcher.ts‎

Lines changed: 91 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -184,68 +184,107 @@ export const fetch_website = async (
184184
httpsAgent: https_agent
185185
})
186186

187-
if (response.status === 200 && typeof response.data === 'string') {
188-
const html = response.data
189-
const { document: doc } = parseHTML(html)
190-
try {
191-
Object.defineProperty(doc, 'documentURI', { value: url })
192-
} catch {
193-
// Ignore
187+
if (response.status === 200) {
188+
const content_type = (
189+
(response.headers as any)['content-type'] || ''
190+
).toLowerCase()
191+
192+
if (
193+
content_type.includes('application/json') ||
194+
typeof response.data === 'object'
195+
) {
196+
const content =
197+
typeof response.data === 'string'
198+
? response.data
199+
: JSON.stringify(response.data, null, 2)
200+
return { content, title: '' }
194201
}
195202

196-
doc.querySelectorAll('pre').forEach((pre) => {
197-
const code = pre.querySelector('code')
198-
const language = (code?.className.match(/language-(\S+)/) ||
199-
pre.className.match(/language-(\S+)/) || [null, ''])[1]
203+
if (typeof response.data === 'string') {
204+
const is_html =
205+
content_type.includes('text/html') ||
206+
response.data.trim().toLowerCase().startsWith('<!doctype html') ||
207+
response.data.trim().toLowerCase().startsWith('<html')
200208

201-
let text = ''
202-
const lines = pre.querySelectorAll('.line')
203-
if (lines.length > 0) {
204-
text = Array.from(lines)
205-
.map((l) => l.textContent)
206-
.join('\n')
207-
} else {
208-
text = pre.textContent || ''
209-
}
209+
if (!is_html) {
210+
const is_text =
211+
content_type.includes('text/') ||
212+
content_type.includes('json') ||
213+
content_type.includes('xml') ||
214+
content_type.includes('javascript') ||
215+
content_type.includes('yaml') ||
216+
content_type.includes('csv') ||
217+
content_type === ''
210218

211-
pre.innerHTML = ''
212-
const new_code = doc.createElement('code')
213-
if (language) {
214-
new_code.className = `language-${language}`
219+
if (is_text) {
220+
return { content: response.data, title: '' }
221+
}
222+
return null
215223
}
216-
new_code.textContent = text
217-
pre.appendChild(new_code)
218224

219-
let current: Element = pre
220-
while (
221-
current.parentElement &&
222-
current.parentElement.tagName === 'DIV' &&
223-
current.parentElement.textContent?.trim() === pre.textContent?.trim()
224-
) {
225-
current = current.parentElement
225+
const html = response.data
226+
const { document: doc } = parseHTML(html)
227+
try {
228+
Object.defineProperty(doc, 'documentURI', { value: url })
229+
} catch {
230+
// Ignore
226231
}
227-
if (current !== pre && current.parentElement) {
228-
current.parentElement.insertBefore(pre, current)
229-
current.remove()
230-
}
231-
})
232232

233-
if (isProbablyReaderable(doc as any)) {
234-
const reader = new Readability(doc as any, { keepClasses: true })
235-
const article = reader.parse()
233+
doc.querySelectorAll('pre').forEach((pre) => {
234+
const code = pre.querySelector('code')
235+
const language = (code?.className.match(/language-(\S+)/) ||
236+
pre.className.match(/language-(\S+)/) || [null, ''])[1]
236237

237-
if (article && article.content) {
238-
const turndown_service = create_turndown_service()
239-
const { document: article_doc } = parseHTML(
240-
`<!DOCTYPE html><html><body>${article.content}</body></html>`
241-
)
242-
let content = turndown_service.turndown(
243-
(article_doc.body || article_doc.documentElement) as any
244-
)
245-
content = remove_markdown_images(content)
238+
let text = ''
239+
const lines = pre.querySelectorAll('.line')
240+
if (lines.length > 0) {
241+
text = Array.from(lines)
242+
.map((l) => l.textContent)
243+
.join('\n')
244+
} else {
245+
text = pre.textContent || ''
246+
}
247+
248+
pre.innerHTML = ''
249+
const new_code = doc.createElement('code')
250+
if (language) {
251+
new_code.className = `language-${language}`
252+
}
253+
new_code.textContent = text
254+
pre.appendChild(new_code)
255+
256+
let current: Element = pre
257+
while (
258+
current.parentElement &&
259+
current.parentElement.tagName === 'DIV' &&
260+
current.parentElement.textContent?.trim() ===
261+
pre.textContent?.trim()
262+
) {
263+
current = current.parentElement
264+
}
265+
if (current !== pre && current.parentElement) {
266+
current.parentElement.insertBefore(pre, current)
267+
current.remove()
268+
}
269+
})
270+
271+
if (isProbablyReaderable(doc as any)) {
272+
const reader = new Readability(doc as any, { keepClasses: true })
273+
const article = reader.parse()
274+
275+
if (article && article.content) {
276+
const turndown_service = create_turndown_service()
277+
const { document: article_doc } = parseHTML(
278+
`<!DOCTYPE html><html><body>${article.content}</body></html>`
279+
)
280+
let content = turndown_service.turndown(
281+
(article_doc.body || article_doc.documentElement) as any
282+
)
283+
content = remove_markdown_images(content)
246284

247-
if (content && content.trim().length > 0) {
248-
return { content, title: article.title || doc.title || '' }
285+
if (content && content.trim().length > 0) {
286+
return { content, title: article.title || doc.title || '' }
287+
}
249288
}
250289
}
251290
}

0 commit comments

Comments
 (0)