feat: add Typst documentation (0.15.1)

lib/docs/filters/typst/clean_html.rb

@@ -0,0 +1,101 @@
+module Docs
+  class Typst
+    class CleanHtmlFilter < Filter
+      def call
+        # The container keeps the sidebar so internal_urls can crawl it.
+        # Once we have collected all links, we keep only the page body.
+        @doc = at_css('main') || raise("missing main on #{current_url}")
+
+        css('.breadcrumbs', '.page-end-buttons', '.sources-link', '.tooltip-context',
+            '.search', 'button.copy').remove
+
+        # Category lists rely on CSS columns to separate names and descriptions.
+        # We separate them with an em-dash to make them readable.
+        css('.catgrid > li > a + span').each { |node| node.before(' — ') }
+
+        # Adjacent type pills are unions, separated visually by the site's CSS.
+        css('.pill + .pill').each { |node| node.before(' or ') }
+
+        # Parameter types, modifiers, and defaults live inside headings.
+        # Move them into a paragraph so headings stay readable.
+        css('h1, h2, h3, h4, h5, h6').each do |heading|
+          heading.css('> .additional-info', '> small').reverse_each do |node|
+            node.css('small.mod').each { |modifier| modifier.after(' ') }
+            node.css('> div').each do |types|
+              types.name = 'span'
+              types.after(' ')
+            end
+            node.name = 'p'
+            heading.after(node)
+          end
+        end
+
+        css('pre').each do |node|
+          code = node.at_css('> code')
+          language = code && code['data-lang']
+          if language
+            # Prism doesn't support Typst yet,
+            # but we leave the language annotation in case it's implemented later.
+            language = 'typst' if language.start_with?('typ')
+            node['data-language'] = language == 'sh' ? 'bash' : language
+          end
+          node.content = node.content
+        end
+
+        css('.code-definition').each do |node|
+          node.css('div.arguments .overview-param').each { |param| param.before("\n  ") }
+          node.css('div.arguments').each do |args|
+            args.add_child("\n")
+            args.name = 'span'
+          end
+          node.name = 'pre'
+        end
+
+        # Open examples work without the source site's toggle scripts.
+        css('details').each { |node| node['open'] = '' }
+        css('summary img').remove
+
+        # The original symbol picker needs JS to display character metadata.
+        # A table retains every symbol and its anchor for offline lookup.
+        css('.symbol-grid').each do |grid|
+          table = Nokogiri::XML::Node.new('table', doc.document)
+          table.inner_html = '<thead><tr><th>Name</th><th>Symbol</th><th>Unicode name</th><th>LaTeX</th></tr></thead><tbody></tbody>'
+          body = table.at_css('tbody')
+          grid.css('li[id]').each do |symbol|
+            row = Nokogiri::XML::Node.new('tr', doc.document)
+            row['id'] = symbol['id']
+            name = symbol.at_css('code') || raise("missing name for ##{symbol['id']} on #{current_url}")
+            code = Nokogiri::XML::Node.new('code', doc.document)
+            code.content = name.content
+            cell = Nokogiri::XML::Node.new('td', doc.document)
+            cell.add_child(code)
+            row.add_child(cell)
+            [symbol['data-value'], symbol['data-unic-name']].each do |value|
+              cell = Nokogiri::XML::Node.new('td', doc.document)
+              cell.content = value.to_s
+              row.add_child(cell)
+            end
+            cell = Nokogiri::XML::Node.new('td', doc.document)
+            if symbol['data-latex-name'].present?
+              latex = Nokogiri::XML::Node.new('code', doc.document)
+              latex.content = symbol['data-latex-name']
+              cell.add_child(latex)
+            end
+            row.add_child(cell)
+            body.add_child(row)
+          end
+          grid.replace(table)
+        end
+        css('p:contains("Click on a")').each do |node|
+          node.remove if node.content.include?('to copy it to the clipboard')
+        end
+
+        css('[class]', '[style]', '[aria-describedby]', '[data-lang]').each do |node|
+          %w(class style aria-describedby data-lang).each { |attribute| node.remove_attribute(attribute) }
+        end
+
+        doc
+      end
+    end
+  end
+end

lib/docs/filters/typst/entries.rb

@@ -0,0 +1,89 @@
+module Docs
+  class Typst
+    class EntriesFilter < Docs::EntriesFilter
+      NAMESPACES = %w(math pdf html).freeze
+
+      def get_name
+        heading = at_css('main h1') || raise("missing h1 on #{current_url}")
+        definition = definition_node(heading)
+        return heading.content.strip unless definition
+
+        prefix = NAMESPACES.include?(category) ? "#{category}." : ''
+        suffix = definition.name == 'code' ? '()' : ''
+        "#{prefix}#{definition.content.strip}#{suffix}"
+      end
+
+      def get_type
+        return 'Language' if %w(syntax styling scripting context).include?(category)
+        return name if slug.split('/').length == 1
+
+        crumb = at_css('.breadcrumbs li:nth-last-child(3) a')
+        raise "missing category breadcrumb on #{current_url}" unless crumb
+        crumb.content.strip
+      end
+
+      def additional_entries
+        return [] if root_page?
+
+        entries = css('main .scoped-definition').map do |node|
+          # Definitions belong to a type/function (array.map, table.cell).
+          # Grouped functions and constants belong to a module (calc.abs, math.upright).
+          prefix = node['id'].start_with?('definitions-') ? name.delete_suffix('()') : namespace
+          # Nested types, such as color.spot, have their own definitions.
+          scopes = node['id'].split('-definitions-')[0...-1]
+          scopes.map! { |scope| scope.sub(/\A(?:definitions|functions)-/, '') }
+          prefix = ([prefix] + scopes).join('.')
+          definition = definition_node(node)
+          raise "missing name in ##{node['id']} on #{current_url}" unless definition
+          suffix = definition.name == 'code' && function_definition?(node) ? '()' : ''
+          ["#{prefix}.#{definition.content.strip}#{suffix}", node['id']]
+        end
+
+        css('main h3[id^="parameters-"] > code').each do |node|
+          entries << ["#{name.delete_suffix('()')}.#{node.content.strip}", node.parent['id']]
+        end
+
+        if category == 'symbols'
+          css('.symbol-grid li[id]').each do |node|
+            entries << ["#{slug.split('/').last}.#{symbol_name(node)}", node['id']]
+          end
+        elsif type == 'Language'
+          css('main h2[id]').each do |node|
+            entries << ["#{name}: #{node.content.strip}", node['id']]
+          end
+        end
+
+        entries
+      end
+
+      private
+
+      def category
+        slug.split('/').first
+      end
+
+      def namespace
+        NAMESPACES.include?(category) ? category : slug.split('/').last
+      end
+
+      def definition_node(heading)
+        heading.at_css('> code') || heading.at_css('> .pill')
+      end
+
+      def symbol_name(node)
+        code = node.at_css('code')
+        raise "missing name for ##{node['id']} on #{current_url}" unless code
+        code.content
+      end
+
+      def function_definition?(node)
+        sibling = node.next_element
+        while sibling && !sibling.name.match?(/\Ah[1-6]\z/)
+          return true if sibling['class'].to_s.split.include?('code-definition')
+          sibling = sibling.next_element
+        end
+        false
+      end
+    end
+  end
+end

lib/docs/scrapers/typst.rb

@@ -0,0 +1,27 @@
+module Docs
+  class Typst < UrlScraper
+    self.type = 'simple'
+    self.release = '0.15.1'
+    self.base_url = 'https://typst.app/docs/reference/'
+    self.links = {
+      home: 'https://typst.app/',
+      code: 'https://github.com/typst/typst'
+    }
+
+    html_filters.push 'typst/entries', 'typst/clean_html'
+
+    options[:container] = '.main-grid'
+    # Some rendered examples exceed the core filter's 120 KB image limit.
+    # The largest image in the bundle is 355 KB.
+    options[:max_image_size] = 400_000
+
+    options[:attribution] = <<-HTML
+      &copy; Typst GmbH and contributors<br>
+      Licensed under the Apache License, Version 2.0.
+    HTML
+
+    def get_latest_version(opts)
+      get_latest_github_release('typst', 'typst', opts)
+    end
+  end
+end

public/icons/docs/typst/SOURCE

@@ -0,0 +1,2 @@
+https://typst.app/assets/favicon-16x16.png
+https://typst.app/assets/favicon-32x32.png