Automatic documentation of variables and functions

doc/manual/Makefile

+EMACS=emacs-21.4
 SRC=../../src
 
-CHAPTERS=texts.xml \
-	 commands.xml
+CHAPTERS=intro.xml \
+	 texts.xml \
+	 etiquette.xml \
+	 commands.xml \
+	 variables.xml \
+         dummy.xml \
+	 glossary.xml
 
+GENFILES=fnc.ent var.ent editcmd.ent commands.xml variables.xml dummy.xml
 INCFILES=$(CHAPTERS:%=%.inc)
 
 
 # Generate the manual from DocBook XML using Jade wrappers
 # I really don't know if the jw command is standard in installations
-# of Jade, but it works fine in Debian/Gnu Linux.
+# of Jade, but it works fine in Debian/Gnu Linux. The path to the 
+# stylesheet is also Linux (maybe Debian) specific.
 
-manual.ps: $(INCFILES) manual.xml chp.xml
-	jw -f docbook -b ps manual.xml
+manual.ps: $(INCFILES) $(GENFILES) manual.xml chp.xml
+	jw -f docbook -d '/usr/share/sgml/docbook/stylesheet/dsssl/ldp/ldp.dsl#print' -b ps manual.xml
 
 
 # These files are generated from the source code.
 
-fnc.ent var.ent commands.xml: $(SRC)/lyskom.el
-	emacs -batch -l $(SRC)/lyskom.el -l ./docgen.el -f lyskom-docgen
+$(GENFILES): $(SRC)/lyskom.el
+	$(EMACS) -batch -l $(SRC)/lyskom.el -l ./docgen.el -f lyskom-docgen
 
 
 # This file is generated from the Makefile and contains references
 # to all the chapters.
 
-chp.xml: Makefile
+chp.xml: Makefile docgen.el
 	echo > $@
 	for i in $(INCFILES) ; do \
 		echo "<!ENTITY $$i SYSTEM \"$$i\">" >> $@ ; \
@@ -38,9 +46,13 @@ chp.xml: Makefile
 $(SRC)/lyskom.el:
 	cd $(SRC) && make lyskom.el
 
+clean:
+	rm -f $(GENFILES)  $(INCFILES) chp.xml
+
 
 # The .xml.inc files are the chapter files without the surrounding
-# book element. They are built using the xml2inc script.
+# book element. Probably XSLT would be the Right Thing, but sed 
+# gets the job done too
 
 %.xml.inc: %.xml
-	./xml2inc < $< > $@
+	sed -e '/<book[ >]/,/<\/book>/! d' -e '/<book[ >]/ d' -e '/<\/book>/ d' < $< > $@

doc/manual/docgen.el

-;;;;; -*-coding: iso-8859-1;-*-
+;;;; -*-coding: iso-8859-1;-*-
 ;;;;;
 ;;;;; $Id$
 ;;;;; Copyright (C) 1991-2002  Lysator Academic Computer Association.
@@ -29,11 +29,21 @@
 ;;;; File: docgen.el
 ;;;;
 ;;;; Generate documentation data for inclusion in the user manual
-;;;;
+;;;; This should be run in Gnu Emacs 21.4. It works OK in 21.3 too,
+;;;; but fails to generate all the key bindings.
 
 
 ;;; TO DO
 ;;;
+
+;;; Extract documentation from edit-text. All commands with the prefix
+;;; kom-edit- whose docstrings do not end in "Excluded from manual."
+;;; should be extracted and put into entities like the regular
+;;; commands. Then create another entity that we can use to insert
+;;; them into the texts.xml file.
+
+;;; Something like this should be done for mship-edit too.
+
 ;;; Variable documentation requires considerably more flexibility
 ;;; than command documentation:
 ;;;
@@ -50,31 +60,22 @@
 ;;;
 ;;; In variables, warn for undocumented (no **) def-kom-vared
 ;;; variables that start with kom-
-;;;
-;;; Generate DTD fragments so we can use entities for command and
-;;; variable references (e.g. &fn:kom-mark-text;) in XML.
-;;;
+
 ;;; Read templates from a file, so we don't hard-code the
 ;;; format here.
-;;;
+
 ;;; Specify output file on the command line.
-;;;
+
 ;;; Specify lyskom.elc on the command line.
-;;;
+
 ;;; Read a list of external references from a file. Ideally we'd read
 ;;; and parse the actual XML, but that is probably too much work. We
 ;;; need to map references like lyskom-read-text-no-prefix-arg to the
 ;;; actual ID to link to. Specify this file on the command line too.
-;;;
+
 ;;; Do variable documentation.
-;;;
-;;; Write or find a stylesheed that doesn't suck. Mainly paragraph
-;;; spacing is way too big, and the period after section numbers is
-;;; just plain wrong. Maybe this can be fixed by setting parameters to
-;;; the DSSSL processor -- see
-;;; /usr/share/sgml/docbook/stylesheet/dsssl/modular/print/dbparam.dsl
-;;; for all available parameters. %para-sep% is interesting.
-;;;
+
+
 
 (require 'cl)
 
@@ -114,116 +115,346 @@
 
 (defun lyskom-docgen-get-name (fn lang)
   (let ((lyskom-language lang))
-    (lyskom-command-name fn)))
+    (lyskom-command-name fn lang)))
 
-(defun lyskom-docgen-get-doc (fn)
+(defun lyskom-docgen-function-doc (fn)
   (let ((doc (documentation fn)))
+    (when (and doc (string-match "\\`\\*" doc))
+      (setq doc (substring doc 1)))
     (when (and doc (string-match "Lisp documentation:" doc))
       (setq doc (substring doc 0 (match-beginning 0))))
-    (lyskom-docgen-transform-refs fn (lyskom-docgen-fixup-sgml doc))))
-
-(defun lyskom-docgen-transform-refs (fn doc)
-  (if doc
-      (let ((result "")
-            (start 0))
-        (while (string-match "`\\(.*?\\)'" doc start)
-          (setq result (concat result (substring doc start (match-beginning 0))))
-          (let ((sym (intern (match-string 1 doc))))
-            (cond 
-             ((assq sym lyskom-docgen-external-refs)
-              (setq result
-                    (concat result
-                            (format "<xref linkend=\"%s\" />" 
-                                    (cdr (assq sym lyskom-docgen-external-refs))))))
-             ((fboundp sym)
-              (unless (memq sym lyskom-commands)
-                (lyskom-docgen-error "%s: reference to non-command %s" fn sym))
-              (setq result
-                    (concat result
-                            (format "<link linkend=\"fn:%s\"><command>%s</command></link> [<link linkend=\"fn:%s\"><function>%s</function></link>]"
-                                    sym
-                                    (lyskom-docgen-get-name fn 'en)
-                                    sym
-                                    sym)))
-              (setq lyskom-docgen-xrefd-functions 
-                    (add-to-list 'lyskom-docgen-xrefd-functions sym)))
-             ((boundp sym)
-              (setq result
-                    (concat result
-                            (format "<link linkend=\"var:%s\">%s</link>"
-                                    sym sym)))
-              (setq lyskom-docgen-xrefd-variables 
-                    (add-to-list 'lyskom-docgen-xrefd-variables sym)))
-             (t (lyskom-docgen-error "%s: undefined reference to %s" fn sym)
-                (setq result (concat result 
-                                     "`"
-                                     (match-string 1 doc)
-                                     "'"))))
-            (setq start (match-end 0))))
-        (setq result (concat result (substring doc start)))
-        (concat "      <para>\n"
-                (replace-in-string result "\n\n+" "\n      </para>\n      <para>\n")
-                "\n      </para>\n")
-        )
-    (lyskom-docgen-error "%s: No documentation!" fn)
-    "<para>Undocumented.</para>"))
+    doc))
+
+(defun lyskom-docgen-variable-doc (var)
+  (let ((doc (documentation-property var 'variable-documentation t)))
+    (and (stringp doc) (string-match "\\`\\*\\*" doc) (substring doc 2))))
+
+
+(defun lyskom-docgen-where-is-internal (command lang &optional map)
+  (let* ((map (or map (symbol-value (intern (format "lyskom-%s-mode-map" lang)))))
+         (trans (when (eq lang 'sv) `((?\] . ,(kbd "")) 
+                                      (?\} . ,(kbd "")) 
+                                      ( . ,(kbd ""))
+                                      ( . ,(kbd ""))
+                                      (3909 . ,(kbd ""))
+                                      (3941 . ,(kbd ""))
+                                      (2277 . ,(kbd ""))
+                                      (2245 . ,(kbd ""))
+                                      (aring . ,(kbd ""))
+                                      (Aring . ,(kbd ""))
+                                      (229 . ,(kbd ""))
+                                      (197 . ,(kbd ""))
+
+                                      (134217733 . ,(kbd "C-"))
+                                      (C-Aring . ,(kbd "C-"))
+                                      (C-aring . ,(kbd "C-"))
+                                      (67111109 . ,(kbd "C-"))
+                                      (67111141 . ,(kbd "C-"))
+                                      (67108989 . ,(kbd "C-"))
+                                      (,(kbd "C-}") . ,(kbd "C-"))
+
+                                      (?\[ . ,(kbd "")) 
+                                      (?\{ . ,(kbd ""))
+                                      ( . ,(kbd ""))
+                                      ( . ,(kbd ""))
+                                      (228 . ,(kbd ""))
+                                      (196 . ,(kbd ""))
+                                      (3908 . ,(kbd ""))
+                                      (3940 . ,(kbd "")) 
+                                      (2276 . ,(kbd ""))
+                                      (2244 . ,(kbd ""))
+                                      (Adiaeresis . ,(kbd ""))
+                                      (adiaeresis . ,(kbd ""))))))
+
+         (delete-duplicates (mapcar (lambda (binding)
+                                      (apply 'vector 
+                                             (apply 'append
+                                                    (mapcar (lambda (x) 
+							      (cond ((listp x) x)
+								    ((vectorp x) (append x nil))
+								    (t (list x))))
+                                                            (mapcar (lambda (key) 
+                                                                      (or (cdr (assoc key trans)) key))
+                                                                    binding)))))
+                                    (where-is-internal command map))
+                            :test 'equal)))
+
+(defun lyskom-docgen-fixup-key-description (desc)
+  (replace-in-string desc "\\(C-[A-Za-z0-9]+\\) TAB " "\\1 C-i "))
 
 (defun lyskom-docgen-format-key-sequence (binding)
-  (mapconcat
-   (lambda (x) 
-     (format "<keycap>%s</keycap>" 
-             (lyskom-docgen-fixup-sgml (key-description x))))
-   binding "; "))
+  (if binding
+      (mapconcat
+       (lambda (x) 
+         (format "<keycap>%s</keycap>" 
+                 (lyskom-docgen-fixup-sgml
+		  (lyskom-docgen-fixup-key-description
+		   (key-description x)))))
+       binding "; ")
+    "<emphasis>unbound</emphasis>"))
+
+(defmacro lyskom-docgen-parse-docstring-error ()
+  `(let ((msg (mapconcat 'identity (cons str lines) "\n")))
+     (error "Parse error in docstring. State: %s. Near: %s" 
+            state (substring msg 0 (min (length msg) 79)))))
+
+(defun lyskom-docgen-parse-docstring (str)
+  (when (string-match "^\\s-*\\**" str)
+    (setq str (substring str (match-end 0))))
+  (let* ((result nil)
+         (state 'initial)
+         (lines (split-string str "\n"))
+         (saved-lines lines)
+         (str nil)
+         (current nil))
+    (while lines
+      (setq str (car lines) lines (cdr lines))
+      (cond
+       ((eq state 'initial)
+        (setq current nil)
+        (cond
+         ((string-match "^\\*" str) (setq state 'ul lines (cons str lines)))
+         ((string-match "^\\s-*<\\s-*$" str) (setq state 'pre))
+         ((string-match "^\\S-" str) (setq state 'para lines (cons str lines)))
+         ((string-match "^\\s-+\\S-" str) (setq state 'table lines (cons str lines)))
+         ((string-match "^\\s-*$" str))
+         (t (lyskom-docgen-parse-docstring-error))))
+
+       ;; An ul block is
+       ;; (ul item item item item)
+       ;; the items are in reverse order
+
+       ((eq state 'ul)
+        (unless (eq 'ul (car current)) 
+          (setq current (cons 'ul nil))
+          (setq result (cons current result)))
+        (cond
+         ((string-match "^\\s-*$" str))
+         ((string-match "^\\*\\s-*" str) 
+          (setq state 'li lines (cons str lines)))
+         (t (setq state 'initial lines (cons str lines)))))
+
+       ((eq state 'li)
+        (cond
+         ((string-match "^\\s-*$" str) (setq state 'ul))
+         ((string-match "^\\*\\s-*" str)
+          (setcdr current (cons (substring str (match-end 0)) (cdr current))))
+         (t (setcar (cdr current) (concat (car (cdr current)) " " str)))))
+
+       ;; a para block is (para . string)
+
+       ((eq state 'para)
+        (unless (eq 'para (car current))
+          (setq current (cons 'para nil))
+          (setq result (cons current result)))
+        (cond
+         ((string-match "^\\s-*$" str) (setq state 'initial))
+         ((string-match "^\\*" str) (setq state 'initial))
+         (t (setcdr current (cons str (cdr current))))))
+
+       ;; a pre block is (pre . lines)
+
+       ((eq state 'pre)
+        (unless (eq 'pre (car current))
+          (setq current (cons 'pre nil))
+          (setq result (cons current result)))
+        (cond ((string-match "^\\s-*>\\s-*$" str) (setq state 'initial))
+              (t (setcdr current (cons str (cdr current))))))
+
+       ;; a table block is (table rows head)
+
+       ((eq state 'table)
+        (unless (eq 'table (car current))
+          (setq current (cons 'table nil))
+          (setq result (cons current result)))
+        (setq lines (cons str lines) state 'thead))
+
+       ((eq state 'thead)
+        (setcdr current (cons (cdr (string-split "\\s-\\s-+" str))
+                              (cdr current)))
+        (setq state 'thsep))
+
+       ((eq state 'thsep)
+        (unless (string-match "\\s-*-+\\s-*$" str)
+          (lyskom-docgen-parse-docstring-error))
+        (setq state 'tbody))
+
+       ((eq state 'tbody)
+        (cond
+         ((string-match "^\\s-*$" str))
+         ((string-match "^\\s-*-+\\s-*$" str) (setq state 'initial))
+         (t (setq state 'trow lines (cons str lines)))))
+
+       ((eq state 'trow)
+        (setcdr current (cons (cdr (string-split "\\s-\\s-+" str))
+                              (cdr current)))
+        (setq state 'trow2))
+
+       ((eq state 'trow2)
+        (cond 
+         ((string-match "^\\s-*$" str) (setq state 'tbody))
+         ((string-match "^\\s-*-+\\s-*$" str) (setq state 'initial))
+         (t (let ((cells (nreverse (cdr (string-split "\\s-\\s-+" str)))))
+              (setcar (cdr current)
+                      (nreverse
+                       (mapcar (lambda (cell)
+                                 (prog1
+                                     (concat cell " " (or (car cells) ""))
+                                   (setq cells (cdr cells))))
+                               (nreverse (car (cdr current))))))))))
+
+       (t (error "Unknown state in docstring parser: %s" state))
+
+
+       ))
+
+    result
+
+   (mapcar (lambda (el)
+              (cond ((eq (car el) 'pre)
+                     (setcdr el (mapconcat 'identity (nreverse (cdr el)) "\n")))
+                    ((eq (car el) 'para)
+                     (setcdr el (mapconcat 'identity (nreverse (cdr el)) " ")))
+                    ((listp (cdr el))
+                     (setcdr el (nreverse (cdr el)))))
+              el)
+            (nreverse result))
+
+))
+
+(defun lyskom-docgen-docstring-to-docbook (sym str)
+  (if str
+      (let ((parse (lyskom-docgen-parse-docstring str)))
+        (mapconcat 'identity
+                   (mapcar
+                    (lambda (el)
+                      (cond ((eq (car el) 'para)
+                             (concat "<para>\n"
+                                     (lyskom-docgen-markup-paragraph
+                                      sym (lyskom-docgen-fixup-sgml (cdr el)))
+                                     "\n</para>"))
+
+                            ((eq (car el) 'ul)
+                             (concat "<itemizedlist mark=\"bullet\">\n"
+                                     (mapconcat (lambda (item)
+                                                  (concat "<listitem>\n"
+                                                          "  <para>\n"
+                                                          (lyskom-docgen-markup-paragraph
+                                                           sym (lyskom-docgen-fixup-sgml item))
+                                                          "\n  </para>"
+                                                          "\n</listitem>"))
+                                                (cdr el)
+                                                "\n")
+                                     "\n</itemizedlist>"))
+
+                            ((eq (car el) 'table)
+                             (concat "<informaltable>\n"
+                                     (format "  <tgroup cols=\"%d\">\n" (length (car (cdr el))))
+                                     "    <thead>\n"
+                                     "      <row>\n"
+                                     (mapconcat (lambda (x)
+                                                  (concat "        <entry>"
+                                                          (lyskom-docgen-markup-paragraph
+                                                           sym (lyskom-docgen-fixup-sgml x))
+                                                          "</entry>"))
+                                                (car (cdr el))
+                                                "\n")
+                                     "\n      </row>"
+                                     "\n    </thead>\n"
+                                     "    <tbody>\n"
+                                     (mapconcat (lambda (row)
+                                                  (concat "      <row>\n"
+                                                          (mapconcat (lambda (x)
+                                                                       (concat "        <entry>"
+                                                                               (lyskom-docgen-markup-paragraph
+                                                                                sym (lyskom-docgen-fixup-sgml x))
+                                                                               "</entry>"))
+                                                                     row "\n")
+                                                          "\n    </row>"))
+                                                (cdr (cdr el))
+                                                "\n")
+                                     "\n    </tbody>"
+                                     "\n  </tgroup>"
+                                     "\n</informaltable>")
+                             )
+
+                            ((eq (car el) 'pre)
+                             (concat "<screen>\n"
+                                     (lyskom-docgen-fixup-sgml (cdr el))
+                                     "\n</screen>"))
+
+                            (t (error "Unknown docstring parse element: %s" el))))
+                    parse)
+                   "\n"))
 
+    (lyskom-docgen-error "%s: No documentation!" fn)
+    "<para>Undocumented</para>"))
 
-(defun lyskom-docgen-generate-commands-chapter-2 ()
-  (lyskom-traverse command lyskom-commands
-    (let* ((name-sv (lyskom-docgen-get-name command 'sv))
-           (name-en (lyskom-docgen-get-name command 'en))
-           (bind-sv (where-is-internal command lyskom-sv-mode-map))
-           (bind-en (where-is-internal command lyskom-en-mode-map))
-           (doc-en (lyskom-docgen-get-doc command)))
-      (lyskom-docgen-princ
-       (format "\
-    <section id=\"fn:%s\"><title>%s [%s]</title>
 
-    <informaltable frame=\"topbot\">
-      <tgroup cols=\"2\">
-        <tbody valign=\"top\">
-          <row>
-            <entry>English name</entry>
-            <entry>%s</entry>
-          </row>
-          <row>
-            <entry>English binding</entry>
-            <entry>%s</entry>
-          </row>
-          <row>
-            <entry>Swedish name</entry>
-            <entry>%s</entry>
-          </row>
-          <row>
-            <entry>Swedish binding</entry>
-            <entry>%s</entry>
-          </row>
-        </tbody>
-      </tgroup>
-    </informaltable>
-%s
-  </section>\n\n" 
-               command 
-               name-en
-               command 
-               name-en
-               (lyskom-docgen-format-key-sequence bind-en)
-               name-sv
-               (lyskom-docgen-format-key-sequence bind-sv)
-               doc-en))
-      (setq lyskom-docgen-found-functions
-            (cons command lyskom-docgen-found-functions)))))
+(defun lyskom-docgen-markup-paragraph (docsym doc)
+  (let ((result "")
+        (start 0)
+        (end nil))
+    (while (string-match "`\\(.*?\\)'" doc start)
+      (setq result (concat result (substring doc start (match-beginning 0)))
+            end (match-end 0))
+      (let* ((matched-string (match-string 1 doc))
+             (sym (intern matched-string)))
+        (cond 
+
+         ;; External reference
+
+         ((assq sym lyskom-docgen-external-refs)
+          (setq result
+                (concat result
+                        (format "<xref linkend=\"%s\" />" 
+                                (cdr (assq sym lyskom-docgen-external-refs))))))
+
+         ;; Reference to a LysKOM function or variable
+
+         ((string-match "^\\(lys\\)?kom-" matched-string)
+          (cond ((fboundp sym)
+                 (setq lyskom-docgen-xrefd-functions (add-to-list 'lyskom-docgen-xrefd-functions sym))
+                 (setq result
+                       (concat result
+                               (format "<link linkend=\"fn:%s\"><command>%s</command></link>" sym sym)
+                               (if (memq sym lyskom-commands)
+                                   (format " (<link linkend=\"fn:%s\"><command>%s</command></link>)"
+                                           sym (lyskom-docgen-get-name sym 'en))
+                                 ""))))
+                ((boundp sym)
+                 (setq lyskom-docgen-xrefd-variables (add-to-list 'lyskom-docgen-xrefd-variables sym))
+                 (setq result
+                       (concat result
+                               (format "<link linkend=\"var:%s\"><varname>%s</varname></link>" sym sym))))
+
+                (t
+                 (lyskom-docgen-error "%s: reference to undefined name %s" docsym sym)
+                 (setq result
+                       (concat result
+                               (format "<literal>%s</literal>" sym))))))
+
+         ;; All uppercase means a replaceable string
+
+         ((string-match "^[-_:A-Z]+$" matched-string)
+          (setq result (concat result (format "<replaceable>%s</replaceable>" matched-string))))
+
+         ;; Everything else is a literal
+
+         (t (setq result (concat result (format "<literal>%s</literal>" matched-string))))
+
+         )
+
+        (setq start end)))
+
+    (setq result (concat result (substring doc start)))))
+
 
 (defun lyskom-docgen-generate-missing-anchors ()
-  (lyskom-docgen-princ "<section><title>Dummy anchors</title><para>\n")
+  (message "Generating dummy.xml")
+  (lyskom-docgen-open-file "dummy.xml")
+  (lyskom-docgen-princ "<book>
+<chapter id=\"dummy\"><title>Dummy anchors</title><para>\n")
   (lyskom-traverse fn (set-difference lyskom-docgen-xrefd-functions
                                       lyskom-docgen-found-functions)
     (lyskom-docgen-princ
@@ -233,7 +464,9 @@
                                        lyskom-docgen-found-variables)
         (lyskom-docgen-princ
          (format "<anchor id=\"var:%s\" />Missing target: %s; \n" var var)))
-  (lyskom-docgen-princ "</para></section>\n"))
+  (lyskom-docgen-princ "</para></chapter>
+</book>\n")
+  (lyskom-docgen-close-file))
 
 
 
@@ -260,8 +493,7 @@
                 (commandp x)
                 (or (string-match "^kom-" (symbol-name x))
                     (string-match "^lyskom-" (symbol-name x))))
-       (lyskom-princ
-        (format "<!ENTITY fn:%s \"<function>%s</function>\">\n" x x)))))
+       (lyskom-princ (format "<!ENTITY fn:%s \"<function>%s</function>\">\n" x x)))))
   (lyskom-docgen-close-file))
 
 (defun lyskom-docgen-generate-var-ent ()
@@ -272,13 +504,52 @@
      (when (and (boundp x)
                 (or (string-match "^kom-" (symbol-name x))
                     (string-match "^lyskom-" (symbol-name x)))
-                (string-match "^\\*\\*"
-                              (or (documentation-property x 'variable-documentation) "")))
-       (lyskom-princ
-        (format "<!ENTITY var:%s \"<varname>%s</varname>\">\n" x x)))))
+                (lyskom-docgen-variable-doc x))
+       (lyskom-princ (format "<!ENTITY var:%s \"<varname>%s</varname>\">\n" x x)))))
   (lyskom-docgen-close-file))
 
 
+(defun lyskom-docgen-generate-editcmd-ent ()
+  (message "Generating editcmd.ent")
+  (lyskom-docgen-open-file "editcmd.ent")
+  (lyskom-docgen-princ "\
+<variablelist>
+  <title>Edit mode commands</title>