Support shell and autoconf highlighting

Julien LepillerMon Jun 28 01:52:43+0200 2021

9fc3447

Support shell and autoconf highlighting

Makefile.am

1111 
    gitile/handler.scm \
1212 
    gitile/pages.scm \
1313 
    gitile/repo.scm \
14-
	gitile/highlight/utils.scm \
15-
	gitile/highlight/gitignore.scm
14+
	gitile/highlight/gitignore.scm \
15+
	gitile/highlight/shell.scm \
16+
	gitile/highlight/utils.scm
1617 
1718 
clean-go:
1819 
	find . -name '*.go' -delete
assets/css/highlight.css
2121 
  color: var(--string);
2222 
}
2323 
24-
.syntax-keyword {
24+
.syntax-keyword, .syntax-builtin {
2525 
  color: var(--keyword2);
2626 
  font-weight: bold;
2727 
}
2828 
29+
.syntax-variable {
30+
  color: var(--keyword);
31+
}
32+
2933 
/* language-specific */
3034 
3135 
/***
gitile/code.scm
2121 
  #:use-module (syntax-highlight)
2222 
  #:use-module (syntax-highlight scheme)
2323 
  #:use-module (gitile highlight gitignore)
24+
  #:use-module (gitile highlight shell)
2425 
  #:use-module ((gitile highlight utils) #:prefix gitile:)
2526 
  #:export (display-code
2627 
            display-formatted-code))
…
3031 
         (language (match extension
3132 
                     ("scm" "scheme")
3233 
                     ("gitignore" "gitignore")
33-
                     (_ "unknown"))))
34+
                     ("sh" "shell")
35+
                     ("bash" "shell")
36+
                     ("m4" "m4")
37+
                     ("ac" "ac")
38+
                     (_ (match (basename path)
39+
                         ("bootstrap" "shell")
40+
                         (_ "unknown"))))))
3441 
    `(table (@ (class ,(string-append "file-content language-" language)))
3542 
       ,@(split-tr (gitile:highlights->sxml (highlight-code content language))))))
3643 
…
3946 
4047 
(define (highlight-code content language)
4148 
  (match language
42-
    ("scheme" (highlight lex-scheme content))
49+
    ("scheme" (highlight (lambda (tokens cursor) (lex-scheme tokens cursor)) content))
50+
    ("shell" (highlight lex-shell content))
4351 
    ("gitignore" (highlight lex-gitignore content))
52+
    ("m4" (highlight lex-m4 content))
53+
    ("ac" (highlight lex-autoconf content))
4454 
    (_ (list content))))
4555 
4656 
(define (split-lines content)
gitile/highlight/shell.scm unknown status 1
1+
;;;; Copyright (C) 2021 Julien Lepiller <julien@lepiller.eu>
2+
;;;;
3+
;;;; SPDX-License-Identifier: AGPL-3.0-or-later
4+
;;;;
5+
;;;; This program is free software: you can redistribute it and/or modify
6+
;;;; it under the terms of the GNU Affero General Public License as published by
7+
;;;; the Free Software Foundation, either version 3 of the License, or
8+
;;;; (at your option) any later version.
9+
;;;;
10+
;;;; This program is distributed in the hope that it will be useful,
11+
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13+
;;;; GNU Affero General Public License for more details.
14+
;;;;
15+
;;;; You should have received a copy of the GNU Affero General Public License
16+
;;;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
17+
;;;;
18+
19+
(define-module (gitile highlight shell)
20+
  #:use-module (ice-9 match)
21+
  #:use-module (srfi srfi-1)
22+
  #:use-module (srfi srfi-11)
23+
  #:use-module (srfi srfi-26)
24+
  #:use-module (syntax-highlight lexers)
25+
  #:use-module (gitile highlight utils)
26+
  #:export (%shell-builtins
27+
            %shell-keywords
28+
            make-shell-lexer
29+
            lex-shell
30+
            make-m4-lexer
31+
            lex-m4
32+
            lex-autoconf))
33+
34+
(define %variable-char-set char-set:letter+digit)
35+
36+
(define %shell-builtins
37+
  '("alias" "bg" "bind" "break" "builtin" "caller" "cd" "command" "compgen"
38+
    "complete" "declare" "dirs" "disown" "echo" "enable" "eval" "exec" "exit"
39+
    "export" "false" "fc" "fg" "getopts" "hash" "help" "history" "jobs" "kill"
40+
    "let" "local" "logout" "popd" "printf" "pushd" "pwd" "read" "readonly" "set"
41+
    "shift" "shopt" "source" "suspend" "test" "time" "times" "trap" "true"
42+
    "type" "typeset" "ulimit" "umask" "unalias" "unset" "wait"))
43+
(define %shell-keywords
44+
  '("if" "fi" "else" "while" "in" "do" "done" "for" "then" "return" "function"
45+
    "case" "select" "continue" "until" "esac" "elif"))
46+
47+
(define (make-shell-lexer builtins keywords)
48+
  (define lex-shell-math
49+
    (lex-any
50+
      (lex-tag 'operator
51+
               (apply lex-any (map lex-string
52+
                                   '("-" "+" "*" "/" "%" "^" "|" "&" "||" "**"))))
53+
      (lex-tag 'number (lex-all
54+
                         (lex-char-set char-set:digit)
55+
                         (lex-string "#")
56+
                         (lex-char-set char-set:digit)))
57+
      (lex-tag 'number (lex-char-set char-set:digit))
58+
      shell-root-lexer))
59+
  (define (shell-math-lexer tokens cursor)
60+
    (lex-shell-math tokens cursor))
61+
62+
  (define lex-shell-curly
63+
    (lex-any
64+
      (lex-tag 'keyword (lex-string ":-"))
65+
      (lex-tag 'variable (lex-char-set %variable-char-set))
66+
      (lex-char-set (char-set-complement (char-set #\} #\: #\" #\' #\` #\" #\\)))
67+
      (lex-string ":")
68+
      shell-root-lexer))
69+
  (define (shell-curly-lexer tokens cursor)
70+
    (lex-shell-curly tokens cursor))
71+
  
72+
  (define lex-shell-interp
73+
    (lex-any
74+
      (lex-all (lex-tag 'keyword (lex-string "$(("))
75+
               (lex-consume-until
76+
                 (lex-tag 'keyword (lex-string "))"))
77+
                 shell-math-lexer))
78+
      (lex-all (lex-tag 'keyword (lex-string "$("))
79+
               (lex-consume-until
80+
                 (lex-tag 'keyword (lex-string ")"))
81+
                 shell-root-lexer))
82+
      (lex-all (lex-tag 'keyword (lex-string "${"))
83+
               (lex-maybe (lex-tag 'keyword (lex-string "#")))
84+
               (lex-consume-until
85+
                 (lex-tag 'keyword (lex-string "}"))
86+
                 shell-curly-lexer))
87+
      (lex-tag 'variable (lex-all (lex-string "$") (lex-char-set %variable-char-set)))
88+
      (lex-tag 'builtin
89+
               (apply lex-any
90+
                      (map lex-string '("$#" "$$" "$?" "$!" "$_" "$*" "$@"
91+
                                        "$-" "$0" "$1" "$2" "$3" "$4" "$5"
92+
                                        "$6" "$7" "$8" "$9"))))
93+
      (lex-string "$")))
94+
  (define (shell-interp-lexer tokens cursor)
95+
    (lex-shell-interp tokens cursor))
96+
  
97+
  (define lex-shell-string
98+
    (lex-any
99+
      (apply lex-any (map lex-string '("\\" "\\." "\\$")))
100+
      (lex-all (lex-string "\\") (lex-char-set (char-set #\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7)))
101+
      (lex-char-set (char-set-complement (char-set #\" #\\ #\$)))
102+
      shell-interp-lexer))
103+
  (define (shell-string-lexer tokens cursor)
104+
    (lex-shell-string tokens cursor))
105+
106+
  (define (shell-heredoc-lexer tokens cursor)
107+
    (define (get-delimiter str start)
108+
      (let ((len (string-length str)))
109+
        (let loop ((index start))
110+
          (cond
111+
            ((>= index len) (substring str start len))
112+
            ((char-set-contains? (char-set-union %variable-char-set (char-set #\$))
113+
                                 (string-ref str index))
114+
             (loop (1+ index)))
115+
            (else (substring str start index))))))
116+
117+
    (let*-values (((result remainder)
118+
                   ((lex-all (lex-tag 'operator (lex-string "<<"))
119+
                             (lex-maybe (lex-tag 'operator (lex-string "-")))
120+
                             (lex-maybe lex-whitespace)
121+
                             (lex-maybe (lex-string "\\")))
122+
                    tokens cursor)))
123+
      (if result
124+
          (let* ((delimiter (get-delimiter (cursor-text remainder)
125+
                                           (cursor-position remainder)))
126+
                 (result (token-add tokens delimiter))
127+
                 (remainder (move-cursor-by cursor (string-length delimiter))))
128+
            (if (equal? delimiter "")
129+
                (fail)
130+
                ((lex-consume-until
131+
                   (lex-string delimiter)
132+
                   shell-string-lexer
133+
                   #:tag 'string) result remainder)))
134+
          (values result remainder))))
135+
136+
  (define lex-root
137+
    (lex-any
138+
      (lex-char-set char-set:whitespace)
139+
      (lex-tag 'builtin (lex-filter
140+
                          (lambda (str)
141+
                            (any (cut string=? <> str) builtins))
142+
                          (lex-char-set %variable-char-set)))
143+
      (lex-tag 'keyword (lex-filter
144+
                          (lambda (str)
145+
                            (any (cut string=? <> str) keywords))
146+
                          (lex-char-set %variable-char-set)))
147+
      (lex-tag 'comment (lex-delimited "#" #:until "\n"))
148+
      (lex-all (lex-tag 'variable (lex-char-set %variable-char-set))
149+
               (lex-char-set char-set:whitespace)
150+
               (lex-maybe (lex-string "+"))
151+
               (lex-string "="))
152+
      (lex-tag 'keyword (lex-string "`"))
153+
      (lex-tag 'string (lex-delimited "'" #:escape #f))
154+
      (lex-tag 'string (lex-delimited "$'" #:until "'" #:escape #f))
155+
      (lex-all (lex-tag 'string (lex-any (lex-string "\"") (lex-string "$\"")))
156+
               (lex-consume-until
157+
                 (lex-string "\"")
158+
                 shell-string-lexer
159+
                 #:tag 'string))
160+
      (lex-tag 'operator
161+
               (apply lex-any (map lex-string '("[" "]" "{" "}" "(" ")" "="))))
162+
      (lex-tag 'operator (lex-string "<<<"))
163+
      shell-heredoc-lexer
164+
      (lex-tag 'operator (lex-string "&&"))
165+
      (lex-tag 'operator (lex-string "||"))
166+
      (lex-string ";")
167+
      (lex-string "&")
168+
      (lex-string "|")
169+
      (lex-char-set char-set:digit)
170+
      (lex-string "\\\n")
171+
      (lex-char-set
172+
        (char-set-complement
173+
          (char-set #\= #\[ #\] #\{ #\} #\( #\) #\$ #\" #\' #\` #\\ #\< #\& #\| #\; #\newline)))
174+
      (lex-string "<")
175+
      shell-interp-lexer))
176+
  (define (shell-root-lexer tokens cursor)
177+
    (lex-root tokens cursor))
178+
179+
  shell-root-lexer)
180+
181+
(define (lex-shell tokens cursor)
182+
  ((lex-consume
183+
    (make-shell-lexer %shell-builtins %shell-keywords))
184+
   tokens cursor))
185+
186+
(define %macro-char-set (char-set-union char-set:letter+digit (char-set #\_)))
187+
188+
(define %m4-builtin-macros
189+
  '("define" "undefine" "defn" "pushdef" "popdef" "ifdef" "shift"
190+
    "changequote" "changecom" "divert" "undivert" "ifelse"
191+
    "incr" "decr" "eval" "len" "dlen" "index" "index" "substr" "translit"
192+
    "include" "sinclude" "syscmd" "maketemp" "m4exit" "m4wrap" "errprint"
193+
    "dumpdef" "traceon" "traceoff" "divnum" "sysval"))
194+
195+
;; @defmacro in doc/autoconf.texi (autoconf sources)
196+
(define %autoconf-builtin-macros
197+
  '("AC_INIT" "AC_PACKAGE_NAME" "PACKAGE_NAME" "AC_PACKAGE_TARNAME"
198+
    "PACKAGE_TARNAME" "AC_PACKAGE_VERSION" "PACKAGE_VERSION"
199+
    "AC_PACKAGE_STRING" "PACKAGE_STRING" "AC_PACKAGE_BUGREPORT"
200+
    "PACKAGE_BUGREPORT" "AC_PACKAGE_URL" "PACKAGE_URL"
201+
    "AC_PREREQ" "AC_AUTOCONF_VERSION" "AC_COPYRIGHT" "AC_REVISION" "AC_CONFIG_SRCDIR"
202+
    "AC_CONFIG_MACRO_DIRS" "AC_CONFIG_MACRO_DIR" "AC_CONFIG_AUX_DIR" "AC_REQUIRE_AUX_FILE"
203+
    "AC_OUTPUT" "AC_PROG_MAKE_SET" "AC_CONFIG_FILES" "AC_CONFIG_HEADERS" "AH_HEADER"
204+
    "AH_TEMPLATE" "AH_VERBATIM" "AH_TOP" "AH_BOTTOM" "AC_CONFIG_COMMANDS"
205+
    "AC_CONFIG_COMMANDS_PRE" "AC_CONFIG_COMMANDS_POST" "AC_CONFIG_LINKS"
206+
    "AC_CONFIG_SUBDIRS" "AC_PREFIX_DEFAULT" "AC_PREFIX_PROGRAM" "AC_INCLUDES_DEFAULT"
207+
    "AC_CHECK_INCLUDES_DEFAULT" "AC_PROG_AWK" "AC_PROG_GREP" "AC_PROG_EGREP"
208+
    "AC_PROG_FGREP" "AC_PROG_INSTALL" "AC_PROG_MKDIR_P" "AC_PROG_LEX" "AC_PROG_LN_S"
209+
    "AC_PROG_RANLIB" "AC_PROG_SED" "AC_PROG_YACC" "AC_CHECK_PROG" "AC_CHECK_PROGS"
210+
    "AC_CHECK_TARGET_TOOL" "AC_CHECK_TOOL" "AC_CHECK_TARGET_TOOLS" "AC_CHECK_TOOLS"
211+
    "AC_PATH_PROG" "AC_PATH_PROGS" "AC_PATH_PROGS_FEATURE_CHECK" "AC_PATH_TARGET_TOOL"
212+
    "AC_PATH_TOOL" "AC_CHECK_FILE" "AC_CHECK_FILES" "AC_CHECK_LIB" "AC_SEARCH_LIBS"
213+
    "AC_FUNC_ALLOCA" "AC_FUNC_CHOWN" "AC_FUNC_CLOSEDIR_VOID" "AC_FUNC_ERROR_AT_LINE"
214+
    "AC_FUNC_FNMATCH" "AC_FUNC_FNMATCH_GNU" "AC_FUNC_FORK" "AC_FUNC_FSEEKO"
215+
    "AC_FUNC_GETGROUPS" "AC_FUNC_GETLOADAVG" "AC_FUNC_GETMNTENT" "AC_FUNC_GETPGRP"
216+
    "AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK" "AC_FUNC_MALLOC" "AC_FUNC_MBRTOWC"
217+
    "AC_FUNC_MEMCMP" "AC_FUNC_MKTIME" "AC_FUNC_MMAP" "AC_FUNC_OBSTACK"
218+
    "AC_FUNC_REALLOC" "AC_FUNC_SELECT_ARGTYPES" "AC_FUNC_SETPGRP" "AC_FUNC_STAT"
219+
    "AC_FUNC_LSTAT" "AC_FUNC_STRCOLL" "AC_FUNC_STRERROR_R" "AC_FUNC_STRFTIME"
220+
    "AC_FUNC_STRTOD" "AC_FUNC_STRTOLD" "AC_FUNC_STRNLEN" "AC_FUNC_UTIME_NULL"
221+
    "AC_FUNC_VPRINTF" "AC_REPLACE_FNMATCH" "AC_CHECK_FUNC" "AC_CHECK_FUNCS"
222+
    "AC_CHECK_FUNCS_ONCE" "AC_LIBOBJ" "AC_LIBSOURCE" "AC_LIBSOURCES"
223+
    "AC_CONFIG_LIBOBJ_DIR" "AC_REPLACE_FUNCS" "AC_CHECK_HEADER_STDBOOL"
224+
    "AC_HEADER_ASSERT" "AC_HEADER_DIRENT" "AC_HEADER_MAJOR" "AC_HEADER_RESOLV"
225+
    "AC_HEADER_STAT" "AC_HEADER_STDBOOL" "AC_HEADER_STDC" "AC_HEADER_SYS_WAIT"
226+
    "AC_HEADER_TIOCGWINSZ" "AC_CHECK_HEADER" "AC_CHECK_HEADERS" "AC_CHECK_HEADERS_ONCE"
227+
    "AC_CHECK_DECL" "AC_CHECK_DECLS" "AC_CHECK_DECLS_ONCE" "AC_STRUCT_DIRENT_D_INO"
228+
    "AC_STRUCT_DIRENT_D_TYPE" "AC_STRUCT_ST_BLOCKS" "AC_STRUCT_TM" "AC_STRUCT_TIMEZONE"
229+
    "AC_CHECK_MEMBER" "AC_CHECK_MEMBERS" "AC_TYPE_GETGROUPS" "AC_TYPE_INT8_T"
230+
    "AC_TYPE_INT16_T" "AC_TYPE_INT32_T" "AC_TYPE_INT64_T" "AC_TYPE_INTMAX_T"
231+
    "AC_TYPE_INTPTR_T" "AC_TYPE_LONG_DOUBLE" "AC_TYPE_LONG_DOUBLE_WIDER"
232+
    "AC_TYPE_LONG_LONG_INT" "AC_TYPE_MBSTATE_T" "AC_TYPE_MODE_T" "AC_TYPE_OFF_T"
233+
    "AC_TYPE_PID_T" "AC_TYPE_SIZE_T" "AC_TYPE_SSIZE_T" "AC_TYPE_UID_T"
234+
    "AC_TYPE_UINT8_T" "AC_TYPE_UINT16_T" "AC_TYPE_UINT32_T" "AC_TYPE_UINT64_T"
235+
    "AC_TYPE_UINTMAX_T" "AC_TYPE_UINTPTR_T" "AC_TYPE_UNSIGNED_LONG_LONG_INT"
236+
    "AC_CHECK_TYPE" "AC_CHECK_TYPES" "AC_CHECK_SIZEOF" "AC_CHECK_ALIGNOF"
237+
    "AC_COMPUTE_INT" "AC_LANG_WERROR" "AC_OPENMP" "AC_PROG_CC" "AC_PROG_CC_C_O"
238+
    "AC_PROG_CPP" "AC_PROG_CPP_WERROR" "AC_C_BACKSLASH_A" "AC_C_BIGENDIAN"
239+
    "AC_C_CONST" "AC_C__GENERIC" "AC_C_RESTRICT" "AC_C_VOLATILE" "AC_C_INLINE"
240+
    "AC_C_CHAR_UNSIGNED" "AC_C_STRINGIZE" "AC_C_FLEXIBLE_ARRAY_MEMBER"
241+
    "AC_C_VARARRAYS" "AC_C_TYPEOF" "AC_C_PROTOTYPES" "AC_PROG_GCC_TRADITIONAL"
242+
    "AC_PROG_CXX" "AC_PROG_CXXCPP" "AC_PROG_CXX_C_O" "AC_PROG_OBJC" "AC_PROG_OBJCPP"
243+
    "AC_PROG_OBJCXX" "AC_PROG_OBJCXXCPP" "AC_ERLANG_PATH_ERLC" "AC_ERLANG_NEED_ERLC"
244+
    "AC_ERLANG_PATH_ERL" "AC_ERLANG_NEED_ERL" "AC_PROG_F77" "AC_PROG_FC"
245+
    "AC_PROG_F77_C_O" "AC_PROG_FC_C_O" "AC_F77_LIBRARY_LDFLAGS" "AC_FC_LIBRARY_LDFLAGS"
246+
    "AC_F77_DUMMY_MAIN" "AC_FC_DUMMY_MAIN" "AC_F77_MAIN" "AC_FC_MAIN" "AC_F77_WRAPPERS"
247+
    "AC_FC_WRAPPERS" "AC_F77_FUNC" "AC_FC_FUNC" "AC_FC_SRCEXT" "AC_FC_PP_SRCEXT"
248+
    "AC_FC_PP_DEFINE" "AC_FC_FREEFORM" "AC_FC_FIXEDFORM" "AC_FC_LINE_LENGTH"
249+
    "AC_FC_CHECK_BOUNDS" "AC_F77_IMPLICIT_NONE" "AC_FC_IMPLICIT_NONE"
250+
    "AC_FC_MODULE_EXTENSION" "AC_FC_MODULE_FLAG" "AC_FC_MODULE_OUTPUT_FLAG"
251+
    "AC_PROG_GO" "AC_PATH_X" "AC_PATH_XTRA" "AC_SYS_INTERPRETER" "AC_SYS_LARGEFILE"
252+
    "AC_SYS_LONG_FILE_NAMES" "AC_SYS_POSIX_TERMIOS" "AC_USE_SYSTEM_EXTENSIONS"
253+
    "AC_ERLANG_SUBST_ERTS_VER" "AC_ERLANG_SUBST_ROOT_DIR" "AC_ERLANG_SUBST_LIB_DIR"
254+
    "AC_ERLANG_CHECK_LIB" "AC_ERLANG_SUBST_INSTALL_LIB_DIR"
255+
    "AC_ERLANG_SUBST_INSTALL_LIB_SUBDIR" "AC_LANG" "AC_LANG_PUSH" "AC_LANG_POP"
256+
    "AC_LANG_ASSERT" "AC_REQUIRE_CPP" "AC_LANG_CONFTEST" "AC_LANG_DEFINES_PROVIDED"
257+
    "AC_LANG_SOURCE" "AC_LANG_PROGRAM" "AC_LANG_CALL" "AC_LANG_FUNC_LINK_TRY"
258+
    "AC_PREPROC_IFELSE" "AC_EGREP_HEADER" "AC_EGREP_CPP" "AC_COMPILE_IFELSE"
259+
    "AC_LINK_IFELSE" "AC_RUN_IFELSE" "AC_DEFINE" "AC_DEFINE" "AC_DEFINE_UNQUOTED"
260+
    "AC_DEFINE_UNQUOTED" "AC_SUBST" "AC_SUBST_FILE" "AC_ARG_VAR" "AC_CACHE_VAL"
261+
    "AC_CACHE_CHECK" "AC_CACHE_LOAD" "AC_CACHE_SAVE" "AC_MSG_CHECKING"
262+
    "AC_MSG_RESULT" "AC_MSG_NOTICE" "AC_MSG_ERROR" "AC_MSG_FAILURE" "AC_MSG_WARN"
263+
    "__file__" "__line__" "__oline__" "dnl" "m4_bpatsubst" "m4_bregexp"
264+
    "m4_copy" "m4_copy_force" "m4_rename" "m4_rename_force" "m4_defn"
265+
    "m4_divert" "m4_dumpdef" "m4_dumpdefs" "m4_esyscmd_s" "m4_exit" "m4_if"
266+
    "m4_include" "m4_sinclude" "m4_mkstemp" "m4_maketemp" "m4_popdef"
267+
    "m4_undefine" "m4_undivert" "m4_wrap" "m4_wrap_lifo" "m4_assert" "m4_errprintn"
268+
    "m4_fatal" "m4_location" "m4_warn" "m4_cleardivert" "m4_divert_once"
269+
    "m4_divert_pop" "m4_divert_push" "m4_divert_text" "m4_init" "m4_bmatch"
270+
    "m4_bpatsubsts" "m4_case" "m4_cond" "m4_default" "m4_default_quoted"
271+
    "m4_default_nblank" "m4_default_nblank_quoted" "m4_define_default"
272+
    "m4_ifblank" "m4_ifnblank" "m4_ifndef" "m4_ifset" "m4_ifval" "m4_ifvaln"
273+
    "m4_n" "m4_argn" "m4_car" "m4_cdr" "m4_for" "m4_foreach" "m4_foreach_w"
274+
    "m4_map" "m4_mapall" "m4_map_sep" "m4_mapall_sep" "m4_map_args"
275+
    "m4_map_args_pair" "m4_map_args_sep" "m4_map_args_w" "m4_shiftn" "m4_shift2"
276+
    "m4_shift3" "m4_stack_foreach" "m4_stack_foreach_lifo" "m4_stack_foreach_sep"
277+
    "m4_stack_foreach_sep_lifo" "m4_apply" "m4_count" "m4_curry" "m4_do"
278+
    "m4_dquote" "m4_dquote_elt" "m4_echo" "m4_expand" "m4_ignore" "m4_make_list"
279+
    "m4_quote" "m4_reverse" "m4_unquote" "m4_append" "m4_append_uniq"
280+
    "m4_append_uniq_w" "m4_chomp" "m4_chomp_all" "m4_combine" "m4_escape"
281+
    "m4_flatten" "m4_join" "m4_joinall" "m4_newline" "m4_normalize" "m4_re_escape"
282+
    "m4_split" "m4_strip" "m4_text_box" "m4_text_wrap" "m4_tolower" "m4_toupper"
283+
    "m4_cmp" "m4_list_cmp" "m4_max" "m4_min" "m4_sign" "m4_version_compare"
284+
    "m4_version_prereq" "m4_set_add" "m4_set_add_all" "m4_set_contains"
285+
    "m4_set_contents" "m4_set_dump" "m4_set_delete" "m4_set_difference"
286+
    "m4_set_intersection" "m4_set_union" "m4_set_empty" "m4_set_foreach"
287+
    "m4_set_list" "m4_set_listc" "m4_set_map" "m4_set_map_sep" "m4_set_remove"
288+
    "m4_set_size" "m4_pattern_forbid" "m4_pattern_allow" "AS_BASENAME"
289+
    "AS_BOX" "AS_CASE" "AS_DIRNAME" "AS_ECHO" "AS_ECHO_N" "AS_ESCAPE"
290+
    "AS_EXECUTABLE_P" "AS_EXIT" "AS_IF" "AS_MKDIR_P" "AS_SET_STATUS" "AS_TR_CPP"
291+
    "AS_TR_SH" "AS_SET_CATFILE" "AS_UNSET" "AS_VERSION_COMPARE" "AS_LITERAL_IF"
292+
    "AS_LITERAL_WORD_IF" "AS_VAR_APPEND" "AS_VAR_ARITH" "AS_VAR_COPY" "AS_VAR_IF"
293+
    "AS_VAR_PUSHDEF" "AS_VAR_POPDEF" "AS_VAR_SET" "AS_VAR_SET_IF" "AS_VAR_TEST_SET"
294+
    "AS_BOURNE_COMPATIBLE" "AS_INIT" "AS_INIT_GENERATED" "AS_LINENO_PREPARE"
295+
    "AS_ME_PREPARE" "AS_TMPDIR" "AS_SHELL_SANITIZE" "AS_MESSAGE_FD" "AS_MESSAGE_LOG_FD"
296+
    "AS_ORIGINAL_STDIN_FD" "AC_DEFUN" "AC_REQUIRE" "AC_BEFORE" "AC_DEFUN_ONCE"
297+
    "AU_DEFUN" "AU_ALIAS" "AC_CANONICAL_BUILD" "AC_CANONICAL_HOST" "AC_CANONICAL_TARGET"
298+
    "AC_PRESERVE_HELP_ORDER" "AC_ARG_WITH" "AC_ARG_ENABLE" "AS_HELP_STRING"
299+
    "AC_DISABLE_OPTION_CHECKING" "AC_ARG_PROGRAM" "AC_AIX" "AC_ALLOCA" "AC_ARG_ARRAY"
300+
    "AC_C_CROSS" "AC_C_LONG_DOUBLE" "AC_CANONICAL_SYSTEM" "AC_CHAR_UNSIGNED"
301+
    "AC_CHECK_TYPE" "AC_CHECKING" "AC_COMPILE_CHECK" "AC_CONST" "AC_CROSS_CHECK"
302+
    "AC_CYGWIN" "AC_DECL_SYS_SIGLIST" "AC_DECL_YYTEXT" "AC_DIAGNOSE" "AC_DIR_HEADER"
303+
    "AC_DYNIX_SEQ" "AC_EXEEXT" "AC_EMXOS2" "AC_ENABLE" "AC_ERROR" "AC_FATAL"
304+
    "AC_FIND_X" "AC_FIND_XTRA" "AC_FOREACH" "AC_FUNC_CHECK" "AC_FUNC_SETVBUF_REVERSED"
305+
    "AC_FUNC_WAIT3" "AC_GCC_TRADITIONAL" "AC_GETGROUPS_T" "AC_GETLOADAVG" "AC_GNU_SOURCE"
306+
    "AC_HAVE_FUNCS" "AC_HAVE_HEADERS" "AC_HAVE_LIBRARY" "AC_HAVE_POUNDBANG"
307+
    "AC_HEADER_CHECK" "AC_HEADER_EGREP" "AC_HEADER_TIME" "AC_HELP_STRING"
308+
    "AC_INLINE" "AC_INT_16_BITS" "AC_IRIX_SUN" "AC_ISC_POSIX" "AC_LANG_C"
309+
    "AC_LANG_CPLUSPLUS" "AC_LANG_FORTRAN77" "AC_LANG_RESTORE" "AC_LANG_SAVE"
310+
    "AC_LINK_FILES" "AC_LN_S" "AC_LONG_64_BITS" "AC_LONG_DOUBLE" "AC_LONG_FILE_NAMES"
311+
    "AC_MAJOR_HEADER" "AC_MEMORY_H" "AC_MINGW32" "AC_MINIX" "AC_MINUS_C_MINUS_O"
312+
    "AC_MMAP" "AC_MODE_T" "AC_OBJEXT" "AC_OBSOLETE" "AC_OFF_T" "AC_OUTPUT"
313+
    "AC_OUTPUT_COMMANDS" "AC_PID_T" "AC_PREFIX" "AC_PROG_CC_C89" "AC_PROG_CC_C99"
314+
    "AC_PROG_CC_STDC" "AC_PROGRAMS_CHECK" "AC_PROGRAMS_PATH" "AC_PROGRAM_CHECK"
315+
    "AC_PROGRAM_EGREP" "AC_PROGRAM_PATH" "AC_REMOTE_TAPE" "AC_RESTARTABLE_SYSCALLS"
316+
    "AC_RETSIGTYPE" "AC_RSH" "AC_SCO_INTL" "AC_SETVBUF_REVERSED" "AC_SET_MAKE"
317+
    "AC_SIZEOF_TYPE" "AC_SIZE_T" "AC_STAT_MACROS_BROKEN" "AC_STDC_HEADERS"
318+
    "AC_STRCOLL" "AC_STRUCT_ST_BLKSIZE" "AC_STRUCT_ST_RDEV" "AC_ST_BLKSIZE"
319+
    "AC_ST_BLOCKS" "AC_ST_RDEV" "AC_SYS_RESTARTABLE_SYSCALLS" "AC_SYS_SIGLIST_DECLARED"
320+
    "AC_TEST_CPP" "AC_TEST_PROGRAM" "AC_TIMEZONE" "AC_TIME_WITH_SYS_TIME"
321+
    "AC_TRY_COMPILE" "AC_TRY_CPP" "AC_TRY_LINK" "AC_TRY_LINK_FUNC"
322+
    "AC_TRY_RUN" "AC_TYPE_SIGNAL" "AC_UID_T" "AC_UNISTD_H" "AC_USG" "AC_UTIME_NULL"
323+
    "AC_VALIDATE_CACHED_SYSTEM_TUPLE" "AC_VERBOSE" "AC_VFORK" "AC_VPRINTF"
324+
    "AC_WAIT3" "AC_WARN" "AC_WARNING" "AC_WITH" "AC_WORDS_BIGENDIAN" "AC_XENIX_DIR"
325+
    "AC_YYTEXT_POINTER" "AT_INIT" "AT_COPYRIGHT" "AT_ARG_OPTION" "AT_ARG_OPTION_ARG"
326+
    "AT_COLOR_TESTS" "AT_TESTED" "AT_PREPARE_TESTS" "AT_PREPARE_EACH_TEST"
327+
    "AT_TEST_HELPER_FN" "AT_BANNER" "AT_SETUP" "AT_KEYWORDS" "AT_CAPTURE_FILE"
328+
    "AT_FAIL_IF" "AT_SKIP_IF" "AT_XFAIL_IF" "AT_CLEANUP" "AT_DATA" "AT_DATA_UNQUOTED"
329+
    "AT_CHECK" "AT_CHECK_UNQUOTED" "AT_CHECK_EUNIT" "AC_CONFIG_TESTDIR"))
330+
331+
(define (make-m4-lexer base-lexer builtins quote-in quote-out)
332+
  (define lex-macro-argument
333+
    (lex-any
334+
      (lex-tag 'comment (lex-delimited "#" #:until "\n"))
335+
      (lex-tag 'comment (lex-delimited "dnl" #:until "\n"))
336+
      (lex-tag 'comment (lex-delimited "DNL" #:until "\n"))
337+
      (lex-string ",")
338+
      lex-whitespace
339+
      (lex-delimited quote-in #:until quote-out #:nested? #t)
340+
      (lex-char-set (char-set-complement (char-set #\,)))))
341+
342+
  (define lex-macro
343+
    (lex-all
344+
      (lex-any
345+
        (lex-tag 'builtin (lex-filter
346+
                            (lambda (str)
347+
                              (pk 'builtin? str)
348+
                              (any (cut string=? <> str) builtins))
349+
                            (lex-char-set %macro-char-set)))
350+
        (lex-tag 'variable (lex-char-set %macro-char-set)))
351+
      (lex-maybe (lex-all
352+
                   (lex-tag 'open (lex-string "("))
353+
                   (lex-consume-until
354+
                     (lex-tag 'close (lex-string ")"))
355+
                     lex-macro-argument)))))
356+
357+
  (lex-consume
358+
    (lex-any
359+
      (lex-tag 'comment (lex-delimited "#" #:until "\n"))
360+
      (lex-tag 'comment (lex-delimited "dnl" #:until "\n"))
361+
      (lex-tag 'comment (lex-delimited "DNL" #:until "\n"))
362+
      lex-macro
363+
      base-lexer)))
364+
365+
(define (lex-m4 tokens cursor)
366+
  ((make-m4-lexer lex-fail %m4-builtin-macros "`" "'")
367+
   tokens cursor))
368+
369+
(define (lex-autoconf tokens cursor)
370+
  ((make-m4-lexer (make-shell-lexer %shell-builtins %shell-keywords)
371+
                 (append %m4-builtin-macros %autoconf-builtin-macros)
372+
                 "[" "]")
373+
   tokens cursor))