X-Git-Url: https://code.consxy.com/gitweb/gitweb.cgi?p=clnl;a=blobdiff_plain;f=src%2Fmain%2Flex.lisp;h=0adeb7d2b4323faf794ce688f53ce08ab142fa22;hp=b0ef2f1d1a591c8b6940093e09c83e52120fcdc5;hb=89cb3482de1fefc2d8e5f67e1e04a0790e8cd243;hpb=471de83db1aee70065808cbc061867e3320bf4b7

diff --git a/src/main/lex.lisp b/src/main/lex.lisp
index b0ef2f1..0adeb7d 100644
--- a/src/main/lex.lisp
+++ b/src/main/lex.lisp
@@ -32,6 +32,20 @@
      *lexes*))))
 
 (defun lex (text)
+ "LEX TEXT => AST
+
+ARGUMENTS AND VALUES:
+
+  TEXT: Some NetLogo code
+  AST: An ambigious AST that can later be parsed
+
+DESCRIPTION:
+
+  LEX lexes NetLogo code.
+
+  LEX checks for some things, in as much as it can without knowing anything
+  about some of the backgrounds of NetLogo.  However, it does the first pass
+  with as much as it can."
  (if (string= "" text)
   (let
    ((lex (find-if (lambda (f) (funcall f *state* :eof)) *lexes* :from-end t :key #'car)))
@@ -50,12 +64,11 @@
 
 ; This part is the actual netlogo spec
 
-(defvar *string-text* "(\\\"|\\r|\\n|\\t|\\\\|\\[^\"]|[^\r\n\"\\])*")
+(defvar *string-text* "(\\\\\"|\\\\n|\\\\r|\\\\t|\\\\|[^\\r\\n\\\"])")
 (defvar *nonnewline_white_space_char* "[ \\t\\b\\012]")
 (defvar *letter* "\\w")
 (defvar *digit* "\\d")
-;(defparameter *identifier-char* "[\\w\\d_\\.?=\*!<>:#\+/%\$\^\'&-]")
-(defvar *identifier-char* "[\\w\\d-.]")
+(defvar *identifier-char* "[\\w\\d_.?=\*!<>:#\+/%$\^'&-]")
 
 ;(defvar *extension-literal-depth* 0)
 ;(defstruct extension-literal text)
@@ -82,18 +95,27 @@
 (deflex :initial "[,\\{\\}\\[\\]\\(\\)]" #'as-symbol)
 (deflex :initial *nonnewline_white_space_char* (constantly nil))
 (deflex :initial "\\n|\\r" (constantly nil))
-;(deflex :initial ";.*[\n\r]?" nil)
-(deflex :initial (format nil "-?\.?[0-9]~A*" *identifier-char*)
+(deflex :initial ";.*[\n\r]?" (constantly nil))
+(deflex :initial (format nil "-?\\.?[0-9]~A*" *identifier-char*)
  (lambda (text)
   (let
    ((num?
      (let
       ((*readtable* (copy-readtable nil))
        (*read-eval* nil))
-      (read-from-string text))))
+      (read-from-string (format nil "~Ad0" text)))))
    (if (numberp num?) num? (error "Invalid number")))))
 
 (deflex :initial (format nil "~A*" *identifier-char*) #'as-symbol)
-;(deflex :initial (format nil "\"~A*\"" *string-text*))
-;(deflex :initial (format nil "\"~A*" *string-text*) (lambda (text) (error "Closing double quote is missing")))
+(deflex :initial (format nil "\"~A*\"" *string-text*)
+ ; While this shouldn't let harmful strings in,
+ ; one can't be too careful
+ (lambda (text)
+  (let
+   ((*readtable* (copy-readtable nil))
+    (*read-eval* nil))
+   (read-from-string text))))
+
+(deflex :initial (format nil "\"~A*" *string-text*)
+ (lambda (text) (declare (ignore text)) (error "Closing double quote is missing")))
 ;(deflex :initial "." (lambda (text) (error "Non standard character is not allowed")))