don't unescape quotes in string literals and identifiers

tools/Language/SQL/SimpleSQL/LexerTests.lhs

@@ -9,6 +9,10 @@ Test for the lexer
 > --import Debug.Trace
 > import Data.Char (isAlpha)
 
+> lexerTests :: TestItem
+> lexerTests = Group "lexerTests" $
+>     [Group "lexer token tests" [ansiLexerTests]]
+
 > ansiLexerTable :: [(String,[Token])]
 > ansiLexerTable =
 >     -- single char symbols
@@ -26,11 +30,13 @@ Test for the lexer
 >            ++ map (\i -> (':':i, [HostParam i])) idens
 >        )
 >     -- quoted identifiers with embedded double quotes
->     ++ [("\"normal \"\" iden\"", [Identifier (Just ("\"","\"")) "normal \" iden"])]
+>     -- the lexer doesn't unescape the quotes
+>     ++ [("\"normal \"\" iden\"", [Identifier (Just ("\"","\"")) "normal \"\" iden"])]
 >     -- strings
+>     -- the lexer doesn't apply escapes at all
 >     ++ [("'string'", [SqlString "'" "'" "string"])
->        ,("'normal '' quote'", [SqlString "'" "'" "normal ' quote"])
->        ,("'normalendquote '''", [SqlString "'" "'" "normalendquote '"])]
+>        ,("'normal '' quote'", [SqlString "'" "'" "normal '' quote"])
+>        ,("'normalendquote '''", [SqlString "'" "'" "normalendquote ''"])]
 >     -- csstrings
 >     ++ map (\c -> (c ++ "'test'", [SqlString (c ++ "'") "'" "test"]))
 >        ["n", "N","b", "B","x", "X", "u&"]
@@ -55,11 +61,6 @@ Test for the lexer
 >        ,"/* this *is/ a comment */"
 >        ]
 
-> lexerTests :: TestItem
-> lexerTests = Group "lexerTests" $
->     [Group "lexer token tests" [ansiLexerTests]]
-
-
 > ansiLexerTests :: TestItem
 > ansiLexerTests = Group "ansiLexerTests" $
 >     [Group "ansi lexer token tests" $ [LexerTest ansi2011 s t |  (s,t) <- ansiLexerTable]
@@ -152,3 +153,68 @@ number number (todo: double check more carefully)
 >    listPred _ [_] = False
 >    listPred (p,p1) (t:t1:ts) | p t && p1 t1 = True
 >                              | otherwise = listPred (p,p1) (t1:ts)
+
+todo: lexing tests
+do quickcheck testing:
+can try to generate valid tokens then check they parse
+
+same as above: can also try to pair tokens, create an accurate
+  function to say which ones can appear adjacent, and test
+
+I think this plus the explicit lists of tokens like above which do
+basic sanity + explicit edge casts will provide a high level of
+assurance.
+
+
+
+> postgresLexerTable :: [(String,[Token])]
+> postgresLexerTable =
+>     -- single char symbols
+>     map (\s -> ([s],[Symbol [s]])) "+-^*/%~&|?<>[]=,;()"
+>     -- multi char symbols
+>     ++ map (\s -> (s,[Symbol s])) [">=","<=","!=","<>","||"]
+>     -- symbols to add: :, ::, .. :=
+>     -- plus generic symbols
+>     ++ (let idens = ["a", "_a", "test", "table", "Stuff", "STUFF"]
+>         -- simple identifiers
+>         in map (\i -> (i, [Identifier Nothing i])) idens
+>            ++ map (\i -> ("\"" ++ i ++ "\"", [Identifier (Just ("\"","\"")) i])) idens
+>            -- todo: in order to make lex . pretty id, need to
+>            -- preserve the case of the u
+>            ++ map (\i -> ("u&\"" ++ i ++ "\"", [Identifier (Just ("u&\"","\"")) i])) idens
+>            -- host param
+>            ++ map (\i -> (':':i, [HostParam i])) idens
+>        )
+>     -- positional var
+>     -- quoted identifiers with embedded double quotes
+>     ++ [("\"normal \"\" iden\"", [Identifier (Just ("\"","\"")) "normal \" iden"])]
+>     -- strings
+>     ++ [("'string'", [SqlString "'" "'" "string"])
+>        ,("'normal '' quote'", [SqlString "'" "'" "normal '' quote"])
+>        ,("'normalendquote '''", [SqlString "'" "'" "normalendquote '"])
+>        ,("e'this '' quote''", [SqlString "e'" "'" "this '' quote '"])
+>        ,("e'this \' quote''", [SqlString "e'" "'" "this \' quote '"])
+>        ]
+>     -- csstrings
+>     ++ map (\c -> (c ++ "'test'", [SqlString (c ++ "'") "'" "test"]))
+>        ["n", "N","b", "B","x", "X", "u&", "e", "E"]
+>     -- numbers
+>     ++ [("10", [SqlNumber "10"])
+>        ,(".1", [SqlNumber ".1"])
+>        ,("5e3", [SqlNumber "5e3"])
+>        ,("5e+3", [SqlNumber "5e+3"])
+>        ,("5e-3", [SqlNumber "5e-3"])
+>        ,("10.2", [SqlNumber "10.2"])
+>        ,("10.2e7", [SqlNumber "10.2e7"])]
+>     -- whitespace
+>     ++ concat [[([a],[Whitespace [a]])
+>                ,([a,b], [Whitespace [a,b]])]
+>               | a <- " \n\t", b <- " \n\t"]
+>     -- line comment
+>     ++ map (\c -> (c, [LineComment c]))
+>        ["--", "-- ", "-- this is a comment", "-- line com\n"]
+>     -- block comment
+>     ++ map (\c -> (c, [BlockComment c]))
+>        ["/**/", "/* */","/* this is a comment */"
+>        ,"/* this *is/ a comment */"
+>        ]