* Bug #14313 fixed - Parser did not create a column separator after spaces and '...
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13
14 #include <stack>
15
16 #include "isatty.hxx"
17 #include "parse.hxx"
18 #include "parser_private.hxx"
19
20 #include "context.hxx"
21
22 extern "C"
23 {
24 #include "charEncoding.h"
25 #include "sci_malloc.h"
26 }
27
28 static std::stack<int> paren_levels;
29
30 static int comment_level = 0;
31 static int last_token = 0;
32 static int exit_status = PARSE_ERROR;
33 static int str_opener_column = 0;
34 static std::string current_file;
35 static std::string program_name;
36
37 static std::string pstBuffer;
38
39 extern void yyerror(std::string);
40
41 #define YY_USER_ACTION                          \
42     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
43 //yylloc.last_column += yyleng;
44
45 /* -*- Verbose Special Debug -*- */
46 //#define DEV
47 //#define TOKENDEV
48
49 #ifdef DEV
50 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
51 #else
52 #define DEBUG(x) /* Nothing */
53 #endif
54
55 %}
56
57 %option stack
58 %option noyywrap
59
60 %x SIMPLESTRING
61 %x DOUBLESTRING
62 %x REGIONCOMMENT
63 %x LINECOMMENT
64 %x LINEBREAK
65
66 %x MATRIX
67
68 %x SHELLMODE
69 %x BEGINID
70
71 spaces                  [ \t\v\f]+
72 integer                 [0-9]+
73 number                  [0-9]+[\.][0-9]*
74 little                  \.[0-9]+
75 bom             \xEF\xBB\xBF
76
77 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
78 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
79
80 hex             [0]x[0-9a-fA-F]+
81 oct             [0]o[0-7]+
82
83
84 utf2            ([\xC2-\xDF][\x80-\xBF])
85 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
86 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
87 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
88 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
89 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
90 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
91 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
92
93 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
94 utf4            ({utf41}|{utf42}|{utf43})
95
96 utf             ({utf2}|{utf3}|{utf4})
97 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
98
99
100 newline                 ("\r"|"\n"|"\r\n")
101 blankline               {spaces}+{newline}
102 emptyline       {newline}({spaces}|[,;])+{newline}
103 next                    \.\.+
104 char_in_line_comment    [^\r\n]*
105 char_in_comment         [^\r\n\/*]*
106
107 boolnot                 ("@"|"~")
108 booltrue                ("%t"|"%T")
109 boolfalse               ("%f"|"%F")
110 booland                 ("&")
111 boolandand              ("&&")
112 boolor                  ("|")
113 booloror                ("||")
114
115 lbrack                  "["
116 rbrack                  "]"
117
118 lparen                  "("
119 rparen                  ")"
120
121 lbrace                  "{"
122 rbrace                  "}"
123
124 dollar                  "$"
125
126 semicolon               ";"
127 comma                   ","
128 colon                   ":"
129
130 startcomment            "//"
131 startblockcomment       "/*"
132 endblockcomment         "*/"
133
134 dquote                  "\""
135 quote                   "'"
136 in_string               [^\"\'\r\n\.]*
137
138 dot             "."
139 dotquote                ".'"
140 dottimes                ".*"
141 dotrdivide              "./"
142 dotldivide              ".\\"
143 dotpower                (".^"|".**")
144
145 plus                    "+"
146 minus                   "-"
147 rdivide                 "/"
148 ldivide                 "\\"
149 times                   "*"
150 power                   ("^"|"**")
151
152 equal                   "="{spaces}*"="
153 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
154 lowerthan               "<"
155 greaterthan             ">"
156 lowerequal              "<"{spaces}*"="
157 greaterequal            ">"{spaces}*"="
158
159 krontimes               ".*."
160 kronrdivide             "./."
161 kronldivide             ".\\."
162
163 controltimes    ("*."[^0-9])
164 controlrdivide  ("/."[^0-9])
165 controlldivide  ("\\."[^0-9])
166
167 assign                  "="
168
169 %%
170 <INITIAL>{bom}/.* {
171 // BOM found ==> ignored
172 }
173
174 <INITIAL,BEGINID>{booltrue}     {
175     BEGIN(INITIAL);
176     return scan_throw(BOOLTRUE);
177 }
178 <INITIAL,BEGINID>{boolfalse}    {
179     BEGIN(INITIAL);
180     return scan_throw(BOOLFALSE);
181 }
182
183 <INITIAL,BEGINID>"if"            {
184         if (last_token != DOT)
185     {
186         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
187     }
188     DEBUG("BEGIN(INITIAL)");
189     BEGIN(INITIAL);
190     return scan_throw(IF);
191 }
192
193 <INITIAL,BEGINID>"then"          {
194     DEBUG("BEGIN(INITIAL)");
195     BEGIN(INITIAL);
196     return scan_throw(THEN);
197 }
198
199 <INITIAL,BEGINID>"else"          {
200         if (last_token != DOT)
201     {
202         // Pop to step out IF
203         ParserSingleInstance::popControlStatus();
204         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
205     }
206     DEBUG("BEGIN(INITIAL)");
207     BEGIN(INITIAL);
208         return scan_throw(ELSE);
209 }
210
211 <INITIAL,BEGINID>"elseif" {
212         if (last_token != DOT)
213     {
214         ParserSingleInstance::popControlStatus();
215         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
216     }
217     DEBUG("BEGIN(INITIAL)");
218     BEGIN(INITIAL);
219         return scan_throw(ELSEIF);
220 }
221
222 <INITIAL,BEGINID>"end"          {
223         if (last_token != DOT)
224     {
225         ParserSingleInstance::popControlStatus();
226     }
227     DEBUG("BEGIN(INITIAL)");
228     BEGIN(INITIAL);
229     return scan_throw(END);
230 }
231
232 <INITIAL,BEGINID>"select"       {
233         if (last_token != DOT)
234     {
235         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
236     }
237     DEBUG("BEGIN(INITIAL)");
238     BEGIN(INITIAL);
239     return scan_throw(SELECT);
240 }
241
242 <INITIAL,BEGINID>"switch"       {
243         if (last_token != DOT)
244     {
245         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
246     }
247     DEBUG("BEGIN(INITIAL)");
248     BEGIN(INITIAL);
249     return scan_throw(SWITCH);
250 }
251
252 <INITIAL,BEGINID>"otherwise" {
253         if (last_token != DOT)
254     {
255         ParserSingleInstance::popControlStatus();
256         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
257     }
258     DEBUG("BEGIN(INITIAL)");
259     BEGIN(INITIAL);
260         return scan_throw(OTHERWISE);
261 }
262
263 <INITIAL,BEGINID>"case"         {
264         if (last_token != DOT)
265     {
266         ParserSingleInstance::popControlStatus();
267         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
268     }
269     DEBUG("BEGIN(INITIAL)");
270     BEGIN(INITIAL);
271     return scan_throw(CASE);
272 }
273
274 <INITIAL,BEGINID>"function" {
275         if (last_token != DOT)
276     {
277         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
278     }
279     DEBUG("BEGIN(INITIAL)");
280     BEGIN(INITIAL);
281     return scan_throw(FUNCTION);
282 }
283
284 <INITIAL,BEGINID>"endfunction" {
285         if (last_token != DOT)
286     {
287         ParserSingleInstance::popControlStatus();
288     }
289     DEBUG("BEGIN(INITIAL)");
290     BEGIN(INITIAL);
291         return scan_throw(ENDFUNCTION);
292 }
293
294 <INITIAL,BEGINID>"for" {
295         if (last_token != DOT)
296     {
297         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
298     }
299     BEGIN(INITIAL);
300     return scan_throw(FOR);
301 }
302
303 <INITIAL,BEGINID>"while"        {
304         if (last_token != DOT)
305     {
306         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
307     }
308         BEGIN(INITIAL);
309         return scan_throw(WHILE);
310 }
311
312 <INITIAL,BEGINID>"do"           {
313         BEGIN(INITIAL);
314     return scan_throw(DO);
315 }
316
317 <INITIAL,BEGINID>"break"                {
318         BEGIN(INITIAL);
319         return scan_throw(BREAK);
320 }
321
322 <INITIAL,BEGINID>"continue"             {
323         BEGIN(INITIAL);
324         return scan_throw(CONTINUE);
325 }
326
327 <INITIAL,BEGINID>"try" {
328         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
329         BEGIN(INITIAL);
330         return scan_throw(TRY);
331 }
332
333 <INITIAL,BEGINID>"catch" {
334     // Pop to step out TRY
335         ParserSingleInstance::popControlStatus();
336         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
337         BEGIN(INITIAL);
338         return scan_throw(CATCH);
339 }
340
341 <INITIAL,BEGINID>"return"       {
342     BEGIN(INITIAL);
343     return scan_throw(RETURN);
344 }
345
346 <INITIAL,BEGINID>"resume"       {
347     BEGIN(INITIAL);
348     return scan_throw(RETURN);
349 }
350
351 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
352         BEGIN(BEGINID);
353 }
354
355 <BEGINID>
356 {
357     {id}                        {
358         wchar_t *pwText = to_wide_string(yytext);
359         if (yytext != NULL && pwText == NULL)
360         {
361             std::string str = "Can\'t convert \'";
362             str += yytext;
363             str += "\' to UTF-8";
364             BEGIN(INITIAL);
365             yyerror(str);
366             return scan_throw(FLEX_ERROR);
367         }
368         yylval.str = new std::wstring(pwText);
369         FREE(pwText);
370         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
371         if (pIT && pIT->isCallable())
372         {
373             scan_throw(ID);
374             BEGIN(SHELLMODE);
375         }
376         else
377         {
378             BEGIN(INITIAL);
379             return scan_throw(ID);
380         }
381     }
382
383 }
384
385 <INITIAL,MATRIX>{boolnot}               {
386   return scan_throw(NOT);
387 }
388 <INITIAL,MATRIX>{dollar}                {
389   return scan_throw(DOLLAR);
390 }
391 <INITIAL,MATRIX>{booltrue}              {
392   return scan_throw(BOOLTRUE);
393 }
394 <INITIAL,MATRIX>{boolfalse}             {
395   return scan_throw(BOOLFALSE);
396 }
397 <INITIAL,MATRIX>{booland}               {
398   return scan_throw(AND);
399 }
400 <INITIAL,MATRIX>{boolandand}    {
401   return scan_throw(ANDAND);
402 }
403 <INITIAL,MATRIX>{boolor}                {
404   return scan_throw(OR);
405 }
406 <INITIAL,MATRIX>{booloror}              {
407   return scan_throw(OROR);
408 }
409
410
411 <INITIAL>{lparen}                       {
412   return scan_throw(LPAREN);
413 }
414 <INITIAL>{rparen}                       {
415   return scan_throw(RPAREN);
416 }
417
418
419 <INITIAL,MATRIX>{semicolon}             {
420         scan_step();
421   return scan_throw(SEMI);
422 }
423
424 <INITIAL,MATRIX>{comma}                 {
425         scan_step();
426   return scan_throw(COMMA);
427 }
428
429 <INITIAL,MATRIX>{colon}                 {
430   return scan_throw(COLON);
431 }
432
433
434 <INITIAL,MATRIX>{lbrace}                {
435   yy_push_state(MATRIX);
436   paren_levels.push(0);
437   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
438   return scan_throw(LBRACE);
439 }
440
441 {rbrace}                        {
442   return scan_throw(RBRACE);
443 }
444
445
446 <INITIAL,MATRIX>{dotquote}              {
447   return scan_throw(DOTQUOTE);
448 }
449 <INITIAL,MATRIX>{dottimes}              {
450   return scan_throw(DOTTIMES);
451 }
452 <INITIAL,MATRIX>{dotrdivide}            {
453   return scan_throw(DOTRDIVIDE);
454 }
455 <INITIAL,MATRIX>{dotldivide}            {
456   return scan_throw(DOTLDIVIDE);
457 }
458 <INITIAL,MATRIX>{dotpower}              {
459   return scan_throw(DOTPOWER);
460 }
461
462
463 {minus}                                 {
464   return scan_throw(MINUS);
465 }
466 {plus}                                  {
467   return scan_throw(PLUS);
468 }
469 <INITIAL,MATRIX>{times}                 {
470   return scan_throw(TIMES);
471 }
472 <INITIAL,MATRIX>{rdivide}               {
473   return scan_throw(RDIVIDE);
474 }
475 <INITIAL,MATRIX>{ldivide}               {
476   return scan_throw(LDIVIDE);
477 }
478 <INITIAL,MATRIX>{power}                 {
479   return scan_throw(POWER);
480 }
481
482 <INITIAL,MATRIX>{krontimes}             {
483   return scan_throw(KRONTIMES);
484 }
485 <INITIAL,MATRIX>{kronrdivide}           {
486   return scan_throw(KRONRDIVIDE);
487 }
488 <INITIAL,MATRIX>{kronldivide}           {
489   return scan_throw(KRONLDIVIDE);
490 }
491
492
493 <INITIAL,MATRIX>{controltimes}          {
494     unput(yytext[yyleng - 1]);
495     return scan_throw(CONTROLTIMES);
496 }
497 <INITIAL,MATRIX>{controlrdivide}                {
498     unput(yytext[yyleng - 1]);
499     return scan_throw(CONTROLRDIVIDE);
500 }
501 <INITIAL,MATRIX>{controlldivide}                {
502     unput(yytext[yyleng - 1]);
503     return scan_throw(CONTROLLDIVIDE);
504 }
505
506
507 <INITIAL,MATRIX>{equal}                 {
508   return scan_throw(EQ);
509 }
510 <INITIAL,MATRIX>{notequal}              {
511   return scan_throw(NE);
512 }
513 <INITIAL,MATRIX>{lowerthan}             {
514   return scan_throw(LT);
515 }
516 <INITIAL,MATRIX>{greaterthan}           {
517   return scan_throw(GT);
518 }
519 <INITIAL,MATRIX>{lowerequal}            {
520   return scan_throw(LE);
521 }
522 <INITIAL,MATRIX>{greaterequal}          {
523   return scan_throw(GE);
524 }
525
526
527 <INITIAL,MATRIX>{assign}                {
528   return scan_throw(ASSIGN);
529  }
530
531
532 <INITIAL,MATRIX>{lbrack}                {
533   DEBUG("yy_push_state(MATRIX)");
534   yy_push_state(MATRIX);
535   paren_levels.push(0);
536   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
537   return scan_throw(LBRACK);
538 }
539
540 <INITIAL>{rbrack}                               {
541   return scan_throw(RBRACK);
542 }
543
544
545 <INITIAL,MATRIX>{dot}                   {
546   return scan_throw(DOT);
547 }
548
549 <INITIAL>{next}                 {
550     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
551     yy_push_state(LINEBREAK);
552 }
553
554 <INITIAL,MATRIX>{integer}               {
555   yylval.number = atof(yytext);
556 #ifdef TOKENDEV
557   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
558 #endif
559 //  scan_step();
560   return scan_throw(VARINT);
561 }
562
563
564 <INITIAL,MATRIX>{floating_D}            {
565   scan_exponent_convert(yytext);
566   yylval.number = atof(yytext);
567 #ifdef TOKENDEV
568   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
569 #endif
570   //scan_step();
571   return scan_throw(VARFLOAT);
572 }
573
574 <INITIAL,MATRIX>{floating_E}            {
575   yylval.number = atof(yytext);
576 #ifdef TOKENDEV
577   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
578 #endif
579   //scan_step();
580   return scan_throw(VARFLOAT);
581 }
582
583
584 <INITIAL,MATRIX>{number}                {
585   yylval.number = atof(yytext);
586 #ifdef TOKENDEV
587   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
588 #endif
589 //  scan_step();
590   return scan_throw(NUM);
591 }
592
593
594 <INITIAL,MATRIX>{little}                {
595   yylval.number = atof(yytext);
596 #ifdef TOKENDEV
597   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
598 #endif
599 //  scan_step();
600   return scan_throw(NUM);
601 }
602
603
604 <INITIAL,MATRIX>{id}                    {
605     wchar_t *pwText = to_wide_string(yytext);
606     if (yytext != NULL && pwText == NULL)
607     {
608         std::string str = "Can\'t convert \'";
609         str += yytext;
610         str += "\' to UTF-8";
611         BEGIN(INITIAL);
612         yyerror(str);
613         return scan_throw(FLEX_ERROR);
614     }
615     yylval.str = new std::wstring(pwText);
616     FREE(pwText);
617 #ifdef TOKENDEV
618   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
619 #endif
620 //  scan_step();
621   return scan_throw(ID);
622 }
623
624
625 <INITIAL,MATRIX>{startblockcomment}     {
626   yylval.comment = new std::wstring();
627   comment_level = 1;
628   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
629   yy_push_state(REGIONCOMMENT);
630 }
631
632
633 <INITIAL,MATRIX>{startcomment}          {
634   pstBuffer.clear();
635   yy_push_state(LINECOMMENT);
636 }
637
638
639 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
640   pstBuffer.clear();
641   str_opener_column = yylloc.first_column;
642   yy_push_state(DOUBLESTRING);
643 }
644
645
646 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
647   /*
648   ** Matrix Transposition special behaviour
649   ** ID' []' toto()' are transposition call
650   */
651   if (last_token == ID
652       || last_token == RBRACK
653       || last_token == RPAREN
654       || last_token == RBRACE
655       || last_token == VARINT
656       || last_token == VARFLOAT
657       || last_token == NUM
658       || last_token == BOOLTRUE
659       || last_token == BOOLFALSE)
660   {
661       return scan_throw(QUOTE);
662   }
663   else
664   {
665       pstBuffer.clear();
666       str_opener_column = yylloc.first_column;
667       yy_push_state(SIMPLESTRING);
668   }
669 }
670
671
672 <INITIAL,MATRIX>{spaces}                {
673   scan_step();
674   scan_throw(SPACES);
675 }
676
677
678 <INITIAL>{newline}              {
679   yylloc.last_line += 1;
680   yylloc.last_column = 1;
681   scan_step();
682   if (last_token != EOL) {
683       return scan_throw(EOL);
684   }
685
686 }
687
688
689 <INITIAL,MATRIX>{blankline}             {
690   yylloc.last_line += 1;
691   yylloc.last_column = 1;
692   scan_step();
693   if (last_token != EOL)
694   {
695       return scan_throw(EOL);
696   }
697   scan_throw(EOL);
698 }
699
700 <INITIAL,MATRIX>{emptyline}             {
701   yylloc.last_line += 2;
702   yylloc.last_column = 1;
703   scan_step();
704   if (last_token != EOL)
705   {
706       return scan_throw(EOL);
707   }
708   scan_throw(EOL);
709 }
710 .                                       {
711     std::string str = "Unexpected token \'";
712     str += yytext;
713     str += "\'";
714     BEGIN(INITIAL);
715     yyerror(str);
716     return scan_throw(FLEX_ERROR);
717 }
718
719
720 <MATRIX>
721 {
722   {lparen} {
723     ++paren_levels.top();
724     return scan_throw(LPAREN);
725   }
726
727   {rparen} {
728     --paren_levels.top();
729     return scan_throw(RPAREN);
730   }
731
732   {spaces}*{lparen} {
733       unput(yytext[yyleng -1]);
734       if (last_token == ID
735           || last_token == RPAREN
736           || last_token == QUOTE
737           || last_token == VARINT
738           || last_token == VARFLOAT
739           || last_token == NUM)
740       {
741           return scan_throw(COMMA);
742       }
743   }
744
745   {spaces}*{colon}{spaces}* {
746       return scan_throw(COLON);
747   }
748
749   {newline} {
750       yylloc.last_line += 1;
751       yylloc.last_column = 1;
752       if(last_token != DOTS && last_token != EOL)
753       {
754           return scan_throw(EOL);
755       }
756       scan_throw(EOL);
757   }
758
759
760   {rbrack}                              {
761     DEBUG("yy_pop_state()");
762     yy_pop_state();
763     paren_levels.pop();
764     ParserSingleInstance::popControlStatus();
765     return scan_throw(RBRACK);
766   }
767
768   {rbrace}                              {
769     yy_pop_state();
770     paren_levels.pop();
771     ParserSingleInstance::popControlStatus();
772     return scan_throw(RBRACE);
773   }
774
775   {plus}                                |
776   {spaces}{plus}{spaces}                {
777     return scan_throw(PLUS);
778   }
779
780   {minus}                               |
781   {spaces}{minus}{spaces}               {
782     return scan_throw(MINUS);
783   }
784
785   {spaces}{plus}                        {
786     // no need to unput the '+'
787     if (last_token != LBRACK
788        && last_token != EOL
789        && last_token != SEMI
790        && last_token != COMMA
791         && paren_levels.top() == 0)
792    {
793        return scan_throw(COMMA);
794    }
795    else
796    {
797        unput('+');
798    }
799   }
800
801   {spaces}{minus}                       {
802     unput('-');
803     if (last_token != LBRACK
804        && last_token != EOL
805        && last_token != SEMI
806        && last_token != COMMA
807         && paren_levels.top() == 0)
808    {
809        return scan_throw(COMMA);
810    }
811   }
812
813   .                                     {
814     std::string str = "Unexpected token \'";
815     str += yytext;
816     str += "\' within a matrix.";
817     BEGIN(INITIAL);
818     yyerror(str);
819     return scan_throw(FLEX_ERROR);
820   }
821
822   {next}{spaces}*{newline}          {
823       /* Just do nothing */
824       yylloc.last_line += 1;
825       yylloc.last_column = 1;
826       if(last_token == SPACES)
827       {
828         unput(' ');
829       }
830
831       scan_step();
832   }
833
834   {next}{spaces}*{startcomment}          {
835       /* Just do nothing */
836       pstBuffer.clear();
837       yy_push_state(LINECOMMENT);
838       scan_throw(DOTS);
839   }
840
841   <<EOF>>       {
842       yy_pop_state();
843       paren_levels.pop();
844   }
845 }
846
847 <LINEBREAK>
848 {
849   {newline}                             {
850     yylloc.last_line += 1;
851     yylloc.last_column = 1;
852     scan_step();
853     yy_pop_state();
854     ParserSingleInstance::popControlStatus();
855   }
856
857   {startblockcomment}                   {
858     ++comment_level;
859     yy_push_state(REGIONCOMMENT);
860   }
861
862   {startcomment}                        {
863     scan_throw(DOTS);
864     pstBuffer.clear();
865     yy_push_state(LINECOMMENT);
866   }
867
868   {spaces}                              {
869       /* Do nothing... */
870   }
871
872   <<EOF>>       {
873       yy_pop_state();
874   }
875   .                                     {
876     std::string str = "Unexpected token \'";
877     str += yytext;
878     str += "\' after line break with .. or ...";
879     BEGIN(INITIAL);
880     yyerror(str);
881     return scan_throw(FLEX_ERROR);
882   }
883 }
884
885
886 <LINECOMMENT>
887 {
888   {newline}     {
889     //yylloc.last_line += 1;
890     //yylloc.last_column = 1;
891     //scan_step();
892     yy_pop_state();
893     for (int i = yyleng - 1 ; i >= 0 ; --i)
894     {
895         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
896         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
897         unput(yytext[i]);
898         yylloc.last_column--;
899     }
900     /*
901     ** To forgot comments after lines break
902     */
903     if (last_token != DOTS)
904     {
905         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
906         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
907         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
908         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
909         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
910         {
911             pstBuffer.clear();
912             std::string str = "Can\'t convert \'";
913             str += pstBuffer.c_str();
914             str += "\' to UTF-8";
915             BEGIN(INITIAL);
916             yyerror(str);
917             return scan_throw(FLEX_ERROR);
918         }
919         yylval.comment = new std::wstring(pwstBuffer);
920         pstBuffer.clear();
921         FREE (pwstBuffer);
922         return scan_throw(COMMENT);
923     }
924     else
925     {
926         pstBuffer.clear();
927     }
928   }
929
930   <<EOF>>       {
931     yy_pop_state();
932     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
933     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
934     {
935         pstBuffer.clear();
936         std::string str = "Can\'t convert \'";
937         str += pstBuffer.c_str();
938         str += "\' to UTF-8";
939         BEGIN(INITIAL);
940         yyerror(str);
941         return scan_throw(FLEX_ERROR);
942     }
943     yylval.comment = new std::wstring(pwstBuffer);
944     pstBuffer.clear();
945     FREE (pwstBuffer);
946     return scan_throw(COMMENT);
947   }
948
949   {char_in_line_comment}         {
950       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
951       // only translate to WCHAR_T when popping state.
952       pstBuffer += yytext;
953   }
954 }
955
956
957 <REGIONCOMMENT>
958 {
959   {endblockcomment}                             {
960     --comment_level;
961     if (comment_level == 0) {
962       ParserSingleInstance::popControlStatus();
963       yy_pop_state();
964       //return scan_throw(BLOCKCOMMENT);
965     }
966   }
967
968   {startblockcomment}                           {
969     ++comment_level;
970     yy_push_state(REGIONCOMMENT);
971   }
972
973   {newline}                                     {
974     yylloc.last_line += 1;
975     yylloc.last_column = 1;
976     scan_step();
977     *yylval.comment += L"\n//";
978   }
979
980   {char_in_comment}                             |
981   .                                             {
982       wchar_t *pwText = to_wide_string(yytext);
983       *yylval.comment += std::wstring(pwText);
984       FREE(pwText);
985   }
986
987  <<EOF>>                                        {
988       yy_pop_state();
989 //    std::string str = "unexpected end of file in a comment";
990 //    scan_error(str);
991   }
992 }
993
994
995 <SIMPLESTRING>
996 {
997   {dquote}{dquote}                              {
998     pstBuffer += "\"";
999   }
1000
1001   {dquote}{quote}                               {
1002     pstBuffer += "'";
1003   }
1004
1005   {quote}{dquote}                               {
1006     pstBuffer += "\"";
1007   }
1008
1009   {quote}{quote}                                {
1010     pstBuffer += "'";
1011   }
1012
1013   {quote}                                       {
1014     yy_pop_state();
1015     //scan_step();
1016     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1017     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1018     {
1019         pstBuffer.clear();
1020         std::string str = "Can\'t convert \'";
1021         str += pstBuffer.c_str();
1022         str += "\' to UTF-8";
1023         BEGIN(INITIAL);
1024         yyerror(str);
1025         return scan_throw(FLEX_ERROR);
1026     }
1027     yylval.str = new std::wstring(pwstBuffer);
1028     pstBuffer.clear();
1029     FREE(pwstBuffer);
1030     yylloc.first_column = str_opener_column;
1031     return scan_throw(STR);
1032   }
1033
1034   {dquote}                  {
1035     pstBuffer.clear();
1036     BEGIN(INITIAL);
1037     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1038     return scan_throw(FLEX_ERROR);
1039   }
1040
1041   {next}{newline}           {
1042       yylloc.last_line += 1;
1043       yylloc.last_column = 1;
1044       /* Do nothing... Just skip */
1045   }
1046
1047   {newline}                                     {
1048     pstBuffer.clear();
1049     yylloc.last_line += 1;
1050     yylloc.last_column = 1;
1051     BEGIN(INITIAL);
1052     yyerror("Unexpected end of line in a string.");
1053     return scan_throw(FLEX_ERROR);
1054   }
1055
1056   <<EOF>>                                       {
1057     pstBuffer.clear();
1058     BEGIN(INITIAL);
1059     yyerror("Unexpected end of file in a string.");
1060     return scan_throw(FLEX_ERROR);
1061   }
1062
1063   {in_string}                                           |
1064   .                                                     {
1065     //scan_step();
1066     pstBuffer += yytext;
1067   }
1068 }
1069
1070
1071 <DOUBLESTRING>
1072 {
1073   {dquote}{dquote}                              {
1074     pstBuffer += "\"";
1075   }
1076
1077   {dquote}{quote}                               {
1078     pstBuffer += "'";
1079   }
1080
1081   {quote}{dquote}               {
1082     pstBuffer += "\"";
1083   }
1084
1085   {quote}{quote}                                {
1086     pstBuffer += "'";
1087   }
1088
1089   {dquote}                      {
1090     yy_pop_state();
1091     //scan_step();
1092     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1093     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1094     {
1095         pstBuffer.clear();
1096         std::string str = "Can\'t convert \'";
1097         str += pstBuffer.c_str();
1098         str += "\' to UTF-8";
1099         BEGIN(INITIAL);
1100         yyerror(str);
1101         return scan_throw(FLEX_ERROR);
1102     }
1103     yylval.str = new std::wstring(pwstBuffer);
1104     pstBuffer.clear();
1105     FREE(pwstBuffer);
1106     yylloc.first_column = str_opener_column;
1107     return scan_throw(STR);
1108   }
1109
1110   {quote}                  {
1111     pstBuffer.clear();
1112     BEGIN(INITIAL);
1113     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1114     return scan_throw(FLEX_ERROR);
1115   }
1116
1117   {next}{newline}           {
1118       yylloc.last_line += 1;
1119       yylloc.last_column = 1;
1120       /* Do nothing... Just skip */
1121   }
1122
1123   {newline} {
1124     pstBuffer.clear();
1125     yylloc.last_line += 1;
1126     yylloc.last_column = 1;
1127     BEGIN(INITIAL);
1128     yyerror("Unexpected end of line in a string.");
1129     return scan_throw(FLEX_ERROR);
1130   }
1131
1132   <<EOF>>   {
1133     pstBuffer.clear();
1134     BEGIN(INITIAL);
1135     yyerror("Unexpected end of file in a string.");
1136     return scan_throw(FLEX_ERROR);
1137   }
1138
1139   {in_string}         |
1140   .                   {
1141    //scan_step();
1142    pstBuffer += yytext;
1143   }
1144 }
1145
1146
1147 <SHELLMODE>
1148 {
1149     {spaces}                    {
1150         if (last_token == ID)
1151         {
1152             scan_throw(SPACES);
1153             return ID;
1154         }
1155     }
1156
1157     {semicolon}                 {
1158         BEGIN(INITIAL);
1159         scan_step();
1160         return scan_throw(SEMI);
1161     }
1162
1163     {comma}                     {
1164         BEGIN(INITIAL);
1165         scan_step();
1166         return scan_throw(COMMA);
1167     }
1168
1169     {newline}                   {
1170         BEGIN(INITIAL);
1171         yylloc.last_line += 1;
1172         yylloc.last_column = 1;
1173         scan_step();
1174         return scan_throw(EOL);
1175     }
1176
1177     {assign} {
1178         if (last_token == STR || last_token == SPACES)
1179         {
1180             wchar_t *pwText = to_wide_string(yytext);
1181             yylval.str = new std::wstring(pwText);
1182             FREE(pwText);
1183             return scan_throw(STR);
1184         }
1185         else
1186         {
1187             BEGIN(INITIAL);
1188             return scan_throw(ASSIGN);
1189         }
1190     }
1191
1192     {lparen} {
1193         if (last_token == STR || last_token == SPACES)
1194         {
1195             wchar_t *pwText = to_wide_string(yytext);
1196             yylval.str = new std::wstring(pwText);
1197             FREE(pwText);
1198             return scan_throw(STR);
1199         }
1200         else
1201         {
1202             BEGIN(INITIAL);
1203             return scan_throw(LPAREN);
1204         }
1205     }
1206
1207     {lowerthan} {
1208         if (last_token == STR || last_token == SPACES)
1209         {
1210             wchar_t *pwText = to_wide_string(yytext);
1211             yylval.str = new std::wstring(pwText);
1212             FREE(pwText);
1213             return scan_throw(STR);
1214         }
1215         else
1216         {
1217             BEGIN(INITIAL);
1218             return scan_throw(LT);
1219         }
1220     }
1221
1222     {greaterthan} {
1223         if (last_token == STR || last_token == SPACES)
1224         {
1225             wchar_t *pwText = to_wide_string(yytext);
1226             yylval.str = new std::wstring(pwText);
1227             FREE(pwText);
1228             return scan_throw(STR);
1229         }
1230         else
1231         {
1232             BEGIN(INITIAL);
1233             return scan_throw(GT);
1234         }
1235     }
1236
1237     {boolnot} {
1238         if (last_token == STR || last_token == SPACES)
1239         {
1240             wchar_t *pwText = to_wide_string(yytext);
1241             yylval.str = new std::wstring(pwText);
1242             FREE(pwText);
1243             return scan_throw(STR);
1244         }
1245         else
1246         {
1247             BEGIN(INITIAL);
1248             return scan_throw(NOT);
1249         }
1250     }
1251
1252
1253     [^ \t\v\f\r\n,;'"]+               {
1254         wchar_t *pwText = to_wide_string(yytext);
1255         yylval.str = new std::wstring(pwText);
1256         FREE(pwText);
1257         return scan_throw(STR);
1258     }
1259
1260     <<EOF>>                     {
1261         BEGIN(INITIAL);
1262     }
1263
1264 }
1265
1266 %%
1267
1268 int scan_throw(int token) {
1269   last_token = token;
1270 #ifdef DEV
1271   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1272 #endif
1273   return token;
1274 }
1275
1276 int get_last_token() {
1277     return last_token;
1278 }
1279
1280 void scan_step() {
1281   yylloc.first_line = yylloc.last_line;
1282   yylloc.first_column = yylloc.last_column;
1283 }
1284
1285 /*
1286 ** convert floating numbers to C standard
1287 ** 1.2d-3 -> 1.2e-3
1288 ** 1.2D-3 -> 1.2e-3
1289 */
1290 void scan_exponent_convert(char *in)
1291 {
1292   for (; *in != 'd' && *in != 'D'; ++in);
1293   *in = 'e';
1294 }
1295
1296 #ifdef _MSC_VER
1297 int isatty (int desc)
1298 {
1299   return 0;
1300 }
1301 #endif