Lexer: slightly improve strings & comments lexing
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "sci_malloc.h"
23 }
24
25 static int comment_level = 0;
26 static int paren_level = 0;
27 static int last_token = 0;
28 static int exit_status = PARSE_ERROR;
29 static std::string current_file;
30 static std::string program_name;
31
32 static std::string *pstBuffer;
33
34 #define YY_USER_ACTION                          \
35  yylloc.last_column += yyleng;
36
37 /* -*- Verbose Special Debug -*- */
38 //#define DEV
39 //#define TOKENDEV
40
41 #ifdef DEV
42 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
43 #else
44 #define DEBUG(x) /* Nothing */
45 #endif
46
47 %}
48
49 %option stack
50 %option noyywrap
51
52 %x SIMPLESTRING
53 %x DOUBLESTRING
54 %x REGIONCOMMENT
55 %x LINECOMMENT
56 %x LINEBREAK
57
58 %x MATRIX
59
60 %x SHELLMODE
61 %x BEGINID
62
63 spaces                  [ \t\v\f]+
64 integer                 [0-9]+
65 number                  [0-9]+[\.][0-9]*
66 little                  \.[0-9]+
67
68 floating                ({little}|{number}|{integer})[deDE][+-]?{integer}
69
70 hex             [0]x[0-9a-fA-F]+
71 oct             [0]o[0-7]+
72
73
74 utf2            ([\xC2-\xDF][\x80-\xBF])
75 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
76 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
77 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
78 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
79 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
80 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
81 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
82
83 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
84 utf4            ({utf41}|{utf42}|{utf43})
85
86 utf             ({utf2}|{utf3}|{utf4})
87 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
88
89
90 newline                 ("\r"|"\n"|"\r\n")
91 blankline               {spaces}+{newline}
92 emptyline       {newline}({spaces}|[,;])+{newline}
93 next                    \.\.+
94 char_in_line_comment    [^\r\n]*
95 char_in_comment         [^\r\n\/*]*
96
97 boolnot                 ("@"|"~")
98 booltrue                ("%t"|"%T")
99 boolfalse               ("%f"|"%F")
100 booland                 ("&")
101 boolandand              ("&&")
102 boolor                  ("|")
103 booloror                ("||")
104
105 lbrack                  "["
106 rbrack                  "]"
107
108 lparen                  "("
109 rparen                  ")"
110
111 lbrace                  "{"
112 rbrace                  "}"
113
114 dollar                  "$"
115
116 semicolon               ";"
117 comma                   ","
118 colon                   ":"
119
120 startcomment            "//"
121 startblockcomment       "/*"
122 endblockcomment         "*/"
123
124 dquote                  "\""
125 quote                   "'"
126 in_string               [^\"\'\r\n\.]*
127
128 dot             "."
129 dotquote                ".'"
130 dottimes                ".*"
131 dotrdivide              "./"
132 dotldivide              ".\\"
133 dotpower                (".^"|".**")
134
135 plus                    "+"
136 minus                   "-"
137 rdivide                 "/"
138 ldivide                 "\\"
139 times                   "*"
140 power                   ("^"|"**")
141
142 equal                   "="{spaces}*"="
143 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
144 lowerthan               "<"
145 greaterthan             ">"
146 lowerequal              "<"{spaces}*"="
147 greaterequal            ">"{spaces}*"="
148
149 krontimes               ".*."
150 kronrdivide             "./."
151 kronldivide             ".\\."
152
153 controltimes    ("*."[^0-9])
154 controlrdivide  ("/."[^0-9])
155 controlldivide  ("\\."[^0-9])
156
157 assign                  "="
158
159 %%
160
161 <INITIAL,BEGINID>"if"            {
162         if (last_token != DOT)
163     {
164         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
165     }
166     DEBUG("BEGIN(INITIAL)");
167     BEGIN(INITIAL);
168     return scan_throw(IF);
169 }
170
171 <INITIAL,BEGINID>"then"          {
172     DEBUG("BEGIN(INITIAL)");
173     BEGIN(INITIAL);
174     return scan_throw(THEN);
175 }
176
177 <INITIAL,BEGINID>"else"          {
178         if (last_token != DOT)
179     {
180         // Pop to step out IF
181         ParserSingleInstance::popControlStatus();
182         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
183     }
184     DEBUG("BEGIN(INITIAL)");
185     BEGIN(INITIAL);
186         return scan_throw(ELSE);
187 }
188
189 <INITIAL,BEGINID>"elseif" {
190         if (last_token != DOT)
191     {
192         ParserSingleInstance::popControlStatus();
193         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
194     }
195     DEBUG("BEGIN(INITIAL)");
196     BEGIN(INITIAL);
197         return scan_throw(ELSEIF);
198 }
199
200 <INITIAL,BEGINID>"end"          {
201         if (last_token != DOT)
202     {
203         ParserSingleInstance::popControlStatus();
204     }
205     DEBUG("BEGIN(INITIAL)");
206     BEGIN(INITIAL);
207     return scan_throw(END);
208 }
209
210 <INITIAL,BEGINID>"select"       {
211         if (last_token != DOT)
212     {
213         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
214     }
215     DEBUG("BEGIN(INITIAL)");
216     BEGIN(INITIAL);
217     return scan_throw(SELECT);
218 }
219
220 <INITIAL,BEGINID>"switch"       {
221         if (last_token != DOT)
222     {
223         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
224     }
225     DEBUG("BEGIN(INITIAL)");
226     BEGIN(INITIAL);
227     return scan_throw(SWITCH);
228 }
229
230 <INITIAL,BEGINID>"otherwise" {
231         if (last_token != DOT)
232     {
233         ParserSingleInstance::popControlStatus();
234         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
235     }
236     DEBUG("BEGIN(INITIAL)");
237     BEGIN(INITIAL);
238         return scan_throw(OTHERWISE);
239 }
240
241 <INITIAL,BEGINID>"case"         {
242         if (last_token != DOT)
243     {
244         ParserSingleInstance::popControlStatus();
245         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
246     }
247     DEBUG("BEGIN(INITIAL)");
248     BEGIN(INITIAL);
249     return scan_throw(CASE);
250 }
251
252 <INITIAL,BEGINID>"function" {
253         if (last_token != DOT)
254     {
255         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
256     }
257     DEBUG("BEGIN(INITIAL)");
258     BEGIN(INITIAL);
259     return scan_throw(FUNCTION);
260 }
261
262 <INITIAL,BEGINID>"endfunction" {
263         if (last_token != DOT)
264     {
265         ParserSingleInstance::popControlStatus();
266     }
267     DEBUG("BEGIN(INITIAL)");
268     BEGIN(INITIAL);
269         return scan_throw(ENDFUNCTION);
270 }
271
272 <INITIAL,BEGINID>"#function"    {
273         if (last_token != DOT)
274     {
275         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
276     }
277     DEBUG("BEGIN(INITIAL)");
278     BEGIN(INITIAL);
279         return scan_throw(HIDDENFUNCTION);
280 }
281
282 <INITIAL,BEGINID>"hidden"       {
283     DEBUG("BEGIN(INITIAL)");
284         BEGIN(INITIAL);
285     return scan_throw(HIDDEN);
286 }
287
288 <INITIAL,BEGINID>"for" {
289         if (last_token != DOT)
290     {
291         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
292     }
293     BEGIN(INITIAL);
294     return scan_throw(FOR);
295 }
296
297 <INITIAL,BEGINID>"while"        {
298         if (last_token != DOT)
299     {
300         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
301     }
302         BEGIN(INITIAL);
303         return scan_throw(WHILE);
304 }
305
306 <INITIAL,BEGINID>"do"           {
307         BEGIN(INITIAL);
308     return scan_throw(DO);
309 }
310
311 <INITIAL,BEGINID>"break"                {
312         BEGIN(INITIAL);
313         return scan_throw(BREAK);
314 }
315
316 <INITIAL,BEGINID>"continue"             {
317         BEGIN(INITIAL);
318         return scan_throw(CONTINUE);
319 }
320
321 <INITIAL,BEGINID>"try" {
322         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
323         BEGIN(INITIAL);
324         return scan_throw(TRY);
325 }
326
327 <INITIAL,BEGINID>"catch" {
328     // Pop to step out TRY
329         ParserSingleInstance::popControlStatus();
330         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
331         BEGIN(INITIAL);
332         return scan_throw(CATCH);
333 }
334
335 <INITIAL,BEGINID>"return"       {
336     BEGIN(INITIAL);
337     return scan_throw(RETURN);
338 }
339
340 <INITIAL,BEGINID>"resume"       {
341     BEGIN(INITIAL);
342     return scan_throw(RETURN);
343 }
344
345 ^{spaces}*/({id}){spaces}[^ \t\v\f(=<>~@] {
346         BEGIN(BEGINID);
347 }
348
349 <BEGINID>
350 {
351     {id}                        {
352         wchar_t *pwText = to_wide_string(yytext);
353         if (yytext != NULL && pwText == NULL)
354         {
355             std::string str = "can not convert'";
356             str += yytext;
357             str += "' to UTF-8";
358             exit_status = SCAN_ERROR;
359             scan_error("can not convert string to UTF-8");
360         }
361         yylval.str = new std::wstring(pwText);
362         FREE(pwText);
363         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
364         if (pIT && pIT->isCallable())
365         {
366             scan_throw(ID);
367             BEGIN(SHELLMODE);
368         }
369         else
370         {
371             BEGIN(INITIAL);
372             return scan_throw(ID);
373         }
374     }
375
376 }
377
378 <INITIAL,MATRIX>{boolnot}               {
379   return scan_throw(NOT);
380 }
381 <INITIAL,MATRIX>{dollar}                {
382   return scan_throw(DOLLAR);
383 }
384 <INITIAL,MATRIX>{booltrue}              {
385   return scan_throw(BOOLTRUE);
386 }
387 <INITIAL,MATRIX>{boolfalse}             {
388   return scan_throw(BOOLFALSE);
389 }
390 <INITIAL,MATRIX>{booland}               {
391   return scan_throw(AND);
392 }
393 <INITIAL,MATRIX>{boolandand}    {
394   return scan_throw(ANDAND);
395 }
396 <INITIAL,MATRIX>{boolor}                {
397   return scan_throw(OR);
398 }
399 <INITIAL,MATRIX>{booloror}              {
400   return scan_throw(OROR);
401 }
402
403
404 <INITIAL>{lparen}                       {
405   return scan_throw(LPAREN);
406 }
407 <INITIAL>{rparen}                       {
408   return scan_throw(RPAREN);
409 }
410
411
412 <INITIAL,MATRIX>{semicolon}             {
413         scan_step();
414   return scan_throw(SEMI);
415 }
416
417 <INITIAL,MATRIX>{comma}                 {
418         scan_step();
419   return scan_throw(COMMA);
420 }
421
422 <INITIAL,MATRIX>{colon}                 {
423   return scan_throw(COLON);
424 }
425
426
427 <INITIAL,MATRIX>{lbrace}                {
428   yy_push_state(MATRIX);
429   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
430   return scan_throw(LBRACE);
431 }
432
433 {rbrace}                        {
434   return scan_throw(RBRACE);
435 }
436
437
438 <INITIAL,MATRIX>{dotquote}              {
439   return scan_throw(DOTQUOTE);
440 }
441 <INITIAL,MATRIX>{dottimes}              {
442   return scan_throw(DOTTIMES);
443 }
444 <INITIAL,MATRIX>{dotrdivide}            {
445   return scan_throw(DOTRDIVIDE);
446 }
447 <INITIAL,MATRIX>{dotldivide}            {
448   return scan_throw(DOTLDIVIDE);
449 }
450 <INITIAL,MATRIX>{dotpower}              {
451   return scan_throw(DOTPOWER);
452 }
453
454
455 {minus}                                 {
456   return scan_throw(MINUS);
457 }
458 {plus}                                  {
459   return scan_throw(PLUS);
460 }
461 <INITIAL,MATRIX>{times}                 {
462   return scan_throw(TIMES);
463 }
464 <INITIAL,MATRIX>{rdivide}               {
465   return scan_throw(RDIVIDE);
466 }
467 <INITIAL,MATRIX>{ldivide}               {
468   return scan_throw(LDIVIDE);
469 }
470 <INITIAL,MATRIX>{power}                 {
471   return scan_throw(POWER);
472 }
473
474 <INITIAL,MATRIX>{krontimes}             {
475   return scan_throw(KRONTIMES);
476 }
477 <INITIAL,MATRIX>{kronrdivide}           {
478   return scan_throw(KRONRDIVIDE);
479 }
480 <INITIAL,MATRIX>{kronldivide}           {
481   return scan_throw(KRONLDIVIDE);
482 }
483
484
485 <INITIAL,MATRIX>{controltimes}          {
486     unput(yytext[yyleng - 1]);
487     return scan_throw(CONTROLTIMES);
488 }
489 <INITIAL,MATRIX>{controlrdivide}                {
490     unput(yytext[yyleng - 1]);
491     return scan_throw(CONTROLRDIVIDE);
492 }
493 <INITIAL,MATRIX>{controlldivide}                {
494     unput(yytext[yyleng - 1]);
495     return scan_throw(CONTROLLDIVIDE);
496 }
497
498
499 <INITIAL,MATRIX>{equal}                 {
500   return scan_throw(EQ);
501 }
502 <INITIAL,MATRIX>{notequal}              {
503   return scan_throw(NE);
504 }
505 <INITIAL,MATRIX>{lowerthan}             {
506   return scan_throw(LT);
507 }
508 <INITIAL,MATRIX>{greaterthan}           {
509   return scan_throw(GT);
510 }
511 <INITIAL,MATRIX>{lowerequal}            {
512   return scan_throw(LE);
513 }
514 <INITIAL,MATRIX>{greaterequal}          {
515   return scan_throw(GE);
516 }
517
518
519 <INITIAL,MATRIX>{assign}                {
520   return scan_throw(ASSIGN);
521  }
522
523
524 <INITIAL,MATRIX>{lbrack}                {
525   DEBUG("yy_push_state(MATRIX)");
526   yy_push_state(MATRIX);
527   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
528   return scan_throw(LBRACK);
529 }
530
531 <INITIAL>{rbrack}                               {
532   return scan_throw(RBRACK);
533 }
534
535
536 <INITIAL,MATRIX>{dot}                   {
537   return scan_throw(DOT);
538 }
539
540 <INITIAL>{next}                 {
541     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
542     yy_push_state(LINEBREAK);
543 }
544
545 <INITIAL,MATRIX>{integer}               {
546   yylval.number = atof(yytext);
547 #ifdef TOKENDEV
548   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
549 #endif
550 //  scan_step();
551   return scan_throw(VARINT);
552 }
553
554
555 <INITIAL,MATRIX>{floating}              {
556   scan_exponent_convert(yytext);
557   yylval.number = atof(yytext);
558 #ifdef TOKENDEV
559   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
560 #endif
561   scan_step();
562   return scan_throw(VARFLOAT);
563 }
564
565
566 <INITIAL,MATRIX>{number}                {
567   yylval.number = atof(yytext);
568 #ifdef TOKENDEV
569   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
570 #endif
571 //  scan_step();
572   return scan_throw(NUM);
573 }
574
575
576 <INITIAL,MATRIX>{little}                {
577   yylval.number = atof(yytext);
578 #ifdef TOKENDEV
579   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
580 #endif
581   scan_step();
582   return scan_throw(NUM);
583 }
584
585
586 <INITIAL,MATRIX>{id}                    {
587     wchar_t *pwText = to_wide_string(yytext);
588     if (yytext != NULL && pwText == NULL)
589     {
590         std::string str = "can not convert'";
591         str += yytext;
592         str += "' to UTF-8";
593         exit_status = SCAN_ERROR;
594         scan_error("can not convert string to UTF-8");
595     }
596     yylval.str = new std::wstring(pwText);
597     FREE(pwText);
598 #ifdef TOKENDEV
599   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
600 #endif
601 //  scan_step();
602   return scan_throw(ID);
603 }
604
605
606 <INITIAL,MATRIX>{startblockcomment}     {
607   yylval.comment = new std::wstring();
608   comment_level = 1;
609   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
610   yy_push_state(REGIONCOMMENT);
611 }
612
613
614 <INITIAL,MATRIX>{startcomment}          {
615   pstBuffer = new std::string();
616   yy_push_state(LINECOMMENT);
617 }
618
619
620 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
621   pstBuffer = new std::string();
622   yy_push_state(DOUBLESTRING);
623 }
624
625
626 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
627   /*
628   ** Matrix Transposition special behaviour
629   ** ID' []' toto()' are transposition call
630   */
631   if (last_token == ID
632       || last_token == RBRACK
633       || last_token == RPAREN
634       || last_token == RBRACE
635       || last_token == VARINT
636       || last_token == VARFLOAT
637       || last_token == NUM
638       || last_token == BOOLTRUE
639       || last_token == BOOLFALSE)
640   {
641       return scan_throw(QUOTE);
642   }
643   else
644   {
645       pstBuffer = new std::string();
646       yy_push_state(SIMPLESTRING);
647   }
648 }
649
650
651 <INITIAL,MATRIX>{spaces}                {
652   scan_step();
653   scan_throw(SPACES);
654 }
655
656
657 <INITIAL>{newline}              {
658   yylloc.last_line += 1;
659   yylloc.last_column = 1;
660   scan_step();
661   if (last_token != EOL) {
662       return scan_throw(EOL);
663   }
664
665 }
666
667
668 <INITIAL,MATRIX>{blankline}             {
669   yylloc.last_line += 1;
670   yylloc.last_column = 1;
671   scan_step();
672   if (last_token != EOL)
673   {
674       return scan_throw(EOL);
675   }
676   scan_throw(EOL);
677 }
678
679 <INITIAL,MATRIX>{emptyline}             {
680   yylloc.last_line += 2;
681   yylloc.last_column = 1;
682   scan_step();
683   if (last_token != EOL)
684   {
685       return scan_throw(EOL);
686   }
687   scan_throw(EOL);
688 }
689 .                                       {
690     std::string str = "unexpected token '";
691     str += yytext;
692     str += "'";
693     exit_status = SCAN_ERROR;
694     scan_error(str);
695 }
696
697
698 <MATRIX>
699 {
700   {lparen} {
701     paren_level++;
702     return scan_throw(LPAREN);
703   }
704
705   {rparen} {
706     paren_level--;
707     return scan_throw(RPAREN);
708   }
709
710   {spaces}*{lparen} {
711       unput(yytext[yyleng -1]);
712       if (last_token == ID
713           || last_token == RPAREN
714           || last_token == QUOTE
715           || last_token == VARINT
716           || last_token == VARFLOAT
717           || last_token == NUM)
718       {
719           return scan_throw(COMMA);
720       }
721   }
722
723   {spaces}*{colon}{spaces}* {
724       return scan_throw(COLON);
725   }
726
727   {newline} {
728       yylloc.last_line += 1;
729       yylloc.last_column = 1;
730       if(last_token != DOTS && last_token != EOL)
731       {
732           return scan_throw(EOL);
733       }
734       scan_throw(EOL);
735   }
736
737
738   {rbrack}                              {
739     DEBUG("yy_pop_state()");
740     yy_pop_state();
741     ParserSingleInstance::popControlStatus();
742     return scan_throw(RBRACK);
743   }
744
745   {rbrace}                              {
746     yy_pop_state();
747     ParserSingleInstance::popControlStatus();
748     return scan_throw(RBRACE);
749   }
750
751   {plus}                                |
752   {spaces}{plus}{spaces}                {
753     return scan_throw(PLUS);
754   }
755
756   {minus}                               |
757   {spaces}{minus}{spaces}               {
758     return scan_throw(MINUS);
759   }
760
761   {spaces}{plus}                        {
762     // no need to unput the '+'
763     if (last_token != LBRACK
764        && last_token != EOL
765        && last_token != SEMI
766        && last_token != COMMA
767        && paren_level == 0)
768    {
769        return scan_throw(COMMA);
770    }
771    else
772    {
773        unput('+');
774    }
775   }
776
777   {spaces}{minus}                       {
778     unput('-');
779     if (last_token != LBRACK
780        && last_token != EOL
781        && last_token != SEMI
782        && last_token != COMMA
783        && paren_level == 0)
784    {
785        return scan_throw(COMMA);
786    }
787   }
788
789   .                                     {
790     std::string str = "unexpected token '";
791     str += yytext;
792     str += "' within a matrix.";
793     exit_status = SCAN_ERROR;
794     scan_error(str);
795   }
796
797   {next}{spaces}*{newline}          {
798       /* Just do nothing */
799       yylloc.last_line += 1;
800       yylloc.last_column = 1;
801       scan_step();
802       scan_throw(EOL);
803   }
804
805   {next}{spaces}*{startcomment}          {
806       /* Just do nothing */
807       pstBuffer = new std::string();
808       yy_push_state(LINECOMMENT);
809       scan_throw(DOTS);
810   }
811
812   <<EOF>>       {
813       yy_pop_state();
814   }
815 }
816
817 <LINEBREAK>
818 {
819   {newline}                             {
820     yylloc.last_line += 1;
821     yylloc.last_column = 1;
822     scan_step();
823     yy_pop_state();
824     ParserSingleInstance::popControlStatus();
825   }
826
827   {startblockcomment}                   {
828     ++comment_level;
829     yy_push_state(REGIONCOMMENT);
830   }
831
832   {startcomment}                        {
833     scan_throw(DOTS);
834     pstBuffer = new std::string();
835     yy_push_state(LINECOMMENT);
836   }
837
838   {spaces}                              {
839       /* Do nothing... */
840   }
841
842   <<EOF>>       {
843       yy_pop_state();
844   }
845   .                                     {
846     std::string str = "unexpected token '";
847     str += yytext;
848     str += "' after line break with .. or ...";
849     exit_status = SCAN_ERROR;
850     scan_error(str);
851   }
852 }
853
854
855 <LINECOMMENT>
856 {
857   {newline}     {
858     //yylloc.last_line += 1;
859     //yylloc.last_column = 1;
860     //scan_step();
861     yy_pop_state();
862     for (int i = yyleng - 1 ; i >= 0 ; --i)
863     {
864         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
865         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
866         unput(yytext[i]);
867         yylloc.last_column--;
868     }
869     /*
870     ** To forgot comments after lines break
871     */
872     if (last_token != DOTS)
873     {
874         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
875         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
876         wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
877         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
878         if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
879         {
880             std::string str = "can not convert'";
881             str += pstBuffer->c_str();
882             str += "' to UTF-8";
883             exit_status = SCAN_ERROR;
884             scan_error("can not convert string to UTF-8");
885         }
886         yylval.comment = new std::wstring(pwstBuffer);
887         delete pstBuffer;
888         FREE (pwstBuffer);
889         return scan_throw(COMMENT);
890     }
891     else
892     {
893         delete pstBuffer;
894     }
895   }
896
897   <<EOF>>       {
898     yy_pop_state();
899     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
900     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
901     {
902         std::string str = "can not convert'";
903         str += pstBuffer->c_str();
904         str += "' to UTF-8";
905         exit_status = SCAN_ERROR;
906         scan_error("can not convert string to UTF-8");
907     }
908     yylval.comment = new std::wstring(pwstBuffer);
909     delete pstBuffer;
910     FREE (pwstBuffer);
911     return scan_throw(COMMENT);
912   }
913
914   {char_in_line_comment}         {
915       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
916       // only translate to WCHAR_T when popping state.
917       *pstBuffer += yytext;
918   }
919 }
920
921
922 <REGIONCOMMENT>
923 {
924   {endblockcomment}                             {
925     --comment_level;
926     if (comment_level == 0) {
927       ParserSingleInstance::popControlStatus();
928       yy_pop_state();
929       //return scan_throw(BLOCKCOMMENT);
930     }
931   }
932
933   {startblockcomment}                           {
934     ++comment_level;
935     yy_push_state(REGIONCOMMENT);
936   }
937
938   {newline}                                     {
939     yylloc.last_line += 1;
940     yylloc.last_column = 1;
941     scan_step();
942     *yylval.comment += L"\n//";
943   }
944
945   {char_in_comment}                             |
946   .                                             {
947       wchar_t *pwText = to_wide_string(yytext);
948       *yylval.comment += std::wstring(pwText);
949       FREE(pwText);
950   }
951
952  <<EOF>>                                        {
953       yy_pop_state();
954 //    std::string str = "unexpected end of file in a comment";
955 //    exit_status = SCAN_ERROR;
956 //    scan_error(str);
957   }
958 }
959
960
961 <SIMPLESTRING>
962 {
963   {dquote}{dquote}                              {
964     *pstBuffer += "\"";
965   }
966
967   {dquote}{quote}                               {
968     *pstBuffer += "'";
969   }
970
971   {quote}{dquote}                               {
972     *pstBuffer += "\"";
973   }
974
975   {quote}{quote}                                {
976     *pstBuffer += "'";
977   }
978
979   {quote}                                       {
980     yy_pop_state();
981     scan_step();
982     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
983     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
984     {
985         std::string str = "can not convert'";
986         str += pstBuffer->c_str();
987         str += "' to UTF-8";
988         exit_status = SCAN_ERROR;
989         scan_error("can not convert string to UTF-8");
990     }
991     yylval.str = new std::wstring(pwstBuffer);
992     delete pstBuffer;
993     FREE(pwstBuffer);
994     return scan_throw(STR);
995   }
996
997   {dquote}                  {
998     std::string str = "Heterogeneous string detected, starting with ' and ending with \".";
999     exit_status = SCAN_ERROR;
1000     scan_error(str);
1001   }
1002
1003   {next}{newline}           {
1004       /* Do nothing... Just skip */
1005   }
1006
1007   {newline}                                     {
1008     std::string str = "unexpected end of line in a string.";
1009     exit_status = SCAN_ERROR;
1010     scan_error(str);
1011     yylloc.last_line += 1;
1012     yylloc.last_column = 1;
1013   }
1014
1015   <<EOF>>                                       {
1016     std::string str = "unexpected end of file in a string.";
1017     exit_status = SCAN_ERROR;
1018     scan_error(str);
1019   }
1020
1021   {in_string}                                           |
1022   .                                                     {
1023     scan_step();
1024     *pstBuffer += yytext;
1025   }
1026 }
1027
1028
1029 <DOUBLESTRING>
1030 {
1031   {dquote}{dquote}                              {
1032     *pstBuffer += "\"";
1033   }
1034
1035   {dquote}{quote}                               {
1036     *pstBuffer += "'";
1037   }
1038
1039   {quote}{dquote}               {
1040     *pstBuffer += "\"";
1041   }
1042
1043   {quote}{quote}                                {
1044     *pstBuffer += "'";
1045   }
1046
1047   {dquote}                      {
1048     yy_pop_state();
1049     scan_step();
1050     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1051     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1052     {
1053         std::string str = "can not convert'";
1054         str += pstBuffer->c_str();
1055         str += "' to UTF-8";
1056         exit_status = SCAN_ERROR;
1057         scan_error("can not convert string to UTF-8");
1058     }
1059     yylval.str = new std::wstring(pwstBuffer);
1060     delete pstBuffer;
1061     FREE(pwstBuffer);
1062     return scan_throw(STR);
1063   }
1064
1065   {quote}                  {
1066     std::string str = "Heterogeneous string detected, starting with \" and ending with '.";
1067     exit_status = SCAN_ERROR;
1068     scan_error(str);
1069   }
1070
1071   {next}{newline}           {
1072       /* Do nothing... Just skip */
1073   }
1074
1075   {newline} {
1076     std::string str = "unexpected end of line in a string";
1077     exit_status = SCAN_ERROR;
1078     scan_error(str);
1079     yylloc.last_line += 1;
1080     yylloc.last_column = 1;
1081   }
1082
1083   <<EOF>>   {
1084     std::string str = "unexpected end of file in a string";
1085     exit_status = SCAN_ERROR;
1086     scan_error(str);
1087   }
1088
1089   {in_string}         |
1090   .                   {
1091    scan_step();
1092    *pstBuffer += yytext;
1093   }
1094 }
1095
1096
1097 <SHELLMODE>
1098 {
1099     {spaces}                    {
1100         if (last_token == ID)
1101         {
1102             scan_throw(SPACES);
1103             return ID;
1104         }
1105     }
1106
1107     {semicolon}                 {
1108         BEGIN(INITIAL);
1109         scan_step();
1110         return scan_throw(SEMI);
1111     }
1112
1113     {comma}                     {
1114         BEGIN(INITIAL);
1115         scan_step();
1116         return scan_throw(COMMA);
1117     }
1118
1119     {newline}                   {
1120         BEGIN(INITIAL);
1121         yylloc.last_line += 1;
1122         yylloc.last_column = 1;
1123         scan_step();
1124         return scan_throw(EOL);
1125     }
1126
1127     {assign} {
1128         if (last_token == STR)
1129         {
1130             wchar_t *pwText = to_wide_string(yytext);
1131             yylval.str = new std::wstring(pwText);
1132             FREE(pwText);
1133             return scan_throw(STR);
1134         }
1135         else
1136         {
1137             BEGIN(INITIAL);
1138             return scan_throw(ASSIGN);
1139         }
1140     }
1141
1142     {lparen} {
1143         if (last_token == STR)
1144         {
1145             wchar_t *pwText = to_wide_string(yytext);
1146             yylval.str = new std::wstring(pwText);
1147             FREE(pwText);
1148             return scan_throw(STR);
1149         }
1150         else
1151         {
1152             BEGIN(INITIAL);
1153             return scan_throw(LPAREN);
1154         }
1155     }
1156
1157     {lowerthan} {
1158         if (last_token == STR)
1159         {
1160             wchar_t *pwText = to_wide_string(yytext);
1161             yylval.str = new std::wstring(pwText);
1162             FREE(pwText);
1163             return scan_throw(STR);
1164         }
1165         else
1166         {
1167             BEGIN(INITIAL);
1168             return scan_throw(LT);
1169         }
1170     }
1171
1172     {greaterthan} {
1173         if (last_token == STR)
1174         {
1175             wchar_t *pwText = to_wide_string(yytext);
1176             yylval.str = new std::wstring(pwText);
1177             FREE(pwText);
1178             return scan_throw(STR);
1179         }
1180         else
1181         {
1182             BEGIN(INITIAL);
1183             return scan_throw(GT);
1184         }
1185     }
1186
1187     {boolnot} {
1188         if (last_token == STR)
1189         {
1190             wchar_t *pwText = to_wide_string(yytext);
1191             yylval.str = new std::wstring(pwText);
1192             FREE(pwText);
1193             return scan_throw(STR);
1194         }
1195         else
1196         {
1197             BEGIN(INITIAL);
1198             return scan_throw(NOT);
1199         }
1200     }
1201
1202
1203     [^ \t\v\f\r\n,;'"]+               {
1204         wchar_t *pwText = to_wide_string(yytext);
1205         yylval.str = new std::wstring(pwText);
1206         FREE(pwText);
1207         return scan_throw(STR);
1208     }
1209
1210     <<EOF>>                     {
1211         BEGIN(INITIAL);
1212     }
1213
1214 }
1215
1216 %%
1217
1218 int scan_throw(int token) {
1219   last_token = token;
1220 #ifdef DEV
1221   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1222 #endif
1223   return token;
1224 }
1225
1226 int get_last_token() {
1227     return last_token;
1228 }
1229
1230 void scan_step() {
1231   yylloc.first_line = yylloc.last_line;
1232   yylloc.first_column = yylloc.last_column;
1233 }
1234
1235 void scan_error(std::string msg)
1236 {
1237     wchar_t* pstMsg = to_wide_string(msg.c_str());
1238
1239     //std::wcerr << pstMsg << std::endl;
1240     ParserSingleInstance::PrintError(pstMsg);
1241     ParserSingleInstance::setExitStatus(Parser::Failed);
1242     ParserSingleInstance::resetControlStatus();
1243     FREE(pstMsg);
1244     last_token = YYEOF;
1245     BEGIN(INITIAL);
1246 }
1247
1248 /*
1249 ** convert floating numbers to C standard
1250 ** 1.2d-3 -> 1.2e-3
1251 ** 1.2D-3 -> 1.2e-3
1252 */
1253 void scan_exponent_convert(char *in)
1254 {
1255   char *pString;
1256   while((pString=strpbrk(in,"d"))!=NULL)
1257     {
1258       *pString='e';
1259     }
1260   while((pString=strpbrk(in,"D"))!=NULL)
1261     {
1262       *pString='e';
1263     }
1264 }
1265
1266 #ifdef _MSC_VER
1267 int isatty (int desc)
1268 {
1269   return 0;
1270 }
1271 #endif