Bug 14193 fixed: Problem with matrix defined in a matrix of structs
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13
14 #include <stack>
15
16 #include "isatty.hxx"
17 #include "parse.hxx"
18 #include "parser_private.hxx"
19
20 #include "context.hxx"
21
22 extern "C"
23 {
24 #include "charEncoding.h"
25 #include "sci_malloc.h"
26 }
27
28 static std::stack<int> paren_levels;
29
30 static int comment_level = 0;
31 static int last_token = 0;
32 static int exit_status = PARSE_ERROR;
33 static int str_opener_column = 0;
34 static std::string current_file;
35 static std::string program_name;
36
37 static std::string pstBuffer;
38
39 extern void yyerror(std::string);
40
41 #define YY_USER_ACTION                          \
42     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
43 //yylloc.last_column += yyleng;
44
45 /* -*- Verbose Special Debug -*- */
46 //#define DEV
47 //#define TOKENDEV
48
49 #ifdef DEV
50 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
51 #else
52 #define DEBUG(x) /* Nothing */
53 #endif
54
55 %}
56
57 %option stack
58 %option noyywrap
59
60 %x SIMPLESTRING
61 %x DOUBLESTRING
62 %x REGIONCOMMENT
63 %x LINECOMMENT
64 %x LINEBREAK
65
66 %x MATRIX
67
68 %x SHELLMODE
69 %x BEGINID
70
71 spaces                  [ \t\v\f]+
72 integer                 [0-9]+
73 number                  [0-9]+[\.][0-9]*
74 little                  \.[0-9]+
75 bom             \xEF\xBB\xBF
76
77 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
78 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
79
80 hex             [0]x[0-9a-fA-F]+
81 oct             [0]o[0-7]+
82
83
84 utf2            ([\xC2-\xDF][\x80-\xBF])
85 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
86 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
87 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
88 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
89 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
90 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
91 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
92
93 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
94 utf4            ({utf41}|{utf42}|{utf43})
95
96 utf             ({utf2}|{utf3}|{utf4})
97 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
98
99
100 newline                 ("\r"|"\n"|"\r\n")
101 blankline               {spaces}+{newline}
102 emptyline       {newline}({spaces}|[,;])+{newline}
103 next                    \.\.+
104 char_in_line_comment    [^\r\n]*
105 char_in_comment         [^\r\n\/*]*
106
107 boolnot                 ("@"|"~")
108 booltrue                ("%t"|"%T")
109 boolfalse               ("%f"|"%F")
110 booland                 ("&")
111 boolandand              ("&&")
112 boolor                  ("|")
113 booloror                ("||")
114
115 lbrack                  "["
116 rbrack                  "]"
117
118 lparen                  "("
119 rparen                  ")"
120
121 lbrace                  "{"
122 rbrace                  "}"
123
124 dollar                  "$"
125
126 semicolon               ";"
127 comma                   ","
128 colon                   ":"
129
130 startcomment            "//"
131 startblockcomment       "/*"
132 endblockcomment         "*/"
133
134 dquote                  "\""
135 quote                   "'"
136 in_string               [^\"\'\r\n\.]*
137
138 dot             "."
139 dotquote                ".'"
140 dottimes                ".*"
141 dotrdivide              "./"
142 dotldivide              ".\\"
143 dotpower                (".^"|".**")
144
145 plus                    "+"
146 minus                   "-"
147 rdivide                 "/"
148 ldivide                 "\\"
149 times                   "*"
150 power                   ("^"|"**")
151
152 equal                   "="{spaces}*"="
153 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
154 lowerthan               "<"
155 greaterthan             ">"
156 lowerequal              "<"{spaces}*"="
157 greaterequal            ">"{spaces}*"="
158
159 krontimes               ".*."
160 kronrdivide             "./."
161 kronldivide             ".\\."
162
163 controltimes    ("*."[^0-9])
164 controlrdivide  ("/."[^0-9])
165 controlldivide  ("\\."[^0-9])
166
167 assign                  "="
168
169 %%
170 <INITIAL>{bom}/.* {
171 // BOM found ==> ignored
172 }
173
174 <INITIAL,BEGINID>{booltrue}     {
175     BEGIN(INITIAL);
176     return scan_throw(BOOLTRUE);
177 }
178 <INITIAL,BEGINID>{boolfalse}    {
179     BEGIN(INITIAL);
180     return scan_throw(BOOLFALSE);
181 }
182
183 <INITIAL,BEGINID>"if"            {
184         if (last_token != DOT)
185     {
186         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
187     }
188     DEBUG("BEGIN(INITIAL)");
189     BEGIN(INITIAL);
190     return scan_throw(IF);
191 }
192
193 <INITIAL,BEGINID>"then"          {
194     DEBUG("BEGIN(INITIAL)");
195     BEGIN(INITIAL);
196     return scan_throw(THEN);
197 }
198
199 <INITIAL,BEGINID>"else"          {
200         if (last_token != DOT)
201     {
202         // Pop to step out IF
203         ParserSingleInstance::popControlStatus();
204         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
205     }
206     DEBUG("BEGIN(INITIAL)");
207     BEGIN(INITIAL);
208         return scan_throw(ELSE);
209 }
210
211 <INITIAL,BEGINID>"elseif" {
212         if (last_token != DOT)
213     {
214         ParserSingleInstance::popControlStatus();
215         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
216     }
217     DEBUG("BEGIN(INITIAL)");
218     BEGIN(INITIAL);
219         return scan_throw(ELSEIF);
220 }
221
222 <INITIAL,BEGINID>"end"          {
223         if (last_token != DOT)
224     {
225         ParserSingleInstance::popControlStatus();
226     }
227     DEBUG("BEGIN(INITIAL)");
228     BEGIN(INITIAL);
229     return scan_throw(END);
230 }
231
232 <INITIAL,BEGINID>"select"       {
233         if (last_token != DOT)
234     {
235         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
236     }
237     DEBUG("BEGIN(INITIAL)");
238     BEGIN(INITIAL);
239     return scan_throw(SELECT);
240 }
241
242 <INITIAL,BEGINID>"switch"       {
243         if (last_token != DOT)
244     {
245         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
246     }
247     DEBUG("BEGIN(INITIAL)");
248     BEGIN(INITIAL);
249     return scan_throw(SWITCH);
250 }
251
252 <INITIAL,BEGINID>"otherwise" {
253         if (last_token != DOT)
254     {
255         ParserSingleInstance::popControlStatus();
256         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
257     }
258     DEBUG("BEGIN(INITIAL)");
259     BEGIN(INITIAL);
260         return scan_throw(OTHERWISE);
261 }
262
263 <INITIAL,BEGINID>"case"         {
264         if (last_token != DOT)
265     {
266         ParserSingleInstance::popControlStatus();
267         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
268     }
269     DEBUG("BEGIN(INITIAL)");
270     BEGIN(INITIAL);
271     return scan_throw(CASE);
272 }
273
274 <INITIAL,BEGINID>"function" {
275         if (last_token != DOT)
276     {
277         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
278     }
279     DEBUG("BEGIN(INITIAL)");
280     BEGIN(INITIAL);
281     return scan_throw(FUNCTION);
282 }
283
284 <INITIAL,BEGINID>"endfunction" {
285         if (last_token != DOT)
286     {
287         ParserSingleInstance::popControlStatus();
288     }
289     DEBUG("BEGIN(INITIAL)");
290     BEGIN(INITIAL);
291         return scan_throw(ENDFUNCTION);
292 }
293
294 <INITIAL,BEGINID>"for" {
295         if (last_token != DOT)
296     {
297         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
298     }
299     BEGIN(INITIAL);
300     return scan_throw(FOR);
301 }
302
303 <INITIAL,BEGINID>"while"        {
304         if (last_token != DOT)
305     {
306         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
307     }
308         BEGIN(INITIAL);
309         return scan_throw(WHILE);
310 }
311
312 <INITIAL,BEGINID>"do"           {
313         BEGIN(INITIAL);
314     return scan_throw(DO);
315 }
316
317 <INITIAL,BEGINID>"break"                {
318         BEGIN(INITIAL);
319         return scan_throw(BREAK);
320 }
321
322 <INITIAL,BEGINID>"continue"             {
323         BEGIN(INITIAL);
324         return scan_throw(CONTINUE);
325 }
326
327 <INITIAL,BEGINID>"try" {
328         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
329         BEGIN(INITIAL);
330         return scan_throw(TRY);
331 }
332
333 <INITIAL,BEGINID>"catch" {
334     // Pop to step out TRY
335         ParserSingleInstance::popControlStatus();
336         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
337         BEGIN(INITIAL);
338         return scan_throw(CATCH);
339 }
340
341 <INITIAL,BEGINID>"return"       {
342     BEGIN(INITIAL);
343     return scan_throw(RETURN);
344 }
345
346 <INITIAL,BEGINID>"resume"       {
347     BEGIN(INITIAL);
348     return scan_throw(RETURN);
349 }
350
351 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
352         BEGIN(BEGINID);
353 }
354
355 <BEGINID>
356 {
357     {id}                        {
358         wchar_t *pwText = to_wide_string(yytext);
359         if (yytext != NULL && pwText == NULL)
360         {
361             std::string str = "Can\'t convert \'";
362             str += yytext;
363             str += "\' to UTF-8";
364             BEGIN(INITIAL);
365             yyerror(str);
366             return scan_throw(FLEX_ERROR);
367         }
368         yylval.str = new std::wstring(pwText);
369         FREE(pwText);
370         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
371         if (pIT && pIT->isCallable())
372         {
373             scan_throw(ID);
374             BEGIN(SHELLMODE);
375         }
376         else
377         {
378             BEGIN(INITIAL);
379             return scan_throw(ID);
380         }
381     }
382
383 }
384
385 <INITIAL,MATRIX>{boolnot}               {
386   return scan_throw(NOT);
387 }
388 <INITIAL,MATRIX>{dollar}                {
389   return scan_throw(DOLLAR);
390 }
391 <INITIAL,MATRIX>{booltrue}              {
392   return scan_throw(BOOLTRUE);
393 }
394 <INITIAL,MATRIX>{boolfalse}             {
395   return scan_throw(BOOLFALSE);
396 }
397 <INITIAL,MATRIX>{booland}               {
398   return scan_throw(AND);
399 }
400 <INITIAL,MATRIX>{boolandand}    {
401   return scan_throw(ANDAND);
402 }
403 <INITIAL,MATRIX>{boolor}                {
404   return scan_throw(OR);
405 }
406 <INITIAL,MATRIX>{booloror}              {
407   return scan_throw(OROR);
408 }
409
410
411 <INITIAL>{lparen}                       {
412   return scan_throw(LPAREN);
413 }
414 <INITIAL>{rparen}                       {
415   return scan_throw(RPAREN);
416 }
417
418
419 <INITIAL,MATRIX>{semicolon}             {
420         scan_step();
421   return scan_throw(SEMI);
422 }
423
424 <INITIAL,MATRIX>{comma}                 {
425         scan_step();
426   return scan_throw(COMMA);
427 }
428
429 <INITIAL,MATRIX>{colon}                 {
430   return scan_throw(COLON);
431 }
432
433
434 <INITIAL,MATRIX>{lbrace}                {
435   yy_push_state(MATRIX);
436   paren_levels.push(0);
437   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
438   return scan_throw(LBRACE);
439 }
440
441 {rbrace}                        {
442   return scan_throw(RBRACE);
443 }
444
445
446 <INITIAL,MATRIX>{dotquote}              {
447   return scan_throw(DOTQUOTE);
448 }
449 <INITIAL,MATRIX>{dottimes}              {
450   return scan_throw(DOTTIMES);
451 }
452 <INITIAL,MATRIX>{dotrdivide}            {
453   return scan_throw(DOTRDIVIDE);
454 }
455 <INITIAL,MATRIX>{dotldivide}            {
456   return scan_throw(DOTLDIVIDE);
457 }
458 <INITIAL,MATRIX>{dotpower}              {
459   return scan_throw(DOTPOWER);
460 }
461
462
463 {minus}                                 {
464   return scan_throw(MINUS);
465 }
466 {plus}                                  {
467   return scan_throw(PLUS);
468 }
469 <INITIAL,MATRIX>{times}                 {
470   return scan_throw(TIMES);
471 }
472 <INITIAL,MATRIX>{rdivide}               {
473   return scan_throw(RDIVIDE);
474 }
475 <INITIAL,MATRIX>{ldivide}               {
476   return scan_throw(LDIVIDE);
477 }
478 <INITIAL,MATRIX>{power}                 {
479   return scan_throw(POWER);
480 }
481
482 <INITIAL,MATRIX>{krontimes}             {
483   return scan_throw(KRONTIMES);
484 }
485 <INITIAL,MATRIX>{kronrdivide}           {
486   return scan_throw(KRONRDIVIDE);
487 }
488 <INITIAL,MATRIX>{kronldivide}           {
489   return scan_throw(KRONLDIVIDE);
490 }
491
492
493 <INITIAL,MATRIX>{controltimes}          {
494     unput(yytext[yyleng - 1]);
495     return scan_throw(CONTROLTIMES);
496 }
497 <INITIAL,MATRIX>{controlrdivide}                {
498     unput(yytext[yyleng - 1]);
499     return scan_throw(CONTROLRDIVIDE);
500 }
501 <INITIAL,MATRIX>{controlldivide}                {
502     unput(yytext[yyleng - 1]);
503     return scan_throw(CONTROLLDIVIDE);
504 }
505
506
507 <INITIAL,MATRIX>{equal}                 {
508   return scan_throw(EQ);
509 }
510 <INITIAL,MATRIX>{notequal}              {
511   return scan_throw(NE);
512 }
513 <INITIAL,MATRIX>{lowerthan}             {
514   return scan_throw(LT);
515 }
516 <INITIAL,MATRIX>{greaterthan}           {
517   return scan_throw(GT);
518 }
519 <INITIAL,MATRIX>{lowerequal}            {
520   return scan_throw(LE);
521 }
522 <INITIAL,MATRIX>{greaterequal}          {
523   return scan_throw(GE);
524 }
525
526
527 <INITIAL,MATRIX>{assign}                {
528   return scan_throw(ASSIGN);
529  }
530
531
532 <INITIAL,MATRIX>{lbrack}                {
533   DEBUG("yy_push_state(MATRIX)");
534   yy_push_state(MATRIX);
535   paren_levels.push(0);
536   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
537   return scan_throw(LBRACK);
538 }
539
540 <INITIAL>{rbrack}                               {
541   return scan_throw(RBRACK);
542 }
543
544
545 <INITIAL,MATRIX>{dot}                   {
546   return scan_throw(DOT);
547 }
548
549 <INITIAL>{next}                 {
550     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
551     yy_push_state(LINEBREAK);
552 }
553
554 <INITIAL,MATRIX>{integer}               {
555   yylval.number = atof(yytext);
556 #ifdef TOKENDEV
557   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
558 #endif
559 //  scan_step();
560   return scan_throw(VARINT);
561 }
562
563
564 <INITIAL,MATRIX>{floating_D}            {
565   scan_exponent_convert(yytext);
566   yylval.number = atof(yytext);
567 #ifdef TOKENDEV
568   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
569 #endif
570   //scan_step();
571   return scan_throw(VARFLOAT);
572 }
573
574 <INITIAL,MATRIX>{floating_E}            {
575   yylval.number = atof(yytext);
576 #ifdef TOKENDEV
577   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
578 #endif
579   //scan_step();
580   return scan_throw(VARFLOAT);
581 }
582
583
584 <INITIAL,MATRIX>{number}                {
585   yylval.number = atof(yytext);
586 #ifdef TOKENDEV
587   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
588 #endif
589 //  scan_step();
590   return scan_throw(NUM);
591 }
592
593
594 <INITIAL,MATRIX>{little}                {
595   yylval.number = atof(yytext);
596 #ifdef TOKENDEV
597   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
598 #endif
599 //  scan_step();
600   return scan_throw(NUM);
601 }
602
603
604 <INITIAL,MATRIX>{id}                    {
605     wchar_t *pwText = to_wide_string(yytext);
606     if (yytext != NULL && pwText == NULL)
607     {
608         std::string str = "Can\'t convert \'";
609         str += yytext;
610         str += "\' to UTF-8";
611         BEGIN(INITIAL);
612         yyerror(str);
613         return scan_throw(FLEX_ERROR);
614     }
615     yylval.str = new std::wstring(pwText);
616     FREE(pwText);
617 #ifdef TOKENDEV
618   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
619 #endif
620 //  scan_step();
621   return scan_throw(ID);
622 }
623
624
625 <INITIAL,MATRIX>{startblockcomment}     {
626   yylval.comment = new std::wstring();
627   comment_level = 1;
628   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
629   yy_push_state(REGIONCOMMENT);
630 }
631
632
633 <INITIAL,MATRIX>{startcomment}          {
634   pstBuffer.clear();
635   yy_push_state(LINECOMMENT);
636 }
637
638
639 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
640   pstBuffer.clear();
641   str_opener_column = yylloc.first_column;
642   yy_push_state(DOUBLESTRING);
643 }
644
645
646 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
647   /*
648   ** Matrix Transposition special behaviour
649   ** ID' []' toto()' are transposition call
650   */
651   if (last_token == ID
652       || last_token == RBRACK
653       || last_token == RPAREN
654       || last_token == RBRACE
655       || last_token == VARINT
656       || last_token == VARFLOAT
657       || last_token == NUM
658       || last_token == BOOLTRUE
659       || last_token == BOOLFALSE)
660   {
661       return scan_throw(QUOTE);
662   }
663   else
664   {
665       pstBuffer.clear();
666       str_opener_column = yylloc.first_column;
667       yy_push_state(SIMPLESTRING);
668   }
669 }
670
671
672 <INITIAL,MATRIX>{spaces}                {
673   scan_step();
674   scan_throw(SPACES);
675 }
676
677
678 <INITIAL>{newline}              {
679   yylloc.last_line += 1;
680   yylloc.last_column = 1;
681   scan_step();
682   if (last_token != EOL) {
683       return scan_throw(EOL);
684   }
685
686 }
687
688
689 <INITIAL,MATRIX>{blankline}             {
690   yylloc.last_line += 1;
691   yylloc.last_column = 1;
692   scan_step();
693   if (last_token != EOL)
694   {
695       return scan_throw(EOL);
696   }
697   scan_throw(EOL);
698 }
699
700 <INITIAL,MATRIX>{emptyline}             {
701   yylloc.last_line += 2;
702   yylloc.last_column = 1;
703   scan_step();
704   if (last_token != EOL)
705   {
706       return scan_throw(EOL);
707   }
708   scan_throw(EOL);
709 }
710 .                                       {
711     std::string str = "Unexpected token \'";
712     str += yytext;
713     str += "\'";
714     BEGIN(INITIAL);
715     yyerror(str);
716     return scan_throw(FLEX_ERROR);
717 }
718
719
720 <MATRIX>
721 {
722   {lparen} {
723     ++paren_levels.top();
724     return scan_throw(LPAREN);
725   }
726
727   {rparen} {
728     --paren_levels.top();
729     return scan_throw(RPAREN);
730   }
731
732   {spaces}*{lparen} {
733       unput(yytext[yyleng -1]);
734       if (last_token == ID
735           || last_token == RPAREN
736           || last_token == QUOTE
737           || last_token == VARINT
738           || last_token == VARFLOAT
739           || last_token == NUM)
740       {
741           return scan_throw(COMMA);
742       }
743   }
744
745   {spaces}*{colon}{spaces}* {
746       return scan_throw(COLON);
747   }
748
749   {newline} {
750       yylloc.last_line += 1;
751       yylloc.last_column = 1;
752       if(last_token != DOTS && last_token != EOL)
753       {
754           return scan_throw(EOL);
755       }
756       scan_throw(EOL);
757   }
758
759
760   {rbrack}                              {
761     DEBUG("yy_pop_state()");
762     yy_pop_state();
763     paren_levels.pop();
764     ParserSingleInstance::popControlStatus();
765     return scan_throw(RBRACK);
766   }
767
768   {rbrace}                              {
769     yy_pop_state();
770     paren_levels.pop();
771     ParserSingleInstance::popControlStatus();
772     return scan_throw(RBRACE);
773   }
774
775   {plus}                                |
776   {spaces}{plus}{spaces}                {
777     return scan_throw(PLUS);
778   }
779
780   {minus}                               |
781   {spaces}{minus}{spaces}               {
782     return scan_throw(MINUS);
783   }
784
785   {spaces}{plus}                        {
786     // no need to unput the '+'
787     if (last_token != LBRACK
788        && last_token != EOL
789        && last_token != SEMI
790        && last_token != COMMA
791         && paren_levels.top() == 0)
792    {
793        return scan_throw(COMMA);
794    }
795    else
796    {
797        unput('+');
798    }
799   }
800
801   {spaces}{minus}                       {
802     unput('-');
803     if (last_token != LBRACK
804        && last_token != EOL
805        && last_token != SEMI
806        && last_token != COMMA
807         && paren_levels.top() == 0)
808    {
809        return scan_throw(COMMA);
810    }
811   }
812
813   .                                     {
814     std::string str = "Unexpected token \'";
815     str += yytext;
816     str += "\' within a matrix.";
817     BEGIN(INITIAL);
818     yyerror(str);
819     return scan_throw(FLEX_ERROR);
820   }
821
822   {next}{spaces}*{newline}          {
823       /* Just do nothing */
824       yylloc.last_line += 1;
825       yylloc.last_column = 1;
826       scan_step();
827       scan_throw(EOL);
828   }
829
830   {next}{spaces}*{startcomment}          {
831       /* Just do nothing */
832       pstBuffer.clear();
833       yy_push_state(LINECOMMENT);
834       scan_throw(DOTS);
835   }
836
837   <<EOF>>       {
838       yy_pop_state();
839       paren_levels.pop();
840   }
841 }
842
843 <LINEBREAK>
844 {
845   {newline}                             {
846     yylloc.last_line += 1;
847     yylloc.last_column = 1;
848     scan_step();
849     yy_pop_state();
850     ParserSingleInstance::popControlStatus();
851   }
852
853   {startblockcomment}                   {
854     ++comment_level;
855     yy_push_state(REGIONCOMMENT);
856   }
857
858   {startcomment}                        {
859     scan_throw(DOTS);
860     pstBuffer.clear();
861     yy_push_state(LINECOMMENT);
862   }
863
864   {spaces}                              {
865       /* Do nothing... */
866   }
867
868   <<EOF>>       {
869       yy_pop_state();
870   }
871   .                                     {
872     std::string str = "Unexpected token \'";
873     str += yytext;
874     str += "\' after line break with .. or ...";
875     BEGIN(INITIAL);
876     yyerror(str);
877     return scan_throw(FLEX_ERROR);
878   }
879 }
880
881
882 <LINECOMMENT>
883 {
884   {newline}     {
885     //yylloc.last_line += 1;
886     //yylloc.last_column = 1;
887     //scan_step();
888     yy_pop_state();
889     for (int i = yyleng - 1 ; i >= 0 ; --i)
890     {
891         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
892         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
893         unput(yytext[i]);
894         yylloc.last_column--;
895     }
896     /*
897     ** To forgot comments after lines break
898     */
899     if (last_token != DOTS)
900     {
901         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
902         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
903         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
904         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
905         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
906         {
907             pstBuffer.clear();
908             std::string str = "Can\'t convert \'";
909             str += pstBuffer.c_str();
910             str += "\' to UTF-8";
911             BEGIN(INITIAL);
912             yyerror(str);
913             return scan_throw(FLEX_ERROR);
914         }
915         yylval.comment = new std::wstring(pwstBuffer);
916         pstBuffer.clear();
917         FREE (pwstBuffer);
918         return scan_throw(COMMENT);
919     }
920     else
921     {
922         pstBuffer.clear();
923     }
924   }
925
926   <<EOF>>       {
927     yy_pop_state();
928     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
929     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
930     {
931         pstBuffer.clear();
932         std::string str = "Can\'t convert \'";
933         str += pstBuffer.c_str();
934         str += "\' to UTF-8";
935         BEGIN(INITIAL);
936         yyerror(str);
937         return scan_throw(FLEX_ERROR);
938     }
939     yylval.comment = new std::wstring(pwstBuffer);
940     pstBuffer.clear();
941     FREE (pwstBuffer);
942     return scan_throw(COMMENT);
943   }
944
945   {char_in_line_comment}         {
946       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
947       // only translate to WCHAR_T when popping state.
948       pstBuffer += yytext;
949   }
950 }
951
952
953 <REGIONCOMMENT>
954 {
955   {endblockcomment}                             {
956     --comment_level;
957     if (comment_level == 0) {
958       ParserSingleInstance::popControlStatus();
959       yy_pop_state();
960       //return scan_throw(BLOCKCOMMENT);
961     }
962   }
963
964   {startblockcomment}                           {
965     ++comment_level;
966     yy_push_state(REGIONCOMMENT);
967   }
968
969   {newline}                                     {
970     yylloc.last_line += 1;
971     yylloc.last_column = 1;
972     scan_step();
973     *yylval.comment += L"\n//";
974   }
975
976   {char_in_comment}                             |
977   .                                             {
978       wchar_t *pwText = to_wide_string(yytext);
979       *yylval.comment += std::wstring(pwText);
980       FREE(pwText);
981   }
982
983  <<EOF>>                                        {
984       yy_pop_state();
985 //    std::string str = "unexpected end of file in a comment";
986 //    scan_error(str);
987   }
988 }
989
990
991 <SIMPLESTRING>
992 {
993   {dquote}{dquote}                              {
994     pstBuffer += "\"";
995   }
996
997   {dquote}{quote}                               {
998     pstBuffer += "'";
999   }
1000
1001   {quote}{dquote}                               {
1002     pstBuffer += "\"";
1003   }
1004
1005   {quote}{quote}                                {
1006     pstBuffer += "'";
1007   }
1008
1009   {quote}                                       {
1010     yy_pop_state();
1011     //scan_step();
1012     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1013     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1014     {
1015         pstBuffer.clear();
1016         std::string str = "Can\'t convert \'";
1017         str += pstBuffer.c_str();
1018         str += "\' to UTF-8";
1019         BEGIN(INITIAL);
1020         yyerror(str);
1021         return scan_throw(FLEX_ERROR);
1022     }
1023     yylval.str = new std::wstring(pwstBuffer);
1024     pstBuffer.clear();
1025     FREE(pwstBuffer);
1026     yylloc.first_column = str_opener_column;
1027     return scan_throw(STR);
1028   }
1029
1030   {dquote}                  {
1031     pstBuffer.clear();
1032     BEGIN(INITIAL);
1033     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1034     return scan_throw(FLEX_ERROR);
1035   }
1036
1037   {next}{newline}           {
1038       yylloc.last_line += 1;
1039       yylloc.last_column = 1;
1040       /* Do nothing... Just skip */
1041   }
1042
1043   {newline}                                     {
1044     pstBuffer.clear();
1045     yylloc.last_line += 1;
1046     yylloc.last_column = 1;
1047     BEGIN(INITIAL);
1048     yyerror("Unexpected end of line in a string.");
1049     return scan_throw(FLEX_ERROR);
1050   }
1051
1052   <<EOF>>                                       {
1053     pstBuffer.clear();
1054     BEGIN(INITIAL);
1055     yyerror("Unexpected end of file in a string.");
1056     return scan_throw(FLEX_ERROR);    
1057   }
1058
1059   {in_string}                                           |
1060   .                                                     {
1061     //scan_step();
1062     pstBuffer += yytext;
1063   }
1064 }
1065
1066
1067 <DOUBLESTRING>
1068 {
1069   {dquote}{dquote}                              {
1070     pstBuffer += "\"";
1071   }
1072
1073   {dquote}{quote}                               {
1074     pstBuffer += "'";
1075   }
1076
1077   {quote}{dquote}               {
1078     pstBuffer += "\"";
1079   }
1080
1081   {quote}{quote}                                {
1082     pstBuffer += "'";
1083   }
1084
1085   {dquote}                      {
1086     yy_pop_state();
1087     //scan_step();
1088     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1089     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1090     {
1091         pstBuffer.clear();
1092         std::string str = "Can\'t convert \'";
1093         str += pstBuffer.c_str();
1094         str += "\' to UTF-8";
1095         BEGIN(INITIAL);
1096         yyerror(str);
1097         return scan_throw(FLEX_ERROR);
1098     }
1099     yylval.str = new std::wstring(pwstBuffer);
1100     pstBuffer.clear();
1101     FREE(pwstBuffer);
1102     yylloc.first_column = str_opener_column;
1103     return scan_throw(STR);
1104   }
1105
1106   {quote}                  {
1107     pstBuffer.clear();
1108     BEGIN(INITIAL);
1109     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1110     return scan_throw(FLEX_ERROR);
1111   }
1112
1113   {next}{newline}           {
1114       yylloc.last_line += 1;
1115       yylloc.last_column = 1;
1116       /* Do nothing... Just skip */
1117   }
1118
1119   {newline} {
1120     pstBuffer.clear();
1121     yylloc.last_line += 1;
1122     yylloc.last_column = 1;
1123     BEGIN(INITIAL);
1124     yyerror("Unexpected end of line in a string.");
1125     return scan_throw(FLEX_ERROR);
1126   }
1127
1128   <<EOF>>   {
1129     pstBuffer.clear();
1130     BEGIN(INITIAL);
1131     yyerror("Unexpected end of file in a string.");
1132     return scan_throw(FLEX_ERROR);    
1133   }
1134
1135   {in_string}         |
1136   .                   {
1137    //scan_step();
1138    pstBuffer += yytext;
1139   }
1140 }
1141
1142
1143 <SHELLMODE>
1144 {
1145     {spaces}                    {
1146         if (last_token == ID)
1147         {
1148             scan_throw(SPACES);
1149             return ID;
1150         }
1151     }
1152
1153     {semicolon}                 {
1154         BEGIN(INITIAL);
1155         scan_step();
1156         return scan_throw(SEMI);
1157     }
1158
1159     {comma}                     {
1160         BEGIN(INITIAL);
1161         scan_step();
1162         return scan_throw(COMMA);
1163     }
1164
1165     {newline}                   {
1166         BEGIN(INITIAL);
1167         yylloc.last_line += 1;
1168         yylloc.last_column = 1;
1169         scan_step();
1170         return scan_throw(EOL);
1171     }
1172
1173     {assign} {
1174         if (last_token == STR || last_token == SPACES)
1175         {
1176             wchar_t *pwText = to_wide_string(yytext);
1177             yylval.str = new std::wstring(pwText);
1178             FREE(pwText);
1179             return scan_throw(STR);
1180         }
1181         else
1182         {
1183             BEGIN(INITIAL);
1184             return scan_throw(ASSIGN);
1185         }
1186     }
1187
1188     {lparen} {
1189         if (last_token == STR || last_token == SPACES)
1190         {
1191             wchar_t *pwText = to_wide_string(yytext);
1192             yylval.str = new std::wstring(pwText);
1193             FREE(pwText);
1194             return scan_throw(STR);
1195         }
1196         else
1197         {
1198             BEGIN(INITIAL);
1199             return scan_throw(LPAREN);
1200         }
1201     }
1202
1203     {lowerthan} {
1204         if (last_token == STR || last_token == SPACES)
1205         {
1206             wchar_t *pwText = to_wide_string(yytext);
1207             yylval.str = new std::wstring(pwText);
1208             FREE(pwText);
1209             return scan_throw(STR);
1210         }
1211         else
1212         {
1213             BEGIN(INITIAL);
1214             return scan_throw(LT);
1215         }
1216     }
1217
1218     {greaterthan} {
1219         if (last_token == STR || last_token == SPACES)
1220         {
1221             wchar_t *pwText = to_wide_string(yytext);
1222             yylval.str = new std::wstring(pwText);
1223             FREE(pwText);
1224             return scan_throw(STR);
1225         }
1226         else
1227         {
1228             BEGIN(INITIAL);
1229             return scan_throw(GT);
1230         }
1231     }
1232
1233     {boolnot} {
1234         if (last_token == STR || last_token == SPACES)
1235         {
1236             wchar_t *pwText = to_wide_string(yytext);
1237             yylval.str = new std::wstring(pwText);
1238             FREE(pwText);
1239             return scan_throw(STR);
1240         }
1241         else
1242         {
1243             BEGIN(INITIAL);
1244             return scan_throw(NOT);
1245         }
1246     }
1247
1248
1249     [^ \t\v\f\r\n,;'"]+               {
1250         wchar_t *pwText = to_wide_string(yytext);
1251         yylval.str = new std::wstring(pwText);
1252         FREE(pwText);
1253         return scan_throw(STR);
1254     }
1255
1256     <<EOF>>                     {
1257         BEGIN(INITIAL);
1258     }
1259
1260 }
1261
1262 %%
1263
1264 int scan_throw(int token) {
1265   last_token = token;
1266 #ifdef DEV
1267   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1268 #endif
1269   return token;
1270 }
1271
1272 int get_last_token() {
1273     return last_token;
1274 }
1275
1276 void scan_step() {
1277   yylloc.first_line = yylloc.last_line;
1278   yylloc.first_column = yylloc.last_column;
1279 }
1280
1281 /*
1282 ** convert floating numbers to C standard
1283 ** 1.2d-3 -> 1.2e-3
1284 ** 1.2D-3 -> 1.2e-3
1285 */
1286 void scan_exponent_convert(char *in)
1287 {
1288   for (; *in != 'd' && *in != 'D'; ++in);
1289   *in = 'e';
1290 }
1291
1292 #ifdef _MSC_VER
1293 int isatty (int desc)
1294 {
1295   return 0;
1296 }
1297 #endif