0a12479d26d089fbaf440840939811a4c19d77eb
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "sci_malloc.h"
23 }
24
25 static int comment_level = 0;
26 static int paren_level = 0;
27 static int last_token = 0;
28 static int exit_status = PARSE_ERROR;
29 static int str_opener_column = 0;
30 static std::string current_file;
31 static std::string program_name;
32
33 static std::string pstBuffer;
34
35 extern void yyerror(std::string);
36
37 #define YY_USER_ACTION                          \
38     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
39 //yylloc.last_column += yyleng;
40
41 /* -*- Verbose Special Debug -*- */
42 //#define DEV
43 //#define TOKENDEV
44
45 #ifdef DEV
46 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
47 #else
48 #define DEBUG(x) /* Nothing */
49 #endif
50
51 %}
52
53 %option stack
54 %option noyywrap
55
56 %x SIMPLESTRING
57 %x DOUBLESTRING
58 %x REGIONCOMMENT
59 %x LINECOMMENT
60 %x LINEBREAK
61
62 %x MATRIX
63
64 %x SHELLMODE
65 %x BEGINID
66
67 spaces                  [ \t\v\f]+
68 integer                 [0-9]+
69 number                  [0-9]+[\.][0-9]*
70 little                  \.[0-9]+
71 bom             \xEF\xBB\xBF
72
73 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
74 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
75
76 hex             [0]x[0-9a-fA-F]+
77 oct             [0]o[0-7]+
78
79
80 utf2            ([\xC2-\xDF][\x80-\xBF])
81 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
82 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
83 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
84 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
85 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
86 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
87 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
88
89 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
90 utf4            ({utf41}|{utf42}|{utf43})
91
92 utf             ({utf2}|{utf3}|{utf4})
93 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
94
95
96 newline                 ("\r"|"\n"|"\r\n")
97 blankline               {spaces}+{newline}
98 emptyline       {newline}({spaces}|[,;])+{newline}
99 next                    \.\.+
100 char_in_line_comment    [^\r\n]*
101 char_in_comment         [^\r\n\/*]*
102
103 boolnot                 ("@"|"~")
104 booltrue                ("%t"|"%T")
105 boolfalse               ("%f"|"%F")
106 booland                 ("&")
107 boolandand              ("&&")
108 boolor                  ("|")
109 booloror                ("||")
110
111 lbrack                  "["
112 rbrack                  "]"
113
114 lparen                  "("
115 rparen                  ")"
116
117 lbrace                  "{"
118 rbrace                  "}"
119
120 dollar                  "$"
121
122 semicolon               ";"
123 comma                   ","
124 colon                   ":"
125
126 startcomment            "//"
127 startblockcomment       "/*"
128 endblockcomment         "*/"
129
130 dquote                  "\""
131 quote                   "'"
132 in_string               [^\"\'\r\n\.]*
133
134 dot             "."
135 dotquote                ".'"
136 dottimes                ".*"
137 dotrdivide              "./"
138 dotldivide              ".\\"
139 dotpower                (".^"|".**")
140
141 plus                    "+"
142 minus                   "-"
143 rdivide                 "/"
144 ldivide                 "\\"
145 times                   "*"
146 power                   ("^"|"**")
147
148 equal                   "="{spaces}*"="
149 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
150 lowerthan               "<"
151 greaterthan             ">"
152 lowerequal              "<"{spaces}*"="
153 greaterequal            ">"{spaces}*"="
154
155 krontimes               ".*."
156 kronrdivide             "./."
157 kronldivide             ".\\."
158
159 controltimes    ("*."[^0-9])
160 controlrdivide  ("/."[^0-9])
161 controlldivide  ("\\."[^0-9])
162
163 assign                  "="
164
165 %%
166 <INITIAL>{bom}/.* {
167 // BOM found ==> ignored
168 }
169
170 <INITIAL,BEGINID>{booltrue}     {
171     BEGIN(INITIAL);
172     return scan_throw(BOOLTRUE);
173 }
174 <INITIAL,BEGINID>{boolfalse}    {
175     BEGIN(INITIAL);
176     return scan_throw(BOOLFALSE);
177 }
178
179 <INITIAL,BEGINID>"if"            {
180         if (last_token != DOT)
181     {
182         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
183     }
184     DEBUG("BEGIN(INITIAL)");
185     BEGIN(INITIAL);
186     return scan_throw(IF);
187 }
188
189 <INITIAL,BEGINID>"then"          {
190     DEBUG("BEGIN(INITIAL)");
191     BEGIN(INITIAL);
192     return scan_throw(THEN);
193 }
194
195 <INITIAL,BEGINID>"else"          {
196         if (last_token != DOT)
197     {
198         // Pop to step out IF
199         ParserSingleInstance::popControlStatus();
200         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
201     }
202     DEBUG("BEGIN(INITIAL)");
203     BEGIN(INITIAL);
204         return scan_throw(ELSE);
205 }
206
207 <INITIAL,BEGINID>"elseif" {
208         if (last_token != DOT)
209     {
210         ParserSingleInstance::popControlStatus();
211         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
212     }
213     DEBUG("BEGIN(INITIAL)");
214     BEGIN(INITIAL);
215         return scan_throw(ELSEIF);
216 }
217
218 <INITIAL,BEGINID>"end"          {
219         if (last_token != DOT)
220     {
221         ParserSingleInstance::popControlStatus();
222     }
223     DEBUG("BEGIN(INITIAL)");
224     BEGIN(INITIAL);
225     return scan_throw(END);
226 }
227
228 <INITIAL,BEGINID>"select"       {
229         if (last_token != DOT)
230     {
231         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
232     }
233     DEBUG("BEGIN(INITIAL)");
234     BEGIN(INITIAL);
235     return scan_throw(SELECT);
236 }
237
238 <INITIAL,BEGINID>"switch"       {
239         if (last_token != DOT)
240     {
241         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
242     }
243     DEBUG("BEGIN(INITIAL)");
244     BEGIN(INITIAL);
245     return scan_throw(SWITCH);
246 }
247
248 <INITIAL,BEGINID>"otherwise" {
249         if (last_token != DOT)
250     {
251         ParserSingleInstance::popControlStatus();
252         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
253     }
254     DEBUG("BEGIN(INITIAL)");
255     BEGIN(INITIAL);
256         return scan_throw(OTHERWISE);
257 }
258
259 <INITIAL,BEGINID>"case"         {
260         if (last_token != DOT)
261     {
262         ParserSingleInstance::popControlStatus();
263         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
264     }
265     DEBUG("BEGIN(INITIAL)");
266     BEGIN(INITIAL);
267     return scan_throw(CASE);
268 }
269
270 <INITIAL,BEGINID>"function" {
271         if (last_token != DOT)
272     {
273         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
274     }
275     DEBUG("BEGIN(INITIAL)");
276     BEGIN(INITIAL);
277     return scan_throw(FUNCTION);
278 }
279
280 <INITIAL,BEGINID>"endfunction" {
281         if (last_token != DOT)
282     {
283         ParserSingleInstance::popControlStatus();
284     }
285     DEBUG("BEGIN(INITIAL)");
286     BEGIN(INITIAL);
287         return scan_throw(ENDFUNCTION);
288 }
289
290 <INITIAL,BEGINID>"for" {
291         if (last_token != DOT)
292     {
293         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
294     }
295     BEGIN(INITIAL);
296     return scan_throw(FOR);
297 }
298
299 <INITIAL,BEGINID>"while"        {
300         if (last_token != DOT)
301     {
302         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
303     }
304         BEGIN(INITIAL);
305         return scan_throw(WHILE);
306 }
307
308 <INITIAL,BEGINID>"do"           {
309         BEGIN(INITIAL);
310     return scan_throw(DO);
311 }
312
313 <INITIAL,BEGINID>"break"                {
314         BEGIN(INITIAL);
315         return scan_throw(BREAK);
316 }
317
318 <INITIAL,BEGINID>"continue"             {
319         BEGIN(INITIAL);
320         return scan_throw(CONTINUE);
321 }
322
323 <INITIAL,BEGINID>"try" {
324         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
325         BEGIN(INITIAL);
326         return scan_throw(TRY);
327 }
328
329 <INITIAL,BEGINID>"catch" {
330     // Pop to step out TRY
331         ParserSingleInstance::popControlStatus();
332         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
333         BEGIN(INITIAL);
334         return scan_throw(CATCH);
335 }
336
337 <INITIAL,BEGINID>"return"       {
338     BEGIN(INITIAL);
339     return scan_throw(RETURN);
340 }
341
342 <INITIAL,BEGINID>"resume"       {
343     BEGIN(INITIAL);
344     return scan_throw(RETURN);
345 }
346
347 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
348         BEGIN(BEGINID);
349 }
350
351 <BEGINID>
352 {
353     {id}                        {
354         wchar_t *pwText = to_wide_string(yytext);
355         if (yytext != NULL && pwText == NULL)
356         {
357             std::string str = "Can\'t convert \'";
358             str += yytext;
359             str += "\' to UTF-8";
360             BEGIN(INITIAL);
361             yyerror(str);
362             return scan_throw(FLEX_ERROR);
363         }
364         yylval.str = new std::wstring(pwText);
365         FREE(pwText);
366         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
367         if (pIT && pIT->isCallable())
368         {
369             scan_throw(ID);
370             BEGIN(SHELLMODE);
371         }
372         else
373         {
374             BEGIN(INITIAL);
375             return scan_throw(ID);
376         }
377     }
378
379 }
380
381 <INITIAL,MATRIX>{boolnot}               {
382   return scan_throw(NOT);
383 }
384 <INITIAL,MATRIX>{dollar}                {
385   return scan_throw(DOLLAR);
386 }
387 <INITIAL,MATRIX>{booltrue}              {
388   return scan_throw(BOOLTRUE);
389 }
390 <INITIAL,MATRIX>{boolfalse}             {
391   return scan_throw(BOOLFALSE);
392 }
393 <INITIAL,MATRIX>{booland}               {
394   return scan_throw(AND);
395 }
396 <INITIAL,MATRIX>{boolandand}    {
397   return scan_throw(ANDAND);
398 }
399 <INITIAL,MATRIX>{boolor}                {
400   return scan_throw(OR);
401 }
402 <INITIAL,MATRIX>{booloror}              {
403   return scan_throw(OROR);
404 }
405
406
407 <INITIAL>{lparen}                       {
408   return scan_throw(LPAREN);
409 }
410 <INITIAL>{rparen}                       {
411   return scan_throw(RPAREN);
412 }
413
414
415 <INITIAL,MATRIX>{semicolon}             {
416         scan_step();
417   return scan_throw(SEMI);
418 }
419
420 <INITIAL,MATRIX>{comma}                 {
421         scan_step();
422   return scan_throw(COMMA);
423 }
424
425 <INITIAL,MATRIX>{colon}                 {
426   return scan_throw(COLON);
427 }
428
429
430 <INITIAL,MATRIX>{lbrace}                {
431   yy_push_state(MATRIX);
432   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
433   return scan_throw(LBRACE);
434 }
435
436 {rbrace}                        {
437   return scan_throw(RBRACE);
438 }
439
440
441 <INITIAL,MATRIX>{dotquote}              {
442   return scan_throw(DOTQUOTE);
443 }
444 <INITIAL,MATRIX>{dottimes}              {
445   return scan_throw(DOTTIMES);
446 }
447 <INITIAL,MATRIX>{dotrdivide}            {
448   return scan_throw(DOTRDIVIDE);
449 }
450 <INITIAL,MATRIX>{dotldivide}            {
451   return scan_throw(DOTLDIVIDE);
452 }
453 <INITIAL,MATRIX>{dotpower}              {
454   return scan_throw(DOTPOWER);
455 }
456
457
458 {minus}                                 {
459   return scan_throw(MINUS);
460 }
461 {plus}                                  {
462   return scan_throw(PLUS);
463 }
464 <INITIAL,MATRIX>{times}                 {
465   return scan_throw(TIMES);
466 }
467 <INITIAL,MATRIX>{rdivide}               {
468   return scan_throw(RDIVIDE);
469 }
470 <INITIAL,MATRIX>{ldivide}               {
471   return scan_throw(LDIVIDE);
472 }
473 <INITIAL,MATRIX>{power}                 {
474   return scan_throw(POWER);
475 }
476
477 <INITIAL,MATRIX>{krontimes}             {
478   return scan_throw(KRONTIMES);
479 }
480 <INITIAL,MATRIX>{kronrdivide}           {
481   return scan_throw(KRONRDIVIDE);
482 }
483 <INITIAL,MATRIX>{kronldivide}           {
484   return scan_throw(KRONLDIVIDE);
485 }
486
487
488 <INITIAL,MATRIX>{controltimes}          {
489     unput(yytext[yyleng - 1]);
490     return scan_throw(CONTROLTIMES);
491 }
492 <INITIAL,MATRIX>{controlrdivide}                {
493     unput(yytext[yyleng - 1]);
494     return scan_throw(CONTROLRDIVIDE);
495 }
496 <INITIAL,MATRIX>{controlldivide}                {
497     unput(yytext[yyleng - 1]);
498     return scan_throw(CONTROLLDIVIDE);
499 }
500
501
502 <INITIAL,MATRIX>{equal}                 {
503   return scan_throw(EQ);
504 }
505 <INITIAL,MATRIX>{notequal}              {
506   return scan_throw(NE);
507 }
508 <INITIAL,MATRIX>{lowerthan}             {
509   return scan_throw(LT);
510 }
511 <INITIAL,MATRIX>{greaterthan}           {
512   return scan_throw(GT);
513 }
514 <INITIAL,MATRIX>{lowerequal}            {
515   return scan_throw(LE);
516 }
517 <INITIAL,MATRIX>{greaterequal}          {
518   return scan_throw(GE);
519 }
520
521
522 <INITIAL,MATRIX>{assign}                {
523   return scan_throw(ASSIGN);
524  }
525
526
527 <INITIAL,MATRIX>{lbrack}                {
528   DEBUG("yy_push_state(MATRIX)");
529   yy_push_state(MATRIX);
530   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
531   return scan_throw(LBRACK);
532 }
533
534 <INITIAL>{rbrack}                               {
535   return scan_throw(RBRACK);
536 }
537
538
539 <INITIAL,MATRIX>{dot}                   {
540   return scan_throw(DOT);
541 }
542
543 <INITIAL>{next}                 {
544     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
545     yy_push_state(LINEBREAK);
546 }
547
548 <INITIAL,MATRIX>{integer}               {
549   yylval.number = atof(yytext);
550 #ifdef TOKENDEV
551   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
552 #endif
553 //  scan_step();
554   return scan_throw(VARINT);
555 }
556
557
558 <INITIAL,MATRIX>{floating_D}            {
559   scan_exponent_convert(yytext);
560   yylval.number = atof(yytext);
561 #ifdef TOKENDEV
562   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
563 #endif
564   //scan_step();
565   return scan_throw(VARFLOAT);
566 }
567
568 <INITIAL,MATRIX>{floating_E}            {
569   yylval.number = atof(yytext);
570 #ifdef TOKENDEV
571   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
572 #endif
573   //scan_step();
574   return scan_throw(VARFLOAT);
575 }
576
577
578 <INITIAL,MATRIX>{number}                {
579   yylval.number = atof(yytext);
580 #ifdef TOKENDEV
581   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
582 #endif
583 //  scan_step();
584   return scan_throw(NUM);
585 }
586
587
588 <INITIAL,MATRIX>{little}                {
589   yylval.number = atof(yytext);
590 #ifdef TOKENDEV
591   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
592 #endif
593 //  scan_step();
594   return scan_throw(NUM);
595 }
596
597
598 <INITIAL,MATRIX>{id}                    {
599     wchar_t *pwText = to_wide_string(yytext);
600     if (yytext != NULL && pwText == NULL)
601     {
602         std::string str = "Can\'t convert \'";
603         str += yytext;
604         str += "\' to UTF-8";
605         BEGIN(INITIAL);
606         yyerror(str);
607         return scan_throw(FLEX_ERROR);
608     }
609     yylval.str = new std::wstring(pwText);
610     FREE(pwText);
611 #ifdef TOKENDEV
612   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
613 #endif
614 //  scan_step();
615   return scan_throw(ID);
616 }
617
618
619 <INITIAL,MATRIX>{startblockcomment}     {
620   yylval.comment = new std::wstring();
621   comment_level = 1;
622   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
623   yy_push_state(REGIONCOMMENT);
624 }
625
626
627 <INITIAL,MATRIX>{startcomment}          {
628   pstBuffer.clear();
629   yy_push_state(LINECOMMENT);
630 }
631
632
633 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
634   pstBuffer.clear();
635   str_opener_column = yylloc.first_column;
636   yy_push_state(DOUBLESTRING);
637 }
638
639
640 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
641   /*
642   ** Matrix Transposition special behaviour
643   ** ID' []' toto()' are transposition call
644   */
645   if (last_token == ID
646       || last_token == RBRACK
647       || last_token == RPAREN
648       || last_token == RBRACE
649       || last_token == VARINT
650       || last_token == VARFLOAT
651       || last_token == NUM
652       || last_token == BOOLTRUE
653       || last_token == BOOLFALSE)
654   {
655       return scan_throw(QUOTE);
656   }
657   else
658   {
659       pstBuffer.clear();
660       str_opener_column = yylloc.first_column;
661       yy_push_state(SIMPLESTRING);
662   }
663 }
664
665
666 <INITIAL,MATRIX>{spaces}                {
667   scan_step();
668   scan_throw(SPACES);
669 }
670
671
672 <INITIAL>{newline}              {
673   yylloc.last_line += 1;
674   yylloc.last_column = 1;
675   scan_step();
676   if (last_token != EOL) {
677       return scan_throw(EOL);
678   }
679
680 }
681
682
683 <INITIAL,MATRIX>{blankline}             {
684   yylloc.last_line += 1;
685   yylloc.last_column = 1;
686   scan_step();
687   if (last_token != EOL)
688   {
689       return scan_throw(EOL);
690   }
691   scan_throw(EOL);
692 }
693
694 <INITIAL,MATRIX>{emptyline}             {
695   yylloc.last_line += 2;
696   yylloc.last_column = 1;
697   scan_step();
698   if (last_token != EOL)
699   {
700       return scan_throw(EOL);
701   }
702   scan_throw(EOL);
703 }
704 .                                       {
705     std::string str = "Unexpected token \'";
706     str += yytext;
707     str += "\'";
708     BEGIN(INITIAL);
709     yyerror(str);
710     return scan_throw(FLEX_ERROR);
711 }
712
713
714 <MATRIX>
715 {
716   {lparen} {
717     paren_level++;
718     return scan_throw(LPAREN);
719   }
720
721   {rparen} {
722     paren_level--;
723     return scan_throw(RPAREN);
724   }
725
726   {spaces}*{lparen} {
727       unput(yytext[yyleng -1]);
728       if (last_token == ID
729           || last_token == RPAREN
730           || last_token == QUOTE
731           || last_token == VARINT
732           || last_token == VARFLOAT
733           || last_token == NUM)
734       {
735           return scan_throw(COMMA);
736       }
737   }
738
739   {spaces}*{colon}{spaces}* {
740       return scan_throw(COLON);
741   }
742
743   {newline} {
744       yylloc.last_line += 1;
745       yylloc.last_column = 1;
746       if(last_token != DOTS && last_token != EOL)
747       {
748           return scan_throw(EOL);
749       }
750       scan_throw(EOL);
751   }
752
753
754   {rbrack}                              {
755     DEBUG("yy_pop_state()");
756     yy_pop_state();
757     ParserSingleInstance::popControlStatus();
758     return scan_throw(RBRACK);
759   }
760
761   {rbrace}                              {
762     yy_pop_state();
763     ParserSingleInstance::popControlStatus();
764     return scan_throw(RBRACE);
765   }
766
767   {plus}                                |
768   {spaces}{plus}{spaces}                {
769     return scan_throw(PLUS);
770   }
771
772   {minus}                               |
773   {spaces}{minus}{spaces}               {
774     return scan_throw(MINUS);
775   }
776
777   {spaces}{plus}                        {
778     // no need to unput the '+'
779     if (last_token != LBRACK
780        && last_token != EOL
781        && last_token != SEMI
782        && last_token != COMMA
783        && paren_level == 0)
784    {
785        return scan_throw(COMMA);
786    }
787    else
788    {
789        unput('+');
790    }
791   }
792
793   {spaces}{minus}                       {
794     unput('-');
795     if (last_token != LBRACK
796        && last_token != EOL
797        && last_token != SEMI
798        && last_token != COMMA
799        && paren_level == 0)
800    {
801        return scan_throw(COMMA);
802    }
803   }
804
805   .                                     {
806     std::string str = "Unexpected token \'";
807     str += yytext;
808     str += "\' within a matrix.";
809     BEGIN(INITIAL);
810     yyerror(str);
811     return scan_throw(FLEX_ERROR);
812   }
813
814   {next}{spaces}*{newline}          {
815       /* Just do nothing */
816       yylloc.last_line += 1;
817       yylloc.last_column = 1;
818       scan_step();
819       scan_throw(EOL);
820   }
821
822   {next}{spaces}*{startcomment}          {
823       /* Just do nothing */
824       pstBuffer.clear();
825       yy_push_state(LINECOMMENT);
826       scan_throw(DOTS);
827   }
828
829   <<EOF>>       {
830       yy_pop_state();
831   }
832 }
833
834 <LINEBREAK>
835 {
836   {newline}                             {
837     yylloc.last_line += 1;
838     yylloc.last_column = 1;
839     scan_step();
840     yy_pop_state();
841     ParserSingleInstance::popControlStatus();
842   }
843
844   {startblockcomment}                   {
845     ++comment_level;
846     yy_push_state(REGIONCOMMENT);
847   }
848
849   {startcomment}                        {
850     scan_throw(DOTS);
851     pstBuffer.clear();
852     yy_push_state(LINECOMMENT);
853   }
854
855   {spaces}                              {
856       /* Do nothing... */
857   }
858
859   <<EOF>>       {
860       yy_pop_state();
861   }
862   .                                     {
863     std::string str = "Unexpected token \'";
864     str += yytext;
865     str += "\' after line break with .. or ...";
866     BEGIN(INITIAL);
867     yyerror(str);
868     return scan_throw(FLEX_ERROR);
869   }
870 }
871
872
873 <LINECOMMENT>
874 {
875   {newline}     {
876     //yylloc.last_line += 1;
877     //yylloc.last_column = 1;
878     //scan_step();
879     yy_pop_state();
880     for (int i = yyleng - 1 ; i >= 0 ; --i)
881     {
882         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
883         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
884         unput(yytext[i]);
885         yylloc.last_column--;
886     }
887     /*
888     ** To forgot comments after lines break
889     */
890     if (last_token != DOTS)
891     {
892         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
893         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
894         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
895         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
896         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
897         {
898             pstBuffer.clear();
899             std::string str = "Can\'t convert \'";
900             str += pstBuffer.c_str();
901             str += "\' to UTF-8";
902             BEGIN(INITIAL);
903             yyerror(str);
904             return scan_throw(FLEX_ERROR);
905         }
906         yylval.comment = new std::wstring(pwstBuffer);
907         pstBuffer.clear();
908         FREE (pwstBuffer);
909         return scan_throw(COMMENT);
910     }
911     else
912     {
913         pstBuffer.clear();
914     }
915   }
916
917   <<EOF>>       {
918     yy_pop_state();
919     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
920     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
921     {
922         pstBuffer.clear();
923         std::string str = "Can\'t convert \'";
924         str += pstBuffer.c_str();
925         str += "\' to UTF-8";
926         BEGIN(INITIAL);
927         yyerror(str);
928         return scan_throw(FLEX_ERROR);
929     }
930     yylval.comment = new std::wstring(pwstBuffer);
931     pstBuffer.clear();
932     FREE (pwstBuffer);
933     return scan_throw(COMMENT);
934   }
935
936   {char_in_line_comment}         {
937       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
938       // only translate to WCHAR_T when popping state.
939       pstBuffer += yytext;
940   }
941 }
942
943
944 <REGIONCOMMENT>
945 {
946   {endblockcomment}                             {
947     --comment_level;
948     if (comment_level == 0) {
949       ParserSingleInstance::popControlStatus();
950       yy_pop_state();
951       //return scan_throw(BLOCKCOMMENT);
952     }
953   }
954
955   {startblockcomment}                           {
956     ++comment_level;
957     yy_push_state(REGIONCOMMENT);
958   }
959
960   {newline}                                     {
961     yylloc.last_line += 1;
962     yylloc.last_column = 1;
963     scan_step();
964     *yylval.comment += L"\n//";
965   }
966
967   {char_in_comment}                             |
968   .                                             {
969       wchar_t *pwText = to_wide_string(yytext);
970       *yylval.comment += std::wstring(pwText);
971       FREE(pwText);
972   }
973
974  <<EOF>>                                        {
975       yy_pop_state();
976 //    std::string str = "unexpected end of file in a comment";
977 //    scan_error(str);
978   }
979 }
980
981
982 <SIMPLESTRING>
983 {
984   {dquote}{dquote}                              {
985     pstBuffer += "\"";
986   }
987
988   {dquote}{quote}                               {
989     pstBuffer += "'";
990   }
991
992   {quote}{dquote}                               {
993     pstBuffer += "\"";
994   }
995
996   {quote}{quote}                                {
997     pstBuffer += "'";
998   }
999
1000   {quote}                                       {
1001     yy_pop_state();
1002     //scan_step();
1003     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1004     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1005     {
1006         pstBuffer.clear();
1007         std::string str = "Can\'t convert \'";
1008         str += pstBuffer.c_str();
1009         str += "\' to UTF-8";
1010         BEGIN(INITIAL);
1011         yyerror(str);
1012         return scan_throw(FLEX_ERROR);
1013     }
1014     yylval.str = new std::wstring(pwstBuffer);
1015     pstBuffer.clear();
1016     FREE(pwstBuffer);
1017     yylloc.first_column = str_opener_column;
1018     return scan_throw(STR);
1019   }
1020
1021   {dquote}                  {
1022     pstBuffer.clear();
1023     BEGIN(INITIAL);
1024     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1025     return scan_throw(FLEX_ERROR);
1026   }
1027
1028   {next}{newline}           {
1029       yylloc.last_line += 1;
1030       yylloc.last_column = 1;
1031       /* Do nothing... Just skip */
1032   }
1033
1034   {newline}                                     {
1035     pstBuffer.clear();
1036     yylloc.last_line += 1;
1037     yylloc.last_column = 1;
1038     BEGIN(INITIAL);
1039     yyerror("Unexpected end of line in a string.");
1040     return scan_throw(FLEX_ERROR);
1041   }
1042
1043   <<EOF>>                                       {
1044     pstBuffer.clear();
1045     BEGIN(INITIAL);
1046     yyerror("Unexpected end of file in a string.");
1047     return scan_throw(FLEX_ERROR);    
1048   }
1049
1050   {in_string}                                           |
1051   .                                                     {
1052     //scan_step();
1053     pstBuffer += yytext;
1054   }
1055 }
1056
1057
1058 <DOUBLESTRING>
1059 {
1060   {dquote}{dquote}                              {
1061     pstBuffer += "\"";
1062   }
1063
1064   {dquote}{quote}                               {
1065     pstBuffer += "'";
1066   }
1067
1068   {quote}{dquote}               {
1069     pstBuffer += "\"";
1070   }
1071
1072   {quote}{quote}                                {
1073     pstBuffer += "'";
1074   }
1075
1076   {dquote}                      {
1077     yy_pop_state();
1078     //scan_step();
1079     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1080     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1081     {
1082         pstBuffer.clear();
1083         std::string str = "Can\'t convert \'";
1084         str += pstBuffer.c_str();
1085         str += "\' to UTF-8";
1086         BEGIN(INITIAL);
1087         yyerror(str);
1088         return scan_throw(FLEX_ERROR);
1089     }
1090     yylval.str = new std::wstring(pwstBuffer);
1091     pstBuffer.clear();
1092     FREE(pwstBuffer);
1093     yylloc.first_column = str_opener_column;
1094     return scan_throw(STR);
1095   }
1096
1097   {quote}                  {
1098     pstBuffer.clear();
1099     BEGIN(INITIAL);
1100     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1101     return scan_throw(FLEX_ERROR);
1102   }
1103
1104   {next}{newline}           {
1105       yylloc.last_line += 1;
1106       yylloc.last_column = 1;
1107       /* Do nothing... Just skip */
1108   }
1109
1110   {newline} {
1111     pstBuffer.clear();
1112     yylloc.last_line += 1;
1113     yylloc.last_column = 1;
1114     BEGIN(INITIAL);
1115     yyerror("Unexpected end of line in a string.");
1116     return scan_throw(FLEX_ERROR);
1117   }
1118
1119   <<EOF>>   {
1120     pstBuffer.clear();
1121     BEGIN(INITIAL);
1122     yyerror("Unexpected end of file in a string.");
1123     return scan_throw(FLEX_ERROR);    
1124   }
1125
1126   {in_string}         |
1127   .                   {
1128    //scan_step();
1129    pstBuffer += yytext;
1130   }
1131 }
1132
1133
1134 <SHELLMODE>
1135 {
1136     {spaces}                    {
1137         if (last_token == ID)
1138         {
1139             scan_throw(SPACES);
1140             return ID;
1141         }
1142     }
1143
1144     {semicolon}                 {
1145         BEGIN(INITIAL);
1146         scan_step();
1147         return scan_throw(SEMI);
1148     }
1149
1150     {comma}                     {
1151         BEGIN(INITIAL);
1152         scan_step();
1153         return scan_throw(COMMA);
1154     }
1155
1156     {newline}                   {
1157         BEGIN(INITIAL);
1158         yylloc.last_line += 1;
1159         yylloc.last_column = 1;
1160         scan_step();
1161         return scan_throw(EOL);
1162     }
1163
1164     {assign} {
1165         if (last_token == STR || last_token == SPACES)
1166         {
1167             wchar_t *pwText = to_wide_string(yytext);
1168             yylval.str = new std::wstring(pwText);
1169             FREE(pwText);
1170             return scan_throw(STR);
1171         }
1172         else
1173         {
1174             BEGIN(INITIAL);
1175             return scan_throw(ASSIGN);
1176         }
1177     }
1178
1179     {lparen} {
1180         if (last_token == STR || last_token == SPACES)
1181         {
1182             wchar_t *pwText = to_wide_string(yytext);
1183             yylval.str = new std::wstring(pwText);
1184             FREE(pwText);
1185             return scan_throw(STR);
1186         }
1187         else
1188         {
1189             BEGIN(INITIAL);
1190             return scan_throw(LPAREN);
1191         }
1192     }
1193
1194     {lowerthan} {
1195         if (last_token == STR || last_token == SPACES)
1196         {
1197             wchar_t *pwText = to_wide_string(yytext);
1198             yylval.str = new std::wstring(pwText);
1199             FREE(pwText);
1200             return scan_throw(STR);
1201         }
1202         else
1203         {
1204             BEGIN(INITIAL);
1205             return scan_throw(LT);
1206         }
1207     }
1208
1209     {greaterthan} {
1210         if (last_token == STR || last_token == SPACES)
1211         {
1212             wchar_t *pwText = to_wide_string(yytext);
1213             yylval.str = new std::wstring(pwText);
1214             FREE(pwText);
1215             return scan_throw(STR);
1216         }
1217         else
1218         {
1219             BEGIN(INITIAL);
1220             return scan_throw(GT);
1221         }
1222     }
1223
1224     {boolnot} {
1225         if (last_token == STR || last_token == SPACES)
1226         {
1227             wchar_t *pwText = to_wide_string(yytext);
1228             yylval.str = new std::wstring(pwText);
1229             FREE(pwText);
1230             return scan_throw(STR);
1231         }
1232         else
1233         {
1234             BEGIN(INITIAL);
1235             return scan_throw(NOT);
1236         }
1237     }
1238
1239
1240     [^ \t\v\f\r\n,;'"]+               {
1241         wchar_t *pwText = to_wide_string(yytext);
1242         yylval.str = new std::wstring(pwText);
1243         FREE(pwText);
1244         return scan_throw(STR);
1245     }
1246
1247     <<EOF>>                     {
1248         BEGIN(INITIAL);
1249     }
1250
1251 }
1252
1253 %%
1254
1255 int scan_throw(int token) {
1256   last_token = token;
1257 #ifdef DEV
1258   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1259 #endif
1260   return token;
1261 }
1262
1263 int get_last_token() {
1264     return last_token;
1265 }
1266
1267 void scan_step() {
1268   yylloc.first_line = yylloc.last_line;
1269   yylloc.first_column = yylloc.last_column;
1270 }
1271
1272 /*
1273 ** convert floating numbers to C standard
1274 ** 1.2d-3 -> 1.2e-3
1275 ** 1.2D-3 -> 1.2e-3
1276 */
1277 void scan_exponent_convert(char *in)
1278 {
1279   for (; *in != 'd' && *in != 'D'; ++in);
1280   *in = 'e';
1281 }
1282
1283 #ifdef _MSC_VER
1284 int isatty (int desc)
1285 {
1286   return 0;
1287 }
1288 #endif