19a43188a14e7fcca48989030e0385c877e044ac
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "sci_malloc.h"
23 }
24
25 static int comment_level = 0;
26 static int paren_level = 0;
27 static int last_token = 0;
28 static int exit_status = PARSE_ERROR;
29 static std::string current_file;
30 static std::string program_name;
31
32 static std::string *pstBuffer;
33
34 #define YY_USER_ACTION                          \
35  yylloc.first_column = yylloc.last_column;yylloc.last_column += yyleng;
36 //yylloc.last_column += yyleng;
37
38 /* -*- Verbose Special Debug -*- */
39 //#define DEV
40 //#define TOKENDEV
41
42 #ifdef DEV
43 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
44 #else
45 #define DEBUG(x) /* Nothing */
46 #endif
47
48 %}
49
50 %option stack
51 %option noyywrap
52
53 %x SIMPLESTRING
54 %x DOUBLESTRING
55 %x REGIONCOMMENT
56 %x LINECOMMENT
57 %x LINEBREAK
58
59 %x MATRIX
60
61 %x SHELLMODE
62 %x BEGINID
63
64 spaces                  [ \t\v\f]+
65 integer                 [0-9]+
66 number                  [0-9]+[\.][0-9]*
67 little                  \.[0-9]+
68 bom             \xEF\xBB\xBF
69
70 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
71 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
72
73 hex             [0]x[0-9a-fA-F]+
74 oct             [0]o[0-7]+
75
76
77 utf2            ([\xC2-\xDF][\x80-\xBF])
78 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
79 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
80 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
81 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
82 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
83 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
84 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
85
86 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
87 utf4            ({utf41}|{utf42}|{utf43})
88
89 utf             ({utf2}|{utf3}|{utf4})
90 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
91
92
93 newline                 ("\r"|"\n"|"\r\n")
94 blankline               {spaces}+{newline}
95 emptyline       {newline}({spaces}|[,;])+{newline}
96 next                    \.\.+
97 char_in_line_comment    [^\r\n]*
98 char_in_comment         [^\r\n\/*]*
99
100 boolnot                 ("@"|"~")
101 booltrue                ("%t"|"%T")
102 boolfalse               ("%f"|"%F")
103 booland                 ("&")
104 boolandand              ("&&")
105 boolor                  ("|")
106 booloror                ("||")
107
108 lbrack                  "["
109 rbrack                  "]"
110
111 lparen                  "("
112 rparen                  ")"
113
114 lbrace                  "{"
115 rbrace                  "}"
116
117 dollar                  "$"
118
119 semicolon               ";"
120 comma                   ","
121 colon                   ":"
122
123 startcomment            "//"
124 startblockcomment       "/*"
125 endblockcomment         "*/"
126
127 dquote                  "\""
128 quote                   "'"
129 in_string               [^\"\'\r\n\.]*
130
131 dot             "."
132 dotquote                ".'"
133 dottimes                ".*"
134 dotrdivide              "./"
135 dotldivide              ".\\"
136 dotpower                (".^"|".**")
137
138 plus                    "+"
139 minus                   "-"
140 rdivide                 "/"
141 ldivide                 "\\"
142 times                   "*"
143 power                   ("^"|"**")
144
145 equal                   "="{spaces}*"="
146 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
147 lowerthan               "<"
148 greaterthan             ">"
149 lowerequal              "<"{spaces}*"="
150 greaterequal            ">"{spaces}*"="
151
152 krontimes               ".*."
153 kronrdivide             "./."
154 kronldivide             ".\\."
155
156 controltimes    ("*."[^0-9])
157 controlrdivide  ("/."[^0-9])
158 controlldivide  ("\\."[^0-9])
159
160 assign                  "="
161
162 %%
163 <INITIAL>{bom}/.* {
164 // BOM found ==> ignored
165 }
166
167 <INITIAL,BEGINID>{booltrue}     {
168     BEGIN(INITIAL);
169     return scan_throw(BOOLTRUE);
170 }
171 <INITIAL,BEGINID>{boolfalse}    {
172     BEGIN(INITIAL);
173     return scan_throw(BOOLFALSE);
174 }
175
176 <INITIAL,BEGINID>"if"            {
177         if (last_token != DOT)
178     {
179         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
180     }
181     DEBUG("BEGIN(INITIAL)");
182     BEGIN(INITIAL);
183     return scan_throw(IF);
184 }
185
186 <INITIAL,BEGINID>"then"          {
187     DEBUG("BEGIN(INITIAL)");
188     BEGIN(INITIAL);
189     return scan_throw(THEN);
190 }
191
192 <INITIAL,BEGINID>"else"          {
193         if (last_token != DOT)
194     {
195         // Pop to step out IF
196         ParserSingleInstance::popControlStatus();
197         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
198     }
199     DEBUG("BEGIN(INITIAL)");
200     BEGIN(INITIAL);
201         return scan_throw(ELSE);
202 }
203
204 <INITIAL,BEGINID>"elseif" {
205         if (last_token != DOT)
206     {
207         ParserSingleInstance::popControlStatus();
208         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
209     }
210     DEBUG("BEGIN(INITIAL)");
211     BEGIN(INITIAL);
212         return scan_throw(ELSEIF);
213 }
214
215 <INITIAL,BEGINID>"end"          {
216         if (last_token != DOT)
217     {
218         ParserSingleInstance::popControlStatus();
219     }
220     DEBUG("BEGIN(INITIAL)");
221     BEGIN(INITIAL);
222     return scan_throw(END);
223 }
224
225 <INITIAL,BEGINID>"select"       {
226         if (last_token != DOT)
227     {
228         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
229     }
230     DEBUG("BEGIN(INITIAL)");
231     BEGIN(INITIAL);
232     return scan_throw(SELECT);
233 }
234
235 <INITIAL,BEGINID>"switch"       {
236         if (last_token != DOT)
237     {
238         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
239     }
240     DEBUG("BEGIN(INITIAL)");
241     BEGIN(INITIAL);
242     return scan_throw(SWITCH);
243 }
244
245 <INITIAL,BEGINID>"otherwise" {
246         if (last_token != DOT)
247     {
248         ParserSingleInstance::popControlStatus();
249         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
250     }
251     DEBUG("BEGIN(INITIAL)");
252     BEGIN(INITIAL);
253         return scan_throw(OTHERWISE);
254 }
255
256 <INITIAL,BEGINID>"case"         {
257         if (last_token != DOT)
258     {
259         ParserSingleInstance::popControlStatus();
260         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
261     }
262     DEBUG("BEGIN(INITIAL)");
263     BEGIN(INITIAL);
264     return scan_throw(CASE);
265 }
266
267 <INITIAL,BEGINID>"function" {
268         if (last_token != DOT)
269     {
270         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
271     }
272     DEBUG("BEGIN(INITIAL)");
273     BEGIN(INITIAL);
274     return scan_throw(FUNCTION);
275 }
276
277 <INITIAL,BEGINID>"endfunction" {
278         if (last_token != DOT)
279     {
280         ParserSingleInstance::popControlStatus();
281     }
282     DEBUG("BEGIN(INITIAL)");
283     BEGIN(INITIAL);
284         return scan_throw(ENDFUNCTION);
285 }
286
287 <INITIAL,BEGINID>"#function"    {
288         if (last_token != DOT)
289     {
290         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
291     }
292     DEBUG("BEGIN(INITIAL)");
293     BEGIN(INITIAL);
294         return scan_throw(HIDDENFUNCTION);
295 }
296
297 <INITIAL,BEGINID>"hidden"       {
298     DEBUG("BEGIN(INITIAL)");
299         BEGIN(INITIAL);
300     return scan_throw(HIDDEN);
301 }
302
303 <INITIAL,BEGINID>"for" {
304         if (last_token != DOT)
305     {
306         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
307     }
308     BEGIN(INITIAL);
309     return scan_throw(FOR);
310 }
311
312 <INITIAL,BEGINID>"while"        {
313         if (last_token != DOT)
314     {
315         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
316     }
317         BEGIN(INITIAL);
318         return scan_throw(WHILE);
319 }
320
321 <INITIAL,BEGINID>"do"           {
322         BEGIN(INITIAL);
323     return scan_throw(DO);
324 }
325
326 <INITIAL,BEGINID>"break"                {
327         BEGIN(INITIAL);
328         return scan_throw(BREAK);
329 }
330
331 <INITIAL,BEGINID>"continue"             {
332         BEGIN(INITIAL);
333         return scan_throw(CONTINUE);
334 }
335
336 <INITIAL,BEGINID>"try" {
337         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
338         BEGIN(INITIAL);
339         return scan_throw(TRY);
340 }
341
342 <INITIAL,BEGINID>"catch" {
343     // Pop to step out TRY
344         ParserSingleInstance::popControlStatus();
345         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
346         BEGIN(INITIAL);
347         return scan_throw(CATCH);
348 }
349
350 <INITIAL,BEGINID>"return"       {
351     BEGIN(INITIAL);
352     return scan_throw(RETURN);
353 }
354
355 <INITIAL,BEGINID>"resume"       {
356     BEGIN(INITIAL);
357     return scan_throw(RETURN);
358 }
359
360 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
361         BEGIN(BEGINID);
362 }
363
364 <BEGINID>
365 {
366     {id}                        {
367         wchar_t *pwText = to_wide_string(yytext);
368         if (yytext != NULL && pwText == NULL)
369         {
370             std::string str = "can not convert'";
371             str += yytext;
372             str += "' to UTF-8";
373             exit_status = SCAN_ERROR;
374             scan_error("can not convert string to UTF-8");
375         }
376         yylval.str = new std::wstring(pwText);
377         FREE(pwText);
378         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
379         if (pIT && pIT->isCallable())
380         {
381             scan_throw(ID);
382             BEGIN(SHELLMODE);
383         }
384         else
385         {
386             BEGIN(INITIAL);
387             return scan_throw(ID);
388         }
389     }
390
391 }
392
393 <INITIAL,MATRIX>{boolnot}               {
394   return scan_throw(NOT);
395 }
396 <INITIAL,MATRIX>{dollar}                {
397   return scan_throw(DOLLAR);
398 }
399 <INITIAL,MATRIX>{booltrue}              {
400   return scan_throw(BOOLTRUE);
401 }
402 <INITIAL,MATRIX>{boolfalse}             {
403   return scan_throw(BOOLFALSE);
404 }
405 <INITIAL,MATRIX>{booland}               {
406   return scan_throw(AND);
407 }
408 <INITIAL,MATRIX>{boolandand}    {
409   return scan_throw(ANDAND);
410 }
411 <INITIAL,MATRIX>{boolor}                {
412   return scan_throw(OR);
413 }
414 <INITIAL,MATRIX>{booloror}              {
415   return scan_throw(OROR);
416 }
417
418
419 <INITIAL>{lparen}                       {
420   return scan_throw(LPAREN);
421 }
422 <INITIAL>{rparen}                       {
423   return scan_throw(RPAREN);
424 }
425
426
427 <INITIAL,MATRIX>{semicolon}             {
428         scan_step();
429   return scan_throw(SEMI);
430 }
431
432 <INITIAL,MATRIX>{comma}                 {
433         scan_step();
434   return scan_throw(COMMA);
435 }
436
437 <INITIAL,MATRIX>{colon}                 {
438   return scan_throw(COLON);
439 }
440
441
442 <INITIAL,MATRIX>{lbrace}                {
443   yy_push_state(MATRIX);
444   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
445   return scan_throw(LBRACE);
446 }
447
448 {rbrace}                        {
449   return scan_throw(RBRACE);
450 }
451
452
453 <INITIAL,MATRIX>{dotquote}              {
454   return scan_throw(DOTQUOTE);
455 }
456 <INITIAL,MATRIX>{dottimes}              {
457   return scan_throw(DOTTIMES);
458 }
459 <INITIAL,MATRIX>{dotrdivide}            {
460   return scan_throw(DOTRDIVIDE);
461 }
462 <INITIAL,MATRIX>{dotldivide}            {
463   return scan_throw(DOTLDIVIDE);
464 }
465 <INITIAL,MATRIX>{dotpower}              {
466   return scan_throw(DOTPOWER);
467 }
468
469
470 {minus}                                 {
471   return scan_throw(MINUS);
472 }
473 {plus}                                  {
474   return scan_throw(PLUS);
475 }
476 <INITIAL,MATRIX>{times}                 {
477   return scan_throw(TIMES);
478 }
479 <INITIAL,MATRIX>{rdivide}               {
480   return scan_throw(RDIVIDE);
481 }
482 <INITIAL,MATRIX>{ldivide}               {
483   return scan_throw(LDIVIDE);
484 }
485 <INITIAL,MATRIX>{power}                 {
486   return scan_throw(POWER);
487 }
488
489 <INITIAL,MATRIX>{krontimes}             {
490   return scan_throw(KRONTIMES);
491 }
492 <INITIAL,MATRIX>{kronrdivide}           {
493   return scan_throw(KRONRDIVIDE);
494 }
495 <INITIAL,MATRIX>{kronldivide}           {
496   return scan_throw(KRONLDIVIDE);
497 }
498
499
500 <INITIAL,MATRIX>{controltimes}          {
501     unput(yytext[yyleng - 1]);
502     return scan_throw(CONTROLTIMES);
503 }
504 <INITIAL,MATRIX>{controlrdivide}                {
505     unput(yytext[yyleng - 1]);
506     return scan_throw(CONTROLRDIVIDE);
507 }
508 <INITIAL,MATRIX>{controlldivide}                {
509     unput(yytext[yyleng - 1]);
510     return scan_throw(CONTROLLDIVIDE);
511 }
512
513
514 <INITIAL,MATRIX>{equal}                 {
515   return scan_throw(EQ);
516 }
517 <INITIAL,MATRIX>{notequal}              {
518   return scan_throw(NE);
519 }
520 <INITIAL,MATRIX>{lowerthan}             {
521   return scan_throw(LT);
522 }
523 <INITIAL,MATRIX>{greaterthan}           {
524   return scan_throw(GT);
525 }
526 <INITIAL,MATRIX>{lowerequal}            {
527   return scan_throw(LE);
528 }
529 <INITIAL,MATRIX>{greaterequal}          {
530   return scan_throw(GE);
531 }
532
533
534 <INITIAL,MATRIX>{assign}                {
535   return scan_throw(ASSIGN);
536  }
537
538
539 <INITIAL,MATRIX>{lbrack}                {
540   DEBUG("yy_push_state(MATRIX)");
541   yy_push_state(MATRIX);
542   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
543   return scan_throw(LBRACK);
544 }
545
546 <INITIAL>{rbrack}                               {
547   return scan_throw(RBRACK);
548 }
549
550
551 <INITIAL,MATRIX>{dot}                   {
552   return scan_throw(DOT);
553 }
554
555 <INITIAL>{next}                 {
556     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
557     yy_push_state(LINEBREAK);
558 }
559
560 <INITIAL,MATRIX>{integer}               {
561   yylval.number = atof(yytext);
562 #ifdef TOKENDEV
563   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
564 #endif
565 //  scan_step();
566   return scan_throw(VARINT);
567 }
568
569
570 <INITIAL,MATRIX>{floating_D}            {
571   scan_exponent_convert(yytext);
572   yylval.number = atof(yytext);
573 #ifdef TOKENDEV
574   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
575 #endif
576   scan_step();
577   return scan_throw(VARFLOAT);
578 }
579
580 <INITIAL,MATRIX>{floating_E}            {
581   yylval.number = atof(yytext);
582 #ifdef TOKENDEV
583   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
584 #endif
585   scan_step();
586   return scan_throw(VARFLOAT);
587 }
588
589
590 <INITIAL,MATRIX>{number}                {
591   yylval.number = atof(yytext);
592 #ifdef TOKENDEV
593   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
594 #endif
595 //  scan_step();
596   return scan_throw(NUM);
597 }
598
599
600 <INITIAL,MATRIX>{little}                {
601   yylval.number = atof(yytext);
602 #ifdef TOKENDEV
603   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
604 #endif
605   scan_step();
606   return scan_throw(NUM);
607 }
608
609
610 <INITIAL,MATRIX>{id}                    {
611     wchar_t *pwText = to_wide_string(yytext);
612     if (yytext != NULL && pwText == NULL)
613     {
614         std::string str = "can not convert'";
615         str += yytext;
616         str += "' to UTF-8";
617         exit_status = SCAN_ERROR;
618         scan_error("can not convert string to UTF-8");
619     }
620     yylval.str = new std::wstring(pwText);
621     FREE(pwText);
622 #ifdef TOKENDEV
623   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
624 #endif
625 //  scan_step();
626   return scan_throw(ID);
627 }
628
629
630 <INITIAL,MATRIX>{startblockcomment}     {
631   yylval.comment = new std::wstring();
632   comment_level = 1;
633   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
634   yy_push_state(REGIONCOMMENT);
635 }
636
637
638 <INITIAL,MATRIX>{startcomment}          {
639   pstBuffer = new std::string();
640   yy_push_state(LINECOMMENT);
641 }
642
643
644 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
645   pstBuffer = new std::string();
646   yy_push_state(DOUBLESTRING);
647 }
648
649
650 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
651   /*
652   ** Matrix Transposition special behaviour
653   ** ID' []' toto()' are transposition call
654   */
655   if (last_token == ID
656       || last_token == RBRACK
657       || last_token == RPAREN
658       || last_token == RBRACE
659       || last_token == VARINT
660       || last_token == VARFLOAT
661       || last_token == NUM
662       || last_token == BOOLTRUE
663       || last_token == BOOLFALSE)
664   {
665       return scan_throw(QUOTE);
666   }
667   else
668   {
669       pstBuffer = new std::string();
670       yy_push_state(SIMPLESTRING);
671   }
672 }
673
674
675 <INITIAL,MATRIX>{spaces}                {
676   scan_step();
677   scan_throw(SPACES);
678 }
679
680
681 <INITIAL>{newline}              {
682   yylloc.last_line += 1;
683   yylloc.last_column = 1;
684   scan_step();
685   if (last_token != EOL) {
686       return scan_throw(EOL);
687   }
688
689 }
690
691
692 <INITIAL,MATRIX>{blankline}             {
693   yylloc.last_line += 1;
694   yylloc.last_column = 1;
695   scan_step();
696   if (last_token != EOL)
697   {
698       return scan_throw(EOL);
699   }
700   scan_throw(EOL);
701 }
702
703 <INITIAL,MATRIX>{emptyline}             {
704   yylloc.last_line += 2;
705   yylloc.last_column = 1;
706   scan_step();
707   if (last_token != EOL)
708   {
709       return scan_throw(EOL);
710   }
711   scan_throw(EOL);
712 }
713 .                                       {
714     std::string str = "unexpected token '";
715     str += yytext;
716     str += "'";
717     exit_status = SCAN_ERROR;
718     scan_error(str);
719 }
720
721
722 <MATRIX>
723 {
724   {lparen} {
725     paren_level++;
726     return scan_throw(LPAREN);
727   }
728
729   {rparen} {
730     paren_level--;
731     return scan_throw(RPAREN);
732   }
733
734   {spaces}*{lparen} {
735       unput(yytext[yyleng -1]);
736       if (last_token == ID
737           || last_token == RPAREN
738           || last_token == QUOTE
739           || last_token == VARINT
740           || last_token == VARFLOAT
741           || last_token == NUM)
742       {
743           return scan_throw(COMMA);
744       }
745   }
746
747   {spaces}*{colon}{spaces}* {
748       return scan_throw(COLON);
749   }
750
751   {newline} {
752       yylloc.last_line += 1;
753       yylloc.last_column = 1;
754       if(last_token != DOTS && last_token != EOL)
755       {
756           return scan_throw(EOL);
757       }
758       scan_throw(EOL);
759   }
760
761
762   {rbrack}                              {
763     DEBUG("yy_pop_state()");
764     yy_pop_state();
765     ParserSingleInstance::popControlStatus();
766     return scan_throw(RBRACK);
767   }
768
769   {rbrace}                              {
770     yy_pop_state();
771     ParserSingleInstance::popControlStatus();
772     return scan_throw(RBRACE);
773   }
774
775   {plus}                                |
776   {spaces}{plus}{spaces}                {
777     return scan_throw(PLUS);
778   }
779
780   {minus}                               |
781   {spaces}{minus}{spaces}               {
782     return scan_throw(MINUS);
783   }
784
785   {spaces}{plus}                        {
786     // no need to unput the '+'
787     if (last_token != LBRACK
788        && last_token != EOL
789        && last_token != SEMI
790        && last_token != COMMA
791        && paren_level == 0)
792    {
793        return scan_throw(COMMA);
794    }
795    else
796    {
797        unput('+');
798    }
799   }
800
801   {spaces}{minus}                       {
802     unput('-');
803     if (last_token != LBRACK
804        && last_token != EOL
805        && last_token != SEMI
806        && last_token != COMMA
807        && paren_level == 0)
808    {
809        return scan_throw(COMMA);
810    }
811   }
812
813   .                                     {
814     std::string str = "unexpected token '";
815     str += yytext;
816     str += "' within a matrix.";
817     exit_status = SCAN_ERROR;
818     scan_error(str);
819   }
820
821   {next}{spaces}*{newline}          {
822       /* Just do nothing */
823       yylloc.last_line += 1;
824       yylloc.last_column = 1;
825       scan_step();
826       scan_throw(EOL);
827   }
828
829   {next}{spaces}*{startcomment}          {
830       /* Just do nothing */
831       pstBuffer = new std::string();
832       yy_push_state(LINECOMMENT);
833       scan_throw(DOTS);
834   }
835
836   <<EOF>>       {
837       yy_pop_state();
838   }
839 }
840
841 <LINEBREAK>
842 {
843   {newline}                             {
844     yylloc.last_line += 1;
845     yylloc.last_column = 1;
846     scan_step();
847     yy_pop_state();
848     ParserSingleInstance::popControlStatus();
849   }
850
851   {startblockcomment}                   {
852     ++comment_level;
853     yy_push_state(REGIONCOMMENT);
854   }
855
856   {startcomment}                        {
857     scan_throw(DOTS);
858     pstBuffer = new std::string();
859     yy_push_state(LINECOMMENT);
860   }
861
862   {spaces}                              {
863       /* Do nothing... */
864   }
865
866   <<EOF>>       {
867       yy_pop_state();
868   }
869   .                                     {
870     std::string str = "unexpected token '";
871     str += yytext;
872     str += "' after line break with .. or ...";
873     exit_status = SCAN_ERROR;
874     scan_error(str);
875   }
876 }
877
878
879 <LINECOMMENT>
880 {
881   {newline}     {
882     //yylloc.last_line += 1;
883     //yylloc.last_column = 1;
884     //scan_step();
885     yy_pop_state();
886     for (int i = yyleng - 1 ; i >= 0 ; --i)
887     {
888         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
889         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
890         unput(yytext[i]);
891         yylloc.last_column--;
892     }
893     /*
894     ** To forgot comments after lines break
895     */
896     if (last_token != DOTS)
897     {
898         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
899         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
900         wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
901         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
902         if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
903         {
904             std::string str = "can not convert'";
905             str += pstBuffer->c_str();
906             str += "' to UTF-8";
907             exit_status = SCAN_ERROR;
908             scan_error("can not convert string to UTF-8");
909         }
910         yylval.comment = new std::wstring(pwstBuffer);
911         delete pstBuffer;
912         FREE (pwstBuffer);
913         return scan_throw(COMMENT);
914     }
915     else
916     {
917         delete pstBuffer;
918     }
919   }
920
921   <<EOF>>       {
922     yy_pop_state();
923     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
924     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
925     {
926         std::string str = "can not convert'";
927         str += pstBuffer->c_str();
928         str += "' to UTF-8";
929         exit_status = SCAN_ERROR;
930         scan_error("can not convert string to UTF-8");
931     }
932     yylval.comment = new std::wstring(pwstBuffer);
933     delete pstBuffer;
934     FREE (pwstBuffer);
935     return scan_throw(COMMENT);
936   }
937
938   {char_in_line_comment}         {
939       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
940       // only translate to WCHAR_T when popping state.
941       *pstBuffer += yytext;
942   }
943 }
944
945
946 <REGIONCOMMENT>
947 {
948   {endblockcomment}                             {
949     --comment_level;
950     if (comment_level == 0) {
951       ParserSingleInstance::popControlStatus();
952       yy_pop_state();
953       //return scan_throw(BLOCKCOMMENT);
954     }
955   }
956
957   {startblockcomment}                           {
958     ++comment_level;
959     yy_push_state(REGIONCOMMENT);
960   }
961
962   {newline}                                     {
963     yylloc.last_line += 1;
964     yylloc.last_column = 1;
965     scan_step();
966     *yylval.comment += L"\n//";
967   }
968
969   {char_in_comment}                             |
970   .                                             {
971       wchar_t *pwText = to_wide_string(yytext);
972       *yylval.comment += std::wstring(pwText);
973       FREE(pwText);
974   }
975
976  <<EOF>>                                        {
977       yy_pop_state();
978 //    std::string str = "unexpected end of file in a comment";
979 //    exit_status = SCAN_ERROR;
980 //    scan_error(str);
981   }
982 }
983
984
985 <SIMPLESTRING>
986 {
987   {dquote}{dquote}                              {
988     *pstBuffer += "\"";
989   }
990
991   {dquote}{quote}                               {
992     *pstBuffer += "'";
993   }
994
995   {quote}{dquote}                               {
996     *pstBuffer += "\"";
997   }
998
999   {quote}{quote}                                {
1000     *pstBuffer += "'";
1001   }
1002
1003   {quote}                                       {
1004     yy_pop_state();
1005     scan_step();
1006     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1007     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1008     {
1009         std::string str = "can not convert'";
1010         str += pstBuffer->c_str();
1011         str += "' to UTF-8";
1012         exit_status = SCAN_ERROR;
1013         scan_error("can not convert string to UTF-8");
1014     }
1015     yylval.str = new std::wstring(pwstBuffer);
1016     delete pstBuffer;
1017     FREE(pwstBuffer);
1018     return scan_throw(STR);
1019   }
1020
1021   {dquote}                  {
1022     std::string str = "Heterogeneous string detected, starting with ' and ending with \".";
1023     exit_status = SCAN_ERROR;
1024     scan_error(str);
1025   }
1026
1027   {next}{newline}           {
1028       /* Do nothing... Just skip */
1029   }
1030
1031   {newline}                                     {
1032     std::string str = "unexpected end of line in a string.";
1033     exit_status = SCAN_ERROR;
1034     scan_error(str);
1035     yylloc.last_line += 1;
1036     yylloc.last_column = 1;
1037   }
1038
1039   <<EOF>>                                       {
1040     std::string str = "unexpected end of file in a string.";
1041     exit_status = SCAN_ERROR;
1042     scan_error(str);
1043   }
1044
1045   {in_string}                                           |
1046   .                                                     {
1047     scan_step();
1048     *pstBuffer += yytext;
1049   }
1050 }
1051
1052
1053 <DOUBLESTRING>
1054 {
1055   {dquote}{dquote}                              {
1056     *pstBuffer += "\"";
1057   }
1058
1059   {dquote}{quote}                               {
1060     *pstBuffer += "'";
1061   }
1062
1063   {quote}{dquote}               {
1064     *pstBuffer += "\"";
1065   }
1066
1067   {quote}{quote}                                {
1068     *pstBuffer += "'";
1069   }
1070
1071   {dquote}                      {
1072     yy_pop_state();
1073     scan_step();
1074     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1075     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1076     {
1077         std::string str = "can not convert'";
1078         str += pstBuffer->c_str();
1079         str += "' to UTF-8";
1080         exit_status = SCAN_ERROR;
1081         scan_error("can not convert string to UTF-8");
1082     }
1083     yylval.str = new std::wstring(pwstBuffer);
1084     delete pstBuffer;
1085     FREE(pwstBuffer);
1086     return scan_throw(STR);
1087   }
1088
1089   {quote}                  {
1090     std::string str = "Heterogeneous string detected, starting with \" and ending with '.";
1091     exit_status = SCAN_ERROR;
1092     scan_error(str);
1093   }
1094
1095   {next}{newline}           {
1096       /* Do nothing... Just skip */
1097   }
1098
1099   {newline} {
1100     std::string str = "unexpected end of line in a string";
1101     exit_status = SCAN_ERROR;
1102     scan_error(str);
1103     yylloc.last_line += 1;
1104     yylloc.last_column = 1;
1105   }
1106
1107   <<EOF>>   {
1108     std::string str = "unexpected end of file in a string";
1109     exit_status = SCAN_ERROR;
1110     scan_error(str);
1111   }
1112
1113   {in_string}         |
1114   .                   {
1115    scan_step();
1116    *pstBuffer += yytext;
1117   }
1118 }
1119
1120
1121 <SHELLMODE>
1122 {
1123     {spaces}                    {
1124         if (last_token == ID)
1125         {
1126             scan_throw(SPACES);
1127             return ID;
1128         }
1129     }
1130
1131     {semicolon}                 {
1132         BEGIN(INITIAL);
1133         scan_step();
1134         return scan_throw(SEMI);
1135     }
1136
1137     {comma}                     {
1138         BEGIN(INITIAL);
1139         scan_step();
1140         return scan_throw(COMMA);
1141     }
1142
1143     {newline}                   {
1144         BEGIN(INITIAL);
1145         yylloc.last_line += 1;
1146         yylloc.last_column = 1;
1147         scan_step();
1148         return scan_throw(EOL);
1149     }
1150
1151     {assign} {
1152         if (last_token == STR || last_token == SPACES)
1153         {
1154             wchar_t *pwText = to_wide_string(yytext);
1155             yylval.str = new std::wstring(pwText);
1156             FREE(pwText);
1157             return scan_throw(STR);
1158         }
1159         else
1160         {
1161             BEGIN(INITIAL);
1162             return scan_throw(ASSIGN);
1163         }
1164     }
1165
1166     {lparen} {
1167         if (last_token == STR || last_token == SPACES)
1168         {
1169             wchar_t *pwText = to_wide_string(yytext);
1170             yylval.str = new std::wstring(pwText);
1171             FREE(pwText);
1172             return scan_throw(STR);
1173         }
1174         else
1175         {
1176             BEGIN(INITIAL);
1177             return scan_throw(LPAREN);
1178         }
1179     }
1180
1181     {lowerthan} {
1182         if (last_token == STR || last_token == SPACES)
1183         {
1184             wchar_t *pwText = to_wide_string(yytext);
1185             yylval.str = new std::wstring(pwText);
1186             FREE(pwText);
1187             return scan_throw(STR);
1188         }
1189         else
1190         {
1191             BEGIN(INITIAL);
1192             return scan_throw(LT);
1193         }
1194     }
1195
1196     {greaterthan} {
1197         if (last_token == STR || last_token == SPACES)
1198         {
1199             wchar_t *pwText = to_wide_string(yytext);
1200             yylval.str = new std::wstring(pwText);
1201             FREE(pwText);
1202             return scan_throw(STR);
1203         }
1204         else
1205         {
1206             BEGIN(INITIAL);
1207             return scan_throw(GT);
1208         }
1209     }
1210
1211     {boolnot} {
1212         if (last_token == STR || last_token == SPACES)
1213         {
1214             wchar_t *pwText = to_wide_string(yytext);
1215             yylval.str = new std::wstring(pwText);
1216             FREE(pwText);
1217             return scan_throw(STR);
1218         }
1219         else
1220         {
1221             BEGIN(INITIAL);
1222             return scan_throw(NOT);
1223         }
1224     }
1225
1226
1227     [^ \t\v\f\r\n,;'"]+               {
1228         wchar_t *pwText = to_wide_string(yytext);
1229         yylval.str = new std::wstring(pwText);
1230         FREE(pwText);
1231         return scan_throw(STR);
1232     }
1233
1234     <<EOF>>                     {
1235         BEGIN(INITIAL);
1236     }
1237
1238 }
1239
1240 %%
1241
1242 int scan_throw(int token) {
1243   last_token = token;
1244 #ifdef DEV
1245   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1246 #endif
1247   return token;
1248 }
1249
1250 int get_last_token() {
1251     return last_token;
1252 }
1253
1254 void scan_step() {
1255   yylloc.first_line = yylloc.last_line;
1256   yylloc.first_column = yylloc.last_column;
1257 }
1258
1259 void scan_error(std::string msg)
1260 {
1261     wchar_t* pstMsg = to_wide_string(msg.c_str());
1262
1263     //std::wcerr << pstMsg << std::endl;
1264     ParserSingleInstance::PrintError(pstMsg);
1265     ParserSingleInstance::setExitStatus(Parser::Failed);
1266     ParserSingleInstance::resetControlStatus();
1267     FREE(pstMsg);
1268     last_token = YYEOF;
1269     BEGIN(INITIAL);
1270 }
1271
1272 /*
1273 ** convert floating numbers to C standard
1274 ** 1.2d-3 -> 1.2e-3
1275 ** 1.2D-3 -> 1.2e-3
1276 */
1277 void scan_exponent_convert(char *in)
1278 {
1279   for (; *in != 'd' && *in != 'D'; ++in);
1280   *in = 'e';
1281 }
1282
1283 #ifdef _MSC_VER
1284 int isatty (int desc)
1285 {
1286   return 0;
1287 }
1288 #endif