Revert "FIX Lexer: 2 *... => 2 * ..."
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "sci_malloc.h"
23 }
24
25 static int comment_level = 0;
26 static int last_token = 0;
27 static int exit_status = PARSE_ERROR;
28 static std::string current_file;
29 static std::string program_name;
30
31 static std::string *pstBuffer;
32
33 #define YY_USER_ACTION                          \
34  yylloc.last_column += yyleng;
35
36 /* -*- Verbose Special Debug -*- */
37 //#define DEV
38 //#define TOKENDEV
39
40 #ifdef DEV
41 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
42 #else
43 #define DEBUG(x) /* Nothing */
44 #endif
45
46 %}
47
48 %option stack
49 %option noyywrap
50
51 %x SIMPLESTRING
52 %x DOUBLESTRING
53 %x REGIONCOMMENT
54 %x LINECOMMENT
55 %x LINEBREAK
56
57 %x MATRIX
58
59 %x SHELLMODE
60 %x BEGINID
61
62 spaces                  [ \t\v\f]+
63 integer                 [0-9]+
64 number                  [0-9]+[\.][0-9]*
65 little                  \.[0-9]+
66
67 floating                ({little}|{number}|{integer})[deDE][+-]?{integer}
68
69 hex             [0]x[0-9a-fA-F]+
70 oct             [0]o[0-7]+
71
72
73 utf2            ([\xC2-\xDF][\x80-\xBF])
74 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
75 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
76 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
77 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
78 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
79 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
80 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
81
82 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
83 utf4            ({utf41}|{utf42}|{utf43})
84
85 utf             ({utf2}|{utf3}|{utf4})
86 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
87
88
89 newline                 ("\r"|"\n"|"\r\n")
90 blankline               {spaces}+{newline}
91 emptyline       {newline}({spaces}|[,;])+{newline}
92 next                    \.\.+
93
94 boolnot                 ("@"|"~")
95 booltrue                ("%t"|"%T")
96 boolfalse               ("%f"|"%F")
97 booland                 ("&")
98 boolandand              ("&&")
99 boolor                  ("|")
100 booloror                ("||")
101
102 lbrack                  "["
103 rbrack                  "]"
104
105 lparen                  "("
106 rparen                  ")"
107
108 lbrace                  "{"
109 rbrace                  "}"
110
111 dollar                  "$"
112
113 semicolon               ";"
114 comma                   ","
115 colon                   ":"
116
117 startcomment            "//"
118 startblockcomment       "/*"
119 endblockcomment         "*/"
120
121 dquote                  "\""
122 quote                   "'"
123
124 dot             "."
125 dotquote                ".'"
126 dottimes                ".*"
127 dotrdivide              "./"
128 dotldivide              ".\\"
129 dotpower                (".^"|".**")
130
131 plus                    "+"
132 minus                   "-"
133 rdivide                 "/"
134 ldivide                 "\\"
135 times                   "*"
136 power                   ("^"|"**")
137
138 equal                   "="{spaces}*"="
139 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
140 lowerthan               "<"
141 greaterthan             ">"
142 lowerequal              "<"{spaces}*"="
143 greaterequal            ">"{spaces}*"="
144
145 krontimes               ".*."
146 kronrdivide             "./."
147 kronldivide             ".\\."
148
149 controltimes    ("*."[^0-9])
150 controlrdivide  ("/."[^0-9])
151 controlldivide  ("\\."[^0-9])
152
153 assign                  "="
154
155 %%
156
157 <INITIAL,BEGINID>"if"            {
158         if (last_token != DOT)
159     {
160         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
161     }
162     DEBUG("BEGIN(INITIAL)");
163     BEGIN(INITIAL);
164     return scan_throw(IF);
165 }
166
167 <INITIAL,BEGINID>"then"          {
168     DEBUG("BEGIN(INITIAL)");
169     BEGIN(INITIAL);
170     return scan_throw(THEN);
171 }
172
173 <INITIAL,BEGINID>"else"          {
174         if (last_token != DOT)
175     {
176         // Pop to step out IF
177         ParserSingleInstance::popControlStatus();
178         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
179     }
180     DEBUG("BEGIN(INITIAL)");
181     BEGIN(INITIAL);
182         return scan_throw(ELSE);
183 }
184
185 <INITIAL,BEGINID>"elseif" {
186         if (last_token != DOT)
187     {
188         ParserSingleInstance::popControlStatus();
189         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
190     }
191     DEBUG("BEGIN(INITIAL)");
192     BEGIN(INITIAL);
193         return scan_throw(ELSEIF);
194 }
195
196 <INITIAL,BEGINID>"end"          {
197         if (last_token != DOT)
198     {
199         ParserSingleInstance::popControlStatus();
200     }
201     DEBUG("BEGIN(INITIAL)");
202     BEGIN(INITIAL);
203     return scan_throw(END);
204 }
205
206 <INITIAL,BEGINID>"select"       {
207         if (last_token != DOT)
208     {
209         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
210     }
211     DEBUG("BEGIN(INITIAL)");
212     BEGIN(INITIAL);
213     return scan_throw(SELECT);
214 }
215
216 <INITIAL,BEGINID>"switch"       {
217         if (last_token != DOT)
218     {
219         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
220     }
221     DEBUG("BEGIN(INITIAL)");
222     BEGIN(INITIAL);
223     return scan_throw(SWITCH);
224 }
225
226 <INITIAL,BEGINID>"otherwise" {
227         if (last_token != DOT)
228     {
229         ParserSingleInstance::popControlStatus();
230         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
231     }
232     DEBUG("BEGIN(INITIAL)");
233     BEGIN(INITIAL);
234         return scan_throw(OTHERWISE);
235 }
236
237 <INITIAL,BEGINID>"case"         {
238         if (last_token != DOT)
239     {
240         ParserSingleInstance::popControlStatus();
241         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
242     }
243     DEBUG("BEGIN(INITIAL)");
244     BEGIN(INITIAL);
245     return scan_throw(CASE);
246 }
247
248 <INITIAL,BEGINID>"function" {
249         if (last_token != DOT)
250     {
251         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
252     }
253     DEBUG("BEGIN(INITIAL)");
254     BEGIN(INITIAL);
255     return scan_throw(FUNCTION);
256 }
257
258 <INITIAL,BEGINID>"endfunction" {
259         if (last_token != DOT)
260     {
261         ParserSingleInstance::popControlStatus();
262     }
263     DEBUG("BEGIN(INITIAL)");
264     BEGIN(INITIAL);
265         return scan_throw(ENDFUNCTION);
266 }
267
268 <INITIAL,BEGINID>"#function"    {
269         if (last_token != DOT)
270     {
271         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
272     }
273     DEBUG("BEGIN(INITIAL)");
274     BEGIN(INITIAL);
275         return scan_throw(HIDDENFUNCTION);
276 }
277
278 <INITIAL,BEGINID>"hidden"       {
279     DEBUG("BEGIN(INITIAL)");
280         BEGIN(INITIAL);
281     return scan_throw(HIDDEN);
282 }
283
284 <INITIAL,BEGINID>"for" {
285         if (last_token != DOT)
286     {
287         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
288     }
289     BEGIN(INITIAL);
290     return scan_throw(FOR);
291 }
292
293 <INITIAL,BEGINID>"while"        {
294         if (last_token != DOT)
295     {
296         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
297     }
298         BEGIN(INITIAL);
299         return scan_throw(WHILE);
300 }
301
302 <INITIAL,BEGINID>"do"           {
303         BEGIN(INITIAL);
304     return scan_throw(DO);
305 }
306
307 <INITIAL,BEGINID>"break"                {
308         BEGIN(INITIAL);
309         return scan_throw(BREAK);
310 }
311
312 <INITIAL,BEGINID>"continue"             {
313         BEGIN(INITIAL);
314         return scan_throw(CONTINUE);
315 }
316
317 <INITIAL,BEGINID>"try" {
318         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
319         BEGIN(INITIAL);
320         return scan_throw(TRY);
321 }
322
323 <INITIAL,BEGINID>"catch" {
324     // Pop to step out TRY
325         ParserSingleInstance::popControlStatus();
326         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
327         BEGIN(INITIAL);
328         return scan_throw(CATCH);
329 }
330
331 <INITIAL,BEGINID>"return"       {
332     BEGIN(INITIAL);
333     return scan_throw(RETURN);
334 }
335
336 <INITIAL,BEGINID>"resume"       {
337     BEGIN(INITIAL);
338     return scan_throw(RETURN);
339 }
340
341 ^{spaces}*/({id}){spaces}[^ \t\v\f(=<>~@] {
342         BEGIN(BEGINID);
343 }
344
345 <BEGINID>
346 {
347     {id}                        {
348         wchar_t *pwText = to_wide_string(yytext);
349         if (yytext != NULL && pwText == NULL)
350         {
351             std::string str = "can not convert'";
352             str += yytext;
353             str += "' to UTF-8";
354             exit_status = SCAN_ERROR;
355             scan_error("can not convert string to UTF-8");
356         }
357         yylval.str = new std::wstring(pwText);
358         FREE(pwText);
359         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
360         if (pIT && pIT->isCallable())
361         {
362             scan_throw(ID);
363             BEGIN(SHELLMODE);
364         }
365         else
366         {
367             BEGIN(INITIAL);
368             return scan_throw(ID);
369         }
370     }
371
372 }
373
374 <INITIAL,MATRIX>{boolnot}               {
375   return scan_throw(NOT);
376 }
377 <INITIAL,MATRIX>{dollar}                {
378   return scan_throw(DOLLAR);
379 }
380 <INITIAL,MATRIX>{booltrue}              {
381   return scan_throw(BOOLTRUE);
382 }
383 <INITIAL,MATRIX>{boolfalse}             {
384   return scan_throw(BOOLFALSE);
385 }
386 <INITIAL,MATRIX>{booland}               {
387   return scan_throw(AND);
388 }
389 <INITIAL,MATRIX>{boolandand}    {
390   return scan_throw(ANDAND);
391 }
392 <INITIAL,MATRIX>{boolor}                {
393   return scan_throw(OR);
394 }
395 <INITIAL,MATRIX>{booloror}              {
396   return scan_throw(OROR);
397 }
398
399
400 <INITIAL,MATRIX>{lparen}                {
401   return scan_throw(LPAREN);
402 }
403 <INITIAL,MATRIX>{rparen}                {
404   return scan_throw(RPAREN);
405 }
406
407
408 <INITIAL,MATRIX>{semicolon}             {
409         scan_step();
410   return scan_throw(SEMI);
411 }
412
413 <INITIAL,MATRIX>{comma}                 {
414         scan_step();
415   return scan_throw(COMMA);
416 }
417
418 <INITIAL,MATRIX>{colon}                 {
419   return scan_throw(COLON);
420 }
421
422
423 <INITIAL,MATRIX>{lbrace}                {
424   yy_push_state(MATRIX);
425   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
426   return scan_throw(LBRACE);
427 }
428
429 {rbrace}                        {
430   return scan_throw(RBRACE);
431 }
432
433
434 <INITIAL,MATRIX>{dotquote}              {
435   return scan_throw(DOTQUOTE);
436 }
437 <INITIAL,MATRIX>{dottimes}              {
438   return scan_throw(DOTTIMES);
439 }
440 <INITIAL,MATRIX>{dotrdivide}            {
441   return scan_throw(DOTRDIVIDE);
442 }
443 <INITIAL,MATRIX>{dotldivide}            {
444   return scan_throw(DOTLDIVIDE);
445 }
446 <INITIAL,MATRIX>{dotpower}              {
447   return scan_throw(DOTPOWER);
448 }
449
450
451 {minus}                                 {
452   return scan_throw(MINUS);
453 }
454 {plus}                                  {
455   return scan_throw(PLUS);
456 }
457 <INITIAL,MATRIX>{times}                 {
458   return scan_throw(TIMES);
459 }
460 <INITIAL,MATRIX>{rdivide}               {
461   return scan_throw(RDIVIDE);
462 }
463 <INITIAL,MATRIX>{ldivide}               {
464   return scan_throw(LDIVIDE);
465 }
466 <INITIAL,MATRIX>{power}                 {
467   return scan_throw(POWER);
468 }
469
470 <INITIAL,MATRIX>{krontimes}             {
471   return scan_throw(KRONTIMES);
472 }
473 <INITIAL,MATRIX>{kronrdivide}           {
474   return scan_throw(KRONRDIVIDE);
475 }
476 <INITIAL,MATRIX>{kronldivide}           {
477   return scan_throw(KRONLDIVIDE);
478 }
479
480
481 <INITIAL,MATRIX>{controltimes}          {
482     unput(yytext[yyleng - 1]);
483     return scan_throw(CONTROLTIMES);
484 }
485 <INITIAL,MATRIX>{controlrdivide}                {
486     unput(yytext[yyleng - 1]);
487     return scan_throw(CONTROLRDIVIDE);
488 }
489 <INITIAL,MATRIX>{controlldivide}                {
490     unput(yytext[yyleng - 1]);
491     return scan_throw(CONTROLLDIVIDE);
492 }
493
494
495 <INITIAL,MATRIX>{equal}                 {
496   return scan_throw(EQ);
497 }
498 <INITIAL,MATRIX>{notequal}              {
499   return scan_throw(NE);
500 }
501 <INITIAL,MATRIX>{lowerthan}             {
502   return scan_throw(LT);
503 }
504 <INITIAL,MATRIX>{greaterthan}           {
505   return scan_throw(GT);
506 }
507 <INITIAL,MATRIX>{lowerequal}            {
508   return scan_throw(LE);
509 }
510 <INITIAL,MATRIX>{greaterequal}          {
511   return scan_throw(GE);
512 }
513
514
515 <INITIAL,MATRIX>{assign}                {
516   return scan_throw(ASSIGN);
517  }
518
519
520 <INITIAL,MATRIX>{lbrack}                {
521   DEBUG("yy_push_state(MATRIX)");
522   yy_push_state(MATRIX);
523   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
524   return scan_throw(LBRACK);
525 }
526
527 <INITIAL>{rbrack}                               {
528   return scan_throw(RBRACK);
529 }
530
531
532 <INITIAL,MATRIX>{dot}                   {
533   return scan_throw(DOT);
534 }
535
536 <INITIAL>{next}                 {
537     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
538     yy_push_state(LINEBREAK);
539 }
540
541 <INITIAL,MATRIX>{integer}               {
542   yylval.number = atof(yytext);
543 #ifdef TOKENDEV
544   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
545 #endif
546 //  scan_step();
547   return scan_throw(VARINT);
548 }
549
550
551 <INITIAL,MATRIX>{floating}              {
552   scan_exponent_convert(yytext);
553   yylval.number = atof(yytext);
554 #ifdef TOKENDEV
555   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
556 #endif
557   scan_step();
558   return scan_throw(VARFLOAT);
559 }
560
561
562 <INITIAL,MATRIX>{number}                {
563   yylval.number = atof(yytext);
564 #ifdef TOKENDEV
565   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
566 #endif
567 //  scan_step();
568   return scan_throw(NUM);
569 }
570
571
572 <INITIAL,MATRIX>{little}                {
573   yylval.number = atof(yytext);
574 #ifdef TOKENDEV
575   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
576 #endif
577   scan_step();
578   return scan_throw(NUM);
579 }
580
581
582 <INITIAL,MATRIX>{id}                    {
583     wchar_t *pwText = to_wide_string(yytext);
584     if (yytext != NULL && pwText == NULL)
585     {
586         std::string str = "can not convert'";
587         str += yytext;
588         str += "' to UTF-8";
589         exit_status = SCAN_ERROR;
590         scan_error("can not convert string to UTF-8");
591     }
592     yylval.str = new std::wstring(pwText);
593     FREE(pwText);
594 #ifdef TOKENDEV
595   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
596 #endif
597 //  scan_step();
598   return scan_throw(ID);
599 }
600
601
602 <INITIAL,MATRIX>{startblockcomment}     {
603   yylval.comment = new std::wstring();
604   comment_level = 1;
605   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
606   yy_push_state(REGIONCOMMENT);
607 }
608
609
610 <INITIAL,MATRIX>{startcomment}          {
611   pstBuffer = new std::string();
612   yy_push_state(LINECOMMENT);
613 }
614
615
616 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
617   pstBuffer = new std::string();
618   yy_push_state(DOUBLESTRING);
619 }
620
621
622 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
623   /*
624   ** Matrix Transposition special behaviour
625   ** ID' []' toto()' are transposition call
626   */
627   if (last_token == ID
628       || last_token == RBRACK
629       || last_token == RPAREN
630       || last_token == RBRACE
631       || last_token == VARINT
632       || last_token == VARFLOAT
633       || last_token == NUM
634       || last_token == BOOLTRUE
635       || last_token == BOOLFALSE)
636   {
637       return scan_throw(QUOTE);
638   }
639   else
640   {
641       pstBuffer = new std::string();
642       yy_push_state(SIMPLESTRING);
643   }
644 }
645
646
647 <INITIAL,MATRIX>{spaces}                {
648   scan_step();
649   scan_throw(SPACES);
650 }
651
652
653 <INITIAL>{newline}              {
654   yylloc.last_line += 1;
655   yylloc.last_column = 1;
656   scan_step();
657   if (last_token != EOL) {
658       return scan_throw(EOL);
659   }
660
661 }
662
663
664 <INITIAL,MATRIX>{blankline}             {
665   yylloc.last_line += 1;
666   yylloc.last_column = 1;
667   scan_step();
668   if (last_token != EOL)
669   {
670       return scan_throw(EOL);
671   }
672   scan_throw(EOL);
673 }
674
675 <INITIAL,MATRIX>{emptyline}             {
676   yylloc.last_line += 2;
677   yylloc.last_column = 1;
678   scan_step();
679   if (last_token != EOL)
680   {
681       return scan_throw(EOL);
682   }
683   scan_throw(EOL);
684 }
685 .                                       {
686     std::string str = "unexpected token '";
687     str += yytext;
688     str += "'";
689     exit_status = SCAN_ERROR;
690     scan_error(str);
691 }
692
693
694 <MATRIX>
695 {
696   {spaces}*{lparen} {
697       unput(yytext[yyleng -1]);
698       if (last_token == ID
699           || last_token == RPAREN
700           || last_token == QUOTE
701           || last_token == VARINT
702           || last_token == VARFLOAT
703           || last_token == NUM)
704       {
705           return scan_throw(COMMA);
706       }
707   }
708
709   {spaces}*{colon}{spaces}* {
710       return scan_throw(COLON);
711   }
712
713   {newline} {
714       yylloc.last_line += 1;
715       yylloc.last_column = 1;
716       if(last_token != DOTS && last_token != EOL)
717       {
718           return scan_throw(EOL);
719       }
720       scan_throw(EOL);
721   }
722
723
724   {rbrack}                              {
725     DEBUG("yy_pop_state()");
726     yy_pop_state();
727     ParserSingleInstance::popControlStatus();
728     return scan_throw(RBRACK);
729   }
730
731   {rbrace}                              {
732     yy_pop_state();
733     ParserSingleInstance::popControlStatus();
734     return scan_throw(RBRACE);
735   }
736
737   {plus}                                |
738   {spaces}{plus}{spaces}                {
739     return scan_throw(PLUS);
740   }
741
742   {minus}                               |
743   {spaces}{minus}{spaces}               {
744     return scan_throw(MINUS);
745   }
746
747   {spaces}{plus}                        {
748     // no need to unput the '+'
749     if (last_token != LBRACK
750        && last_token != EOL
751        && last_token != SEMI
752        && last_token != COMMA)
753    {
754        return scan_throw(COMMA);
755    }  
756   }
757  
758   {spaces}{minus}                       {
759     unput('-');
760     if (last_token != LBRACK
761        && last_token != EOL
762        && last_token != SEMI
763        && last_token != COMMA)
764    {
765        return scan_throw(COMMA);
766    }  
767   }
768
769   .                                     {
770     std::string str = "unexpected token '";
771     str += yytext;
772     str += "' within a matrix.";
773     exit_status = SCAN_ERROR;
774     scan_error(str);
775   }
776
777   {next}{spaces}*{newline}          {
778       /* Just do nothing */
779       yylloc.last_line += 1;
780       yylloc.last_column = 1;
781       scan_step();
782       scan_throw(EOL);
783   }
784
785   {next}{spaces}*{startcomment}          {
786       /* Just do nothing */
787       pstBuffer = new std::string();
788       yy_push_state(LINECOMMENT);
789       scan_throw(DOTS);
790   }
791
792   <<EOF>>       {
793       yy_pop_state();
794   }
795 }
796
797 <LINEBREAK>
798 {
799   {newline}                             {
800     yylloc.last_line += 1;
801     yylloc.last_column = 1;
802     scan_step();
803     yy_pop_state();
804     ParserSingleInstance::popControlStatus();
805   }
806
807   {startblockcomment}                   {
808     ++comment_level;
809     yy_push_state(REGIONCOMMENT);
810   }
811
812   {startcomment}                        {
813     scan_throw(DOTS);
814     pstBuffer = new std::string();
815     yy_push_state(LINECOMMENT);
816   }
817
818   {spaces}                              {
819       /* Do nothing... */
820   }
821
822   <<EOF>>       {
823       yy_pop_state();
824   }
825   .                                     {
826     std::string str = "unexpected token '";
827     str += yytext;
828     str += "' after line break with .. or ...";
829     exit_status = SCAN_ERROR;
830     scan_error(str);
831   }
832 }
833
834
835 <LINECOMMENT>
836 {
837   {newline}     {
838     //yylloc.last_line += 1;
839     //yylloc.last_column = 1;
840     //scan_step();
841     yy_pop_state();
842     for (int i = yyleng - 1 ; i >= 0 ; --i)
843     {
844         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
845         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
846         unput(yytext[i]);
847         yylloc.last_column--;
848     }
849     /*
850     ** To forgot comments after lines break
851     */
852     if (last_token != DOTS)
853     {
854         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
855         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
856         wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
857         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
858         if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
859         {
860             std::string str = "can not convert'";
861             str += pstBuffer->c_str();
862             str += "' to UTF-8";
863             exit_status = SCAN_ERROR;
864             scan_error("can not convert string to UTF-8");
865         }
866         yylval.comment = new std::wstring(pwstBuffer);
867         delete pstBuffer;
868         FREE (pwstBuffer);
869         return scan_throw(COMMENT);
870     }
871   }
872
873   <<EOF>>       {
874     yy_pop_state();
875     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
876     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
877     {
878         std::string str = "can not convert'";
879         str += pstBuffer->c_str();
880         str += "' to UTF-8";
881         exit_status = SCAN_ERROR;
882         scan_error("can not convert string to UTF-8");
883     }
884     yylval.comment = new std::wstring(pwstBuffer);
885     delete pstBuffer;
886     FREE (pwstBuffer);
887     return scan_throw(COMMENT);
888   }
889
890   .         {
891      // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
892      // only translate to WCHAR_T when popping state.
893      *pstBuffer += yytext;
894   }
895
896 }
897
898
899 <REGIONCOMMENT>
900 {
901   {endblockcomment}                             {
902     --comment_level;
903     if (comment_level == 0) {
904       ParserSingleInstance::popControlStatus();
905       yy_pop_state();
906       //return scan_throw(BLOCKCOMMENT);
907     }
908   }
909
910   {startblockcomment}                           {
911     ++comment_level;
912     yy_push_state(REGIONCOMMENT);
913   }
914
915   {newline}                                     {
916     yylloc.last_line += 1;
917     yylloc.last_column = 1;
918     scan_step();
919     *yylval.comment += L"\n//";
920   }
921
922   .                                             {
923       wchar_t *pwText = to_wide_string(yytext);
924       *yylval.comment += std::wstring(pwText);
925       FREE(pwText);
926   }
927
928  <<EOF>>                                        {
929       yy_pop_state();
930 //    std::string str = "unexpected end of file in a comment";
931 //    exit_status = SCAN_ERROR;
932 //    scan_error(str);
933   }
934 }
935
936
937 <SIMPLESTRING>
938 {
939   {dquote}{dquote}                              {
940     *pstBuffer += "\"";
941   }
942
943   {dquote}{quote}                               {
944     *pstBuffer += "'";
945   }
946
947   {quote}{dquote}                               {
948     *pstBuffer += "\"";
949   }
950
951   {quote}{quote}                                {
952     *pstBuffer += "'";
953   }
954
955   {quote}                                       {
956     yy_pop_state();
957     scan_step();
958     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
959     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
960     {
961         std::string str = "can not convert'";
962         str += pstBuffer->c_str();
963         str += "' to UTF-8";
964         exit_status = SCAN_ERROR;
965         scan_error("can not convert string to UTF-8");
966     }
967     yylval.str = new std::wstring(pwstBuffer);
968     delete pstBuffer;
969     FREE(pwstBuffer);
970     return scan_throw(STR);
971   }
972
973   {dquote}                  {
974     std::string str = "Heterogeneous string detected, starting with ' and ending with \".";
975     exit_status = SCAN_ERROR;
976     scan_error(str);
977   }
978
979   {next}{newline}           {
980       /* Do nothing... Just skip */
981   }
982
983   {newline}                                     {
984     std::string str = "unexpected end of line in a string.";
985     exit_status = SCAN_ERROR;
986     scan_error(str);
987     yylloc.last_line += 1;
988     yylloc.last_column = 1;
989   }
990
991   <<EOF>>                                       {
992     std::string str = "unexpected end of file in a string.";
993     exit_status = SCAN_ERROR;
994     scan_error(str);
995   }
996
997   .                                             {
998     scan_step();
999     *pstBuffer += yytext;
1000   }
1001 }
1002
1003
1004 <DOUBLESTRING>
1005 {
1006   {dquote}{dquote}                              {
1007     *pstBuffer += "\"";
1008   }
1009
1010   {dquote}{quote}                               {
1011     *pstBuffer += "'";
1012   }
1013
1014   {quote}{dquote}               {
1015     *pstBuffer += "\"";
1016   }
1017
1018   {quote}{quote}                                {
1019     *pstBuffer += "'";
1020   }
1021
1022   {dquote}                      {
1023     yy_pop_state();
1024     scan_step();
1025     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1026     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1027     {
1028         std::string str = "can not convert'";
1029         str += pstBuffer->c_str();
1030         str += "' to UTF-8";
1031         exit_status = SCAN_ERROR;
1032         scan_error("can not convert string to UTF-8");
1033     }
1034     yylval.str = new std::wstring(pwstBuffer);
1035     delete pstBuffer;
1036     FREE(pwstBuffer);
1037     return scan_throw(STR);
1038   }
1039
1040   {quote}                  {
1041     std::string str = "Heterogeneous string detected, starting with \" and ending with '.";
1042     exit_status = SCAN_ERROR;
1043     scan_error(str);
1044   }
1045
1046   {next}{newline}           {
1047       /* Do nothing... Just skip */
1048   }
1049
1050   {newline} {
1051     std::string str = "unexpected end of line in a string";
1052     exit_status = SCAN_ERROR;
1053     scan_error(str);
1054     yylloc.last_line += 1;
1055     yylloc.last_column = 1;
1056   }
1057
1058   <<EOF>>   {
1059     std::string str = "unexpected end of file in a string";
1060     exit_status = SCAN_ERROR;
1061     scan_error(str);
1062   }
1063
1064   .         {
1065    scan_step();
1066    *pstBuffer += yytext;
1067   }
1068 }
1069
1070
1071 <SHELLMODE>
1072 {
1073     {spaces}                    {
1074         if (last_token == ID)
1075         {
1076             scan_throw(SPACES);
1077             return ID;
1078         }
1079     }
1080
1081     {semicolon}                 {
1082         BEGIN(INITIAL);
1083         scan_step();
1084         return scan_throw(SEMI);
1085     }
1086
1087     {comma}                     {
1088         BEGIN(INITIAL);
1089         scan_step();
1090         return scan_throw(COMMA);
1091     }
1092
1093     {newline}                   {
1094         BEGIN(INITIAL);
1095         yylloc.last_line += 1;
1096         yylloc.last_column = 1;
1097         scan_step();
1098         return scan_throw(EOL);
1099     }
1100
1101     {assign} {
1102         if (last_token == STR)
1103         {
1104             wchar_t *pwText = to_wide_string(yytext);
1105             yylval.str = new std::wstring(pwText);
1106             FREE(pwText);
1107             return scan_throw(STR);
1108         }
1109         else
1110         {
1111             BEGIN(INITIAL);
1112             return scan_throw(ASSIGN);
1113         }
1114     }
1115
1116     {lparen} {
1117         if (last_token == STR)
1118         {
1119             wchar_t *pwText = to_wide_string(yytext);
1120             yylval.str = new std::wstring(pwText);
1121             FREE(pwText);
1122             return scan_throw(STR);
1123         }
1124         else
1125         {
1126             BEGIN(INITIAL);
1127             return scan_throw(LPAREN);
1128         }
1129     }
1130
1131     {lowerthan} {
1132         if (last_token == STR)
1133         {
1134             wchar_t *pwText = to_wide_string(yytext);
1135             yylval.str = new std::wstring(pwText);
1136             FREE(pwText);
1137             return scan_throw(STR);
1138         }
1139         else
1140         {
1141             BEGIN(INITIAL);
1142             return scan_throw(LT);
1143         }
1144     }
1145
1146     {greaterthan} {
1147         if (last_token == STR)
1148         {
1149             wchar_t *pwText = to_wide_string(yytext);
1150             yylval.str = new std::wstring(pwText);
1151             FREE(pwText);
1152             return scan_throw(STR);
1153         }
1154         else
1155         {
1156             BEGIN(INITIAL);
1157             return scan_throw(GT);
1158         }
1159     }
1160
1161     {boolnot} {
1162         if (last_token == STR)
1163         {
1164             wchar_t *pwText = to_wide_string(yytext);
1165             yylval.str = new std::wstring(pwText);
1166             FREE(pwText);
1167             return scan_throw(STR);
1168         }
1169         else
1170         {
1171             BEGIN(INITIAL);
1172             return scan_throw(NOT);
1173         }
1174     }
1175
1176
1177     [^ \t\v\f\r\n,;'"]+               {
1178         wchar_t *pwText = to_wide_string(yytext);
1179         yylval.str = new std::wstring(pwText);
1180         FREE(pwText);
1181         return scan_throw(STR);
1182     }
1183
1184     <<EOF>>                     {
1185         BEGIN(INITIAL);
1186     }
1187
1188 }
1189
1190 %%
1191
1192 int scan_throw(int token) {
1193   last_token = token;
1194 #ifdef DEV
1195   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1196 #endif
1197   return token;
1198 }
1199
1200 int get_last_token() {
1201     return last_token;
1202 }
1203
1204 void scan_step() {
1205   yylloc.first_line = yylloc.last_line;
1206   yylloc.first_column = yylloc.last_column;
1207 }
1208
1209 void scan_error(std::string msg)
1210 {
1211     wchar_t* pstMsg = to_wide_string(msg.c_str());
1212
1213     //std::wcerr << pstMsg << std::endl;
1214     ParserSingleInstance::PrintError(pstMsg);
1215     ParserSingleInstance::setExitStatus(Parser::Failed);
1216     ParserSingleInstance::resetControlStatus();
1217     FREE(pstMsg);
1218     last_token = YYEOF;
1219     BEGIN(INITIAL);
1220 }
1221
1222 /*
1223 ** convert floating numbers to C standard
1224 ** 1.2d-3 -> 1.2e-3
1225 ** 1.2D-3 -> 1.2e-3
1226 */
1227 void scan_exponent_convert(char *in)
1228 {
1229   char *pString;
1230   while((pString=strpbrk(in,"d"))!=NULL)
1231     {
1232       *pString='e';
1233     }
1234   while((pString=strpbrk(in,"D"))!=NULL)
1235     {
1236       *pString='e';
1237     }
1238 }
1239
1240 #ifdef _MSC_VER
1241 int isatty (int desc)
1242 {
1243   return 0;
1244 }
1245 #endif