* Bug #9560 fixed - (1./M) was parsed as (1. / M) instead of (1 ./ M)
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  * Copyright (C) 2012 - 2016 - Scilab Enterprises
7  *
8  * This file is hereby licensed under the terms of the GNU GPL v2.0,
9  * pursuant to article 5.3.4 of the CeCILL v.2.1.
10  * This file was originally licensed under the terms of the CeCILL v2.1,
11  * and continues to be available under such terms.
12  * For more information, see the COPYING file which you should have received
13  * along with this program.
14  * === LICENSE_END ===
15  *
16  */
17
18 #include <stack>
19
20 #include "isatty.hxx"
21 #include "parse.hxx"
22 #include "parser_private.hxx"
23
24 #include "context.hxx"
25
26 extern "C"
27 {
28 #include "charEncoding.h"
29 #include "sci_malloc.h"
30 }
31
32 static std::stack<int> paren_levels;
33
34 static int comment_level = 0;
35 static int last_token = 0;
36 static int exit_status = PARSE_ERROR;
37 static int str_opener_column = 0;
38 static std::string current_file;
39 static std::string program_name;
40
41 static std::string pstBuffer;
42
43 extern void yyerror(std::string);
44
45 #define YY_USER_ACTION                          \
46     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
47 //yylloc.last_column += yyleng;
48
49 /* -*- Verbose Special Debug -*- */
50 //#define DEV
51 //#define TOKENDEV
52
53 #ifdef DEV
54 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
55 #else
56 #define DEBUG(x) /* Nothing */
57 #endif
58
59 %}
60
61 %option stack
62 %option noyywrap
63
64 %x SIMPLESTRING
65 %x DOUBLESTRING
66 %x REGIONCOMMENT
67 %x LINECOMMENT
68 %x LINEBREAK
69
70 %x MATRIX
71
72 %x SHELLMODE
73 %x BEGINID
74
75 spaces                  [ \t\v\f]+
76 integer                 [0-9]+
77 number                  [0-9]+[\.][0-9]*
78 little                  \.[0-9]+
79 bom             \xEF\xBB\xBF
80
81 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
82 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
83
84 hex             [0]x[0-9a-fA-F]+
85 oct             [0]o[0-7]+
86
87
88 utf2            ([\xC2-\xDF][\x80-\xBF])
89 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
90 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
91 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
92 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
93 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
94 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
95 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
96
97 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
98 utf4            ({utf41}|{utf42}|{utf43})
99
100 utf             ({utf2}|{utf3}|{utf4})
101 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
102
103
104 newline                 ("\r"|"\n"|"\r\n")
105 blankline               {spaces}+{newline}
106 emptyline       {newline}({spaces}|[,;])+{newline}
107 next                    \.\.+
108 char_in_line_comment    [^\r\n]*
109 char_in_comment         [^\r\n\/*]*
110
111 boolnot                 ("@"|"~")
112 booltrue                ("%t"|"%T")
113 boolfalse               ("%f"|"%F")
114 booland                 ("&")
115 boolandand              ("&&")
116 boolor                  ("|")
117 booloror                ("||")
118
119 lbrack                  "["
120 rbrack                  "]"
121
122 lparen                  "("
123 rparen                  ")"
124
125 lbrace                  "{"
126 rbrace                  "}"
127
128 dollar                  "$"
129
130 semicolon               ";"
131 comma                   ","
132 colon                   ":"
133
134 startcomment            "//"
135 startblockcomment       "/*"
136 endblockcomment         "*/"
137
138 dquote                  "\""
139 quote                   "'"
140 in_string               [^\"\'\r\n\.]*
141
142 dot             "."
143 dotquote                ".'"
144 dottimes                ".*"
145 dotrdivide              "./"
146 dotldivide              ".\\"
147 dotpower                (".^"|".**")
148
149 plus                    "+"
150 minus                   "-"
151 rdivide                 "/"
152 ldivide                 "\\"
153 times                   "*"
154 power                   ("^"|"**")
155
156 equal                   "="{spaces}*"="
157 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
158 lowerthan               "<"
159 greaterthan             ">"
160 lowerequal              "<"{spaces}*"="
161 greaterequal            ">"{spaces}*"="
162
163 krontimes               ".*."
164 kronrdivide             "./."
165 kronldivide             ".\\."
166
167 controltimes    ("*."[^0-9])
168 controlrdivide  ("/."[^0-9])
169 controlldivide  ("\\."[^0-9])
170
171 assign                  "="
172
173 %%
174 <INITIAL>{bom}/.* {
175 // BOM found ==> ignored
176 }
177
178 <INITIAL,BEGINID>{booltrue}     {
179     BEGIN(INITIAL);
180     return scan_throw(BOOLTRUE);
181 }
182 <INITIAL,BEGINID>{boolfalse}    {
183     BEGIN(INITIAL);
184     return scan_throw(BOOLFALSE);
185 }
186
187 <INITIAL,BEGINID>"if"            {
188         if (last_token != DOT)
189     {
190         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
191     }
192     DEBUG("BEGIN(INITIAL)");
193     BEGIN(INITIAL);
194     return scan_throw(IF);
195 }
196
197 <INITIAL,BEGINID>"then"          {
198     DEBUG("BEGIN(INITIAL)");
199     BEGIN(INITIAL);
200     return scan_throw(THEN);
201 }
202
203 <INITIAL,BEGINID>"else"          {
204         if (last_token != DOT)
205     {
206         // Pop to step out IF
207         ParserSingleInstance::popControlStatus();
208         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
209     }
210     DEBUG("BEGIN(INITIAL)");
211     BEGIN(INITIAL);
212         return scan_throw(ELSE);
213 }
214
215 <INITIAL,BEGINID>"elseif" {
216         if (last_token != DOT)
217     {
218         ParserSingleInstance::popControlStatus();
219         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
220     }
221     DEBUG("BEGIN(INITIAL)");
222     BEGIN(INITIAL);
223         return scan_throw(ELSEIF);
224 }
225
226 <INITIAL,BEGINID>"end"          {
227         if (last_token != DOT)
228     {
229         ParserSingleInstance::popControlStatus();
230     }
231     DEBUG("BEGIN(INITIAL)");
232     BEGIN(INITIAL);
233     return scan_throw(END);
234 }
235
236 <INITIAL,BEGINID>"select"       {
237         if (last_token != DOT)
238     {
239         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
240     }
241     DEBUG("BEGIN(INITIAL)");
242     BEGIN(INITIAL);
243     return scan_throw(SELECT);
244 }
245
246 <INITIAL,BEGINID>"switch"       {
247         if (last_token != DOT)
248     {
249         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
250     }
251     DEBUG("BEGIN(INITIAL)");
252     BEGIN(INITIAL);
253     return scan_throw(SWITCH);
254 }
255
256 <INITIAL,BEGINID>"otherwise" {
257         if (last_token != DOT)
258     {
259         ParserSingleInstance::popControlStatus();
260         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
261     }
262     DEBUG("BEGIN(INITIAL)");
263     BEGIN(INITIAL);
264         return scan_throw(OTHERWISE);
265 }
266
267 <INITIAL,BEGINID>"case"         {
268         if (last_token != DOT)
269     {
270         ParserSingleInstance::popControlStatus();
271         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
272     }
273     DEBUG("BEGIN(INITIAL)");
274     BEGIN(INITIAL);
275     return scan_throw(CASE);
276 }
277
278 <INITIAL,BEGINID>"function" {
279         if (last_token != DOT)
280     {
281         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
282     }
283     DEBUG("BEGIN(INITIAL)");
284     BEGIN(INITIAL);
285     return scan_throw(FUNCTION);
286 }
287
288 <INITIAL,BEGINID>"endfunction" {
289         if (last_token != DOT)
290     {
291         ParserSingleInstance::popControlStatus();
292     }
293     DEBUG("BEGIN(INITIAL)");
294     BEGIN(INITIAL);
295         return scan_throw(ENDFUNCTION);
296 }
297
298 <INITIAL,BEGINID>"for" {
299         if (last_token != DOT)
300     {
301         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
302     }
303     BEGIN(INITIAL);
304     return scan_throw(FOR);
305 }
306
307 <INITIAL,BEGINID>"while"        {
308         if (last_token != DOT)
309     {
310         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
311     }
312         BEGIN(INITIAL);
313         return scan_throw(WHILE);
314 }
315
316 <INITIAL,BEGINID>"do"           {
317         BEGIN(INITIAL);
318     return scan_throw(DO);
319 }
320
321 <INITIAL,BEGINID>"break"                {
322         BEGIN(INITIAL);
323         return scan_throw(BREAK);
324 }
325
326 <INITIAL,BEGINID>"continue"             {
327         BEGIN(INITIAL);
328         return scan_throw(CONTINUE);
329 }
330
331 <INITIAL,BEGINID>"try" {
332         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
333         BEGIN(INITIAL);
334         return scan_throw(TRY);
335 }
336
337 <INITIAL,BEGINID>"catch" {
338     // Pop to step out TRY
339         ParserSingleInstance::popControlStatus();
340         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
341         BEGIN(INITIAL);
342         return scan_throw(CATCH);
343 }
344
345 <INITIAL,BEGINID>"return"       {
346     BEGIN(INITIAL);
347     return scan_throw(RETURN);
348 }
349
350 <INITIAL,BEGINID>"resume"       {
351     BEGIN(INITIAL);
352     return scan_throw(RETURN);
353 }
354
355 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
356         BEGIN(BEGINID);
357 }
358
359 <BEGINID>
360 {
361     {id}                        {
362         wchar_t *pwText = to_wide_string(yytext);
363         if (yytext != NULL && pwText == NULL)
364         {
365             std::string str = "Can\'t convert \'";
366             str += yytext;
367             str += "\' to UTF-8";
368             BEGIN(INITIAL);
369             yyerror(str);
370             return scan_throw(FLEX_ERROR);
371         }
372         yylval.str = new std::wstring(pwText);
373         FREE(pwText);
374         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
375         if (pIT && pIT->isCallable())
376         {
377             scan_throw(ID);
378             BEGIN(SHELLMODE);
379         }
380         else
381         {
382             BEGIN(INITIAL);
383             return scan_throw(ID);
384         }
385     }
386
387 }
388
389 <INITIAL,MATRIX>{boolnot}               {
390   return scan_throw(NOT);
391 }
392 <INITIAL,MATRIX>{dollar}                {
393   return scan_throw(DOLLAR);
394 }
395 <INITIAL,MATRIX>{booltrue}              {
396   return scan_throw(BOOLTRUE);
397 }
398 <INITIAL,MATRIX>{boolfalse}             {
399   return scan_throw(BOOLFALSE);
400 }
401 <INITIAL,MATRIX>{booland}               {
402   return scan_throw(AND);
403 }
404 <INITIAL,MATRIX>{boolandand}    {
405   return scan_throw(ANDAND);
406 }
407 <INITIAL,MATRIX>{boolor}                {
408   return scan_throw(OR);
409 }
410 <INITIAL,MATRIX>{booloror}              {
411   return scan_throw(OROR);
412 }
413
414
415 <INITIAL>{lparen}                       {
416   return scan_throw(LPAREN);
417 }
418 <INITIAL>{rparen}                       {
419   return scan_throw(RPAREN);
420 }
421
422
423 <INITIAL,MATRIX>{semicolon}             {
424         scan_step();
425   return scan_throw(SEMI);
426 }
427
428 <INITIAL,MATRIX>{comma}                 {
429         scan_step();
430   return scan_throw(COMMA);
431 }
432
433 <INITIAL,MATRIX>{colon}                 {
434   return scan_throw(COLON);
435 }
436
437
438 <INITIAL,MATRIX>{lbrace}                {
439   yy_push_state(MATRIX);
440   paren_levels.push(0);
441   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
442   return scan_throw(LBRACE);
443 }
444
445 {rbrace}                        {
446   return scan_throw(RBRACE);
447 }
448
449
450 <INITIAL,MATRIX>{dotquote}              {
451   return scan_throw(DOTQUOTE);
452 }
453 <INITIAL,MATRIX>{dottimes}              {
454   return scan_throw(DOTTIMES);
455 }
456 <INITIAL,MATRIX>{dotrdivide}            {
457   return scan_throw(DOTRDIVIDE);
458 }
459 <INITIAL,MATRIX>{dotldivide}            {
460   return scan_throw(DOTLDIVIDE);
461 }
462 <INITIAL,MATRIX>{dotpower}              {
463   return scan_throw(DOTPOWER);
464 }
465
466
467 {minus}                                 {
468   return scan_throw(MINUS);
469 }
470 {plus}                                  {
471   return scan_throw(PLUS);
472 }
473 <INITIAL,MATRIX>{times}                 {
474   return scan_throw(TIMES);
475 }
476 <INITIAL,MATRIX>{rdivide}               {
477   return scan_throw(RDIVIDE);
478 }
479 <INITIAL,MATRIX>{ldivide}               {
480   return scan_throw(LDIVIDE);
481 }
482 <INITIAL,MATRIX>{power}                 {
483   return scan_throw(POWER);
484 }
485
486 <INITIAL,MATRIX>{krontimes}             {
487   return scan_throw(KRONTIMES);
488 }
489 <INITIAL,MATRIX>{kronrdivide}           {
490   return scan_throw(KRONRDIVIDE);
491 }
492 <INITIAL,MATRIX>{kronldivide}           {
493   return scan_throw(KRONLDIVIDE);
494 }
495
496
497 <INITIAL,MATRIX>{controltimes}          {
498     unput(yytext[yyleng - 1]);
499     return scan_throw(CONTROLTIMES);
500 }
501 <INITIAL,MATRIX>{controlrdivide}                {
502     unput(yytext[yyleng - 1]);
503     return scan_throw(CONTROLRDIVIDE);
504 }
505 <INITIAL,MATRIX>{controlldivide}                {
506     unput(yytext[yyleng - 1]);
507     return scan_throw(CONTROLLDIVIDE);
508 }
509
510
511 <INITIAL,MATRIX>{equal}                 {
512   return scan_throw(EQ);
513 }
514 <INITIAL,MATRIX>{notequal}              {
515   return scan_throw(NE);
516 }
517 <INITIAL,MATRIX>{lowerthan}             {
518   return scan_throw(LT);
519 }
520 <INITIAL,MATRIX>{greaterthan}           {
521   return scan_throw(GT);
522 }
523 <INITIAL,MATRIX>{lowerequal}            {
524   return scan_throw(LE);
525 }
526 <INITIAL,MATRIX>{greaterequal}          {
527   return scan_throw(GE);
528 }
529
530
531 <INITIAL,MATRIX>{assign}                {
532   return scan_throw(ASSIGN);
533  }
534
535
536 <INITIAL,MATRIX>{lbrack}                {
537   DEBUG("yy_push_state(MATRIX)");
538   yy_push_state(MATRIX);
539   paren_levels.push(0);
540   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
541   return scan_throw(LBRACK);
542 }
543
544 <INITIAL>{rbrack}                               {
545   return scan_throw(RBRACK);
546 }
547
548
549 <INITIAL,MATRIX>{dot}                   {
550   return scan_throw(DOT);
551 }
552
553 <INITIAL>{next}                 {
554     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
555     yy_push_state(LINEBREAK);
556 }
557
558 <INITIAL,MATRIX>{integer}               {
559   yylval.number = atof(yytext);
560 #ifdef TOKENDEV
561   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
562 #endif
563 //  scan_step();
564   return scan_throw(VARINT);
565 }
566
567
568 <INITIAL,MATRIX>{floating_D}            {
569   scan_exponent_convert(yytext);
570   yylval.number = atof(yytext);
571 #ifdef TOKENDEV
572   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
573 #endif
574   //scan_step();
575   return scan_throw(VARFLOAT);
576 }
577
578 <INITIAL,MATRIX>{floating_E}            {
579   yylval.number = atof(yytext);
580 #ifdef TOKENDEV
581   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
582 #endif
583   //scan_step();
584   return scan_throw(VARFLOAT);
585 }
586
587 <INITIAL,MATRIX>[0-9]+[\.]/[\*^\\\/]            {
588
589   yylval.number = atof(yytext);
590 #ifdef TOKENDEV
591   std::cout << "--> [DEBUG] NUMBER WITH DOT AS LAST CHARACTER : " << yytext << std::endl;
592 #endif
593 //  scan_step();
594   unput('.');
595   yylloc.last_column--;
596   return scan_throw(NUM);
597 }
598
599 <INITIAL,MATRIX>{number}                {
600   yylval.number = atof(yytext);
601 #ifdef TOKENDEV
602   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
603 #endif
604 //  scan_step();
605   return scan_throw(NUM);
606 }
607
608
609 <INITIAL,MATRIX>{little}                {
610   yylval.number = atof(yytext);
611 #ifdef TOKENDEV
612   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
613 #endif
614 //  scan_step();
615   return scan_throw(NUM);
616 }
617
618
619 <INITIAL,MATRIX>{id}                    {
620     wchar_t *pwText = to_wide_string(yytext);
621     if (yytext != NULL && pwText == NULL)
622     {
623         std::string str = "Can\'t convert \'";
624         str += yytext;
625         str += "\' to UTF-8";
626         BEGIN(INITIAL);
627         yyerror(str);
628         return scan_throw(FLEX_ERROR);
629     }
630     yylval.str = new std::wstring(pwText);
631     FREE(pwText);
632 #ifdef TOKENDEV
633   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
634 #endif
635 //  scan_step();
636   return scan_throw(ID);
637 }
638
639
640 <INITIAL,MATRIX>{startblockcomment}     {
641   yylval.comment = new std::wstring();
642   comment_level = 1;
643   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
644   yy_push_state(REGIONCOMMENT);
645 }
646
647
648 <INITIAL,MATRIX>{startcomment}          {
649   pstBuffer.clear();
650   yy_push_state(LINECOMMENT);
651 }
652
653
654 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
655   pstBuffer.clear();
656   str_opener_column = yylloc.first_column;
657   yy_push_state(DOUBLESTRING);
658 }
659
660
661 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
662   /*
663   ** Matrix Transposition special behaviour
664   ** ID' []' toto()' are transposition call
665   */
666   if (last_token == ID
667       || last_token == RBRACK
668       || last_token == RPAREN
669       || last_token == RBRACE
670       || last_token == VARINT
671       || last_token == VARFLOAT
672       || last_token == NUM
673       || last_token == BOOLTRUE
674       || last_token == BOOLFALSE)
675   {
676       return scan_throw(QUOTE);
677   }
678   else
679   {
680       pstBuffer.clear();
681       str_opener_column = yylloc.first_column;
682       yy_push_state(SIMPLESTRING);
683   }
684 }
685
686
687 <INITIAL,MATRIX>{spaces}                {
688         scan_step();
689         scan_throw(SPACES);
690 }
691
692
693 <INITIAL>{newline}              {
694   yylloc.last_line += 1;
695   yylloc.last_column = 1;
696   scan_step();
697   if (last_token != EOL) {
698       return scan_throw(EOL);
699   }
700
701 }
702
703
704 <INITIAL,MATRIX>{blankline}             {
705   yylloc.last_line += 1;
706   yylloc.last_column = 1;
707   scan_step();
708   if (last_token != EOL)
709   {
710       return scan_throw(EOL);
711   }
712   scan_throw(EOL);
713 }
714
715 <INITIAL,MATRIX>{emptyline}             {
716   yylloc.last_line += 2;
717   yylloc.last_column = 1;
718   scan_step();
719   if (last_token != EOL)
720   {
721       return scan_throw(EOL);
722   }
723   scan_throw(EOL);
724 }
725 .                                       {
726     std::string str = "Unexpected token \'";
727     str += yytext;
728     str += "\'";
729     BEGIN(INITIAL);
730     yyerror(str);
731     return scan_throw(FLEX_ERROR);
732 }
733
734
735 <MATRIX>
736 {
737   {lparen} {
738     ++paren_levels.top();
739     return scan_throw(LPAREN);
740   }
741
742   {rparen} {
743     --paren_levels.top();
744     return scan_throw(RPAREN);
745   }
746
747   {spaces}*{lparen} {
748       unput(yytext[yyleng -1]);
749       if (last_token == ID
750           || last_token == RPAREN
751           || last_token == QUOTE
752           || last_token == VARINT
753           || last_token == VARFLOAT
754           || last_token == NUM)
755       {
756           return scan_throw(COMMA);
757       }
758   }
759
760   {spaces}*{colon}{spaces}* {
761       return scan_throw(COLON);
762   }
763
764   {newline} {
765       yylloc.last_line += 1;
766       yylloc.last_column = 1;
767       if(last_token != DOTS && last_token != EOL)
768       {
769           return scan_throw(EOL);
770       }
771       scan_throw(EOL);
772   }
773
774   {rbrack}                              {
775     DEBUG("yy_pop_state()");
776     yy_pop_state();
777     paren_levels.pop();
778     ParserSingleInstance::popControlStatus();
779     return scan_throw(RBRACK);
780   }
781
782   {rbrace}                              {
783     yy_pop_state();
784     paren_levels.pop();
785     ParserSingleInstance::popControlStatus();
786     return scan_throw(RBRACE);
787   }
788
789   {plus}                                |
790   {spaces}{plus}{spaces}                {
791     return scan_throw(PLUS);
792   }
793
794   {minus}                               |
795   {spaces}{minus}{spaces}               {
796     return scan_throw(MINUS);
797   }
798
799   {spaces}{plus}                        {
800     // no need to unput the '+'
801     if (last_token != LBRACK
802        && last_token != EOL
803        && last_token != SEMI
804        && last_token != COMMA
805        && last_token != DOTTIMES
806        && last_token != DOTRDIVIDE
807        && last_token != DOTLDIVIDE
808        && last_token != DOTPOWER
809        && last_token != MINUS
810        && last_token != PLUS
811        && last_token != TIMES
812        && last_token != RDIVIDE
813        && last_token != LDIVIDE
814        && last_token != POWER
815        && last_token != KRONTIMES
816        && last_token != KRONRDIVIDE
817        && last_token != KRONLDIVIDE
818        && last_token != EQ
819        && last_token != NE
820        && last_token != LT
821        && last_token != GT
822        && last_token != LE
823        && last_token != GE
824       && paren_levels.top() == 0)
825    {
826        return scan_throw(COMMA);
827    }
828    else
829    {
830        unput('+');
831        yylloc.last_column--;
832    }
833   }
834
835   {spaces}{minus}                       {
836     unput('-');
837     yylloc.last_column--;
838     if (last_token != LBRACK
839        && last_token != EOL
840        && last_token != SEMI
841        && last_token != COMMA
842        && last_token != DOTTIMES
843        && last_token != DOTRDIVIDE
844        && last_token != DOTLDIVIDE
845        && last_token != DOTPOWER
846        && last_token != MINUS
847        && last_token != PLUS
848        && last_token != TIMES
849        && last_token != RDIVIDE
850        && last_token != LDIVIDE
851        && last_token != POWER
852        && last_token != KRONTIMES
853        && last_token != KRONRDIVIDE
854        && last_token != KRONLDIVIDE
855        && last_token != EQ
856        && last_token != NE
857        && last_token != LT
858        && last_token != GT
859        && last_token != LE
860        && last_token != GE
861        && paren_levels.top() == 0)
862    {
863        return scan_throw(COMMA);
864    }
865   }
866
867   .                                     {
868     std::string str = "Unexpected token \'";
869     str += yytext;
870     str += "\' within a matrix.";
871     BEGIN(INITIAL);
872     yyerror(str);
873     return scan_throw(FLEX_ERROR);
874   }
875
876   {next}{spaces}*{newline}          {
877       yylloc.last_line += 1;
878       yylloc.last_column = 1;
879       scan_step();
880   }
881
882   {next}{spaces}*{startcomment}.*{newline}          {
883       yylloc.last_line += 1;
884       yylloc.last_column = 1;
885       scan_step();
886   }
887
888   {spaces}{next}{spaces}*{newline}          {
889       yylloc.last_line += 1;
890       yylloc.last_column = 1;
891       scan_step();
892       unput(' ');
893       yylloc.last_column--;
894   }
895
896   {spaces}{next}{spaces}*{startcomment}.*{newline}          {
897       yylloc.last_line += 1;
898       yylloc.last_column = 1;
899       scan_step();
900       unput(' ');
901       yylloc.last_column--;
902   }
903
904   <<EOF>>       {
905       yy_pop_state();
906       paren_levels.pop();
907   }
908 }
909
910 <LINEBREAK>
911 {
912   {newline}                             {
913     yylloc.last_line += 1;
914     yylloc.last_column = 1;
915     scan_step();
916     yy_pop_state();
917     ParserSingleInstance::popControlStatus();
918   }
919
920   {startblockcomment}                   {
921     ++comment_level;
922     yy_push_state(REGIONCOMMENT);
923   }
924
925   {startcomment}                        {
926     scan_throw(DOTS);
927     pstBuffer.clear();
928     yy_push_state(LINECOMMENT);
929   }
930
931   {spaces}                              {
932       /* Do nothing... */
933   }
934
935   <<EOF>>       {
936       yy_pop_state();
937   }
938   .                                     {
939     std::string str = "Unexpected token \'";
940     str += yytext;
941     str += "\' after line break with .. or ...";
942     BEGIN(INITIAL);
943     yyerror(str);
944     return scan_throw(FLEX_ERROR);
945   }
946 }
947
948
949 <LINECOMMENT>
950 {
951   {newline}     {
952     //yylloc.last_line += 1;
953     //yylloc.last_column = 1;
954     //scan_step();
955     yy_pop_state();
956     for (int i = yyleng - 1 ; i >= 0 ; --i)
957     {
958         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
959         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
960         unput(yytext[i]);
961         yylloc.last_column--;
962     }
963     /*
964     ** To forgot comments after lines break
965     */
966     if (last_token != DOTS)
967     {
968         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
969         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
970         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
971         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
972         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
973         {
974             pstBuffer.clear();
975             std::string str = "Can\'t convert \'";
976             str += pstBuffer.c_str();
977             str += "\' to UTF-8";
978             BEGIN(INITIAL);
979             yyerror(str);
980             return scan_throw(FLEX_ERROR);
981         }
982         yylval.comment = new std::wstring(pwstBuffer);
983         pstBuffer.clear();
984         FREE (pwstBuffer);
985         return scan_throw(COMMENT);
986     }
987     else
988     {
989         pstBuffer.clear();
990     }
991   }
992
993   <<EOF>>       {
994     yy_pop_state();
995     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
996     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
997     {
998         pstBuffer.clear();
999         std::string str = "Can\'t convert \'";
1000         str += pstBuffer.c_str();
1001         str += "\' to UTF-8";
1002         BEGIN(INITIAL);
1003         yyerror(str);
1004         return scan_throw(FLEX_ERROR);
1005     }
1006     yylval.comment = new std::wstring(pwstBuffer);
1007     pstBuffer.clear();
1008     FREE (pwstBuffer);
1009     return scan_throw(COMMENT);
1010   }
1011
1012   {char_in_line_comment}         {
1013       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1014       // only translate to WCHAR_T when popping state.
1015       pstBuffer += yytext;
1016   }
1017 }
1018
1019
1020 <REGIONCOMMENT>
1021 {
1022   {endblockcomment}                             {
1023     --comment_level;
1024     if (comment_level == 0) {
1025       ParserSingleInstance::popControlStatus();
1026       yy_pop_state();
1027       //return scan_throw(BLOCKCOMMENT);
1028     }
1029   }
1030
1031   {startblockcomment}                           {
1032     ++comment_level;
1033     yy_push_state(REGIONCOMMENT);
1034   }
1035
1036   {newline}                                     {
1037     yylloc.last_line += 1;
1038     yylloc.last_column = 1;
1039     scan_step();
1040     *yylval.comment += L"\n//";
1041   }
1042
1043   {char_in_comment}                             |
1044   .                                             {
1045       wchar_t *pwText = to_wide_string(yytext);
1046       *yylval.comment += std::wstring(pwText);
1047       FREE(pwText);
1048   }
1049
1050  <<EOF>>                                        {
1051       yy_pop_state();
1052 //    std::string str = "unexpected end of file in a comment";
1053 //    scan_error(str);
1054   }
1055 }
1056
1057
1058 <SIMPLESTRING>
1059 {
1060   {dquote}{dquote}                              {
1061     pstBuffer += "\"";
1062   }
1063
1064   {dquote}{quote}                               {
1065     pstBuffer += "'";
1066   }
1067
1068   {quote}{dquote}                               {
1069     pstBuffer += "\"";
1070   }
1071
1072   {quote}{quote}                                {
1073     pstBuffer += "'";
1074   }
1075
1076   {quote}                                       {
1077     yy_pop_state();
1078     //scan_step();
1079     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1080     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1081     {
1082         pstBuffer.clear();
1083         std::string str = "Can\'t convert \'";
1084         str += pstBuffer.c_str();
1085         str += "\' to UTF-8";
1086         BEGIN(INITIAL);
1087         yyerror(str);
1088         return scan_throw(FLEX_ERROR);
1089     }
1090     yylval.str = new std::wstring(pwstBuffer);
1091     pstBuffer.clear();
1092     FREE(pwstBuffer);
1093     yylloc.first_column = str_opener_column;
1094     return scan_throw(STR);
1095   }
1096
1097   {dquote}                  {
1098     pstBuffer.clear();
1099     BEGIN(INITIAL);
1100     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1101     return scan_throw(FLEX_ERROR);
1102   }
1103
1104   {next}{newline}           {
1105       yylloc.last_line += 1;
1106       yylloc.last_column = 1;
1107       /* Do nothing... Just skip */
1108   }
1109
1110   {newline}                                     {
1111     pstBuffer.clear();
1112     yylloc.last_line += 1;
1113     yylloc.last_column = 1;
1114     BEGIN(INITIAL);
1115     yyerror("Unexpected end of line in a string.");
1116     return scan_throw(FLEX_ERROR);
1117   }
1118
1119   <<EOF>>                                       {
1120     pstBuffer.clear();
1121     BEGIN(INITIAL);
1122     yyerror("Unexpected end of file in a string.");
1123     return scan_throw(FLEX_ERROR);
1124   }
1125
1126   {in_string}                                           |
1127   .                                                     {
1128     //scan_step();
1129     pstBuffer += yytext;
1130   }
1131 }
1132
1133
1134 <DOUBLESTRING>
1135 {
1136   {dquote}{dquote}                              {
1137     pstBuffer += "\"";
1138   }
1139
1140   {dquote}{quote}                               {
1141     pstBuffer += "'";
1142   }
1143
1144   {quote}{dquote}               {
1145     pstBuffer += "\"";
1146   }
1147
1148   {quote}{quote}                                {
1149     pstBuffer += "'";
1150   }
1151
1152   {dquote}                      {
1153     yy_pop_state();
1154     //scan_step();
1155     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1156     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1157     {
1158         pstBuffer.clear();
1159         std::string str = "Can\'t convert \'";
1160         str += pstBuffer.c_str();
1161         str += "\' to UTF-8";
1162         BEGIN(INITIAL);
1163         yyerror(str);
1164         return scan_throw(FLEX_ERROR);
1165     }
1166     yylval.str = new std::wstring(pwstBuffer);
1167     pstBuffer.clear();
1168     FREE(pwstBuffer);
1169     yylloc.first_column = str_opener_column;
1170     return scan_throw(STR);
1171   }
1172
1173   {quote}                  {
1174     pstBuffer.clear();
1175     BEGIN(INITIAL);
1176     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1177     return scan_throw(FLEX_ERROR);
1178   }
1179
1180   {next}{newline}           {
1181       yylloc.last_line += 1;
1182       yylloc.last_column = 1;
1183       /* Do nothing... Just skip */
1184   }
1185
1186   {newline} {
1187     pstBuffer.clear();
1188     yylloc.last_line += 1;
1189     yylloc.last_column = 1;
1190     BEGIN(INITIAL);
1191     yyerror("Unexpected end of line in a string.");
1192     return scan_throw(FLEX_ERROR);
1193   }
1194
1195   <<EOF>>   {
1196     pstBuffer.clear();
1197     BEGIN(INITIAL);
1198     yyerror("Unexpected end of file in a string.");
1199     return scan_throw(FLEX_ERROR);
1200   }
1201
1202   {in_string}         |
1203   .                   {
1204    //scan_step();
1205    pstBuffer += yytext;
1206   }
1207 }
1208
1209
1210 <SHELLMODE>
1211 {
1212     {spaces}                    {
1213         if (last_token == ID)
1214         {
1215             scan_throw(SPACES);
1216             return ID;
1217         }
1218     }
1219
1220     {semicolon}                 {
1221         BEGIN(INITIAL);
1222         scan_step();
1223         return scan_throw(SEMI);
1224     }
1225
1226     {comma}                     {
1227         BEGIN(INITIAL);
1228         scan_step();
1229         return scan_throw(COMMA);
1230     }
1231
1232     {newline}                   {
1233         BEGIN(INITIAL);
1234         yylloc.last_line += 1;
1235         yylloc.last_column = 1;
1236         scan_step();
1237         return scan_throw(EOL);
1238     }
1239
1240     {assign} {
1241         if (last_token == STR || last_token == SPACES)
1242         {
1243             wchar_t *pwText = to_wide_string(yytext);
1244             yylval.str = new std::wstring(pwText);
1245             FREE(pwText);
1246             return scan_throw(STR);
1247         }
1248         else
1249         {
1250             BEGIN(INITIAL);
1251             return scan_throw(ASSIGN);
1252         }
1253     }
1254
1255     {lparen} {
1256         if (last_token == STR || last_token == SPACES)
1257         {
1258             wchar_t *pwText = to_wide_string(yytext);
1259             yylval.str = new std::wstring(pwText);
1260             FREE(pwText);
1261             return scan_throw(STR);
1262         }
1263         else
1264         {
1265             BEGIN(INITIAL);
1266             return scan_throw(LPAREN);
1267         }
1268     }
1269
1270     {lowerthan} {
1271         if (last_token == STR || last_token == SPACES)
1272         {
1273             wchar_t *pwText = to_wide_string(yytext);
1274             yylval.str = new std::wstring(pwText);
1275             FREE(pwText);
1276             return scan_throw(STR);
1277         }
1278         else
1279         {
1280             BEGIN(INITIAL);
1281             return scan_throw(LT);
1282         }
1283     }
1284
1285     {greaterthan} {
1286         if (last_token == STR || last_token == SPACES)
1287         {
1288             wchar_t *pwText = to_wide_string(yytext);
1289             yylval.str = new std::wstring(pwText);
1290             FREE(pwText);
1291             return scan_throw(STR);
1292         }
1293         else
1294         {
1295             BEGIN(INITIAL);
1296             return scan_throw(GT);
1297         }
1298     }
1299
1300     {boolnot} {
1301         if (last_token == STR || last_token == SPACES)
1302         {
1303             wchar_t *pwText = to_wide_string(yytext);
1304             yylval.str = new std::wstring(pwText);
1305             FREE(pwText);
1306             return scan_throw(STR);
1307         }
1308         else
1309         {
1310             BEGIN(INITIAL);
1311             return scan_throw(NOT);
1312         }
1313     }
1314
1315
1316     [^ \t\v\f\r\n,;'"]+               {
1317         wchar_t *pwText = to_wide_string(yytext);
1318         yylval.str = new std::wstring(pwText);
1319         FREE(pwText);
1320         return scan_throw(STR);
1321     }
1322
1323     <<EOF>>                     {
1324         BEGIN(INITIAL);
1325     }
1326
1327 }
1328
1329 %%
1330
1331 int scan_throw(int token) {
1332   last_token = token;
1333 #ifdef DEV
1334   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1335 #endif
1336   return token;
1337 }
1338
1339 int get_last_token() {
1340     return last_token;
1341 }
1342
1343 void scan_step() {
1344   yylloc.first_line = yylloc.last_line;
1345   yylloc.first_column = yylloc.last_column;
1346 }
1347
1348 /*
1349 ** convert floating numbers to C standard
1350 ** 1.2d-3 -> 1.2e-3
1351 ** 1.2D-3 -> 1.2e-3
1352 */
1353 void scan_exponent_convert(char *in)
1354 {
1355   for (; *in != 'd' && *in != 'D'; ++in);
1356   *in = 'e';
1357 }
1358
1359 #ifdef _MSC_VER
1360 int isatty (int desc)
1361 {
1362   return 0;
1363 }
1364 #endif