e9520f57ed7f123c2005a057fea47bcf2af762d0
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  * Copyright (C) 2012 - 2016 - Scilab Enterprises
7  *
8  * This file is hereby licensed under the terms of the GNU GPL v2.0,
9  * pursuant to article 5.3.4 of the CeCILL v.2.1.
10  * This file was originally licensed under the terms of the CeCILL v2.1,
11  * and continues to be available under such terms.
12  * For more information, see the COPYING file which you should have received
13  * along with this program.
14  * === LICENSE_END ===
15  *
16  */
17
18 #include <stack>
19
20 #include "isatty.hxx"
21 #include "parse.hxx"
22 #include "parser_private.hxx"
23
24 #include "context.hxx"
25
26 extern "C"
27 {
28 #include "charEncoding.h"
29 #include "sci_malloc.h"
30 }
31
32 static std::stack<int> paren_levels;
33
34 static int comment_level = 0;
35 static int last_token = 0;
36 static int exit_status = PARSE_ERROR;
37 static int str_opener_column = 0;
38 static std::string current_file;
39 static std::string program_name;
40
41 static std::string pstBuffer;
42
43 extern void yyerror(std::string);
44
45 #define YY_USER_ACTION                          \
46     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
47 //yylloc.last_column += yyleng;
48
49 /* -*- Verbose Special Debug -*- */
50 //#define DEV
51 //#define TOKENDEV
52
53 #ifdef DEV
54 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
55 #else
56 #define DEBUG(x) /* Nothing */
57 #endif
58
59 %}
60
61 %option stack
62 %option noyywrap
63
64 %x SIMPLESTRING
65 %x DOUBLESTRING
66 %x REGIONCOMMENT
67 %x LINECOMMENT
68 %x LINEBREAK
69
70 %x MATRIX
71
72 %x SHELLMODE
73 %x BEGINID
74
75 spaces                  [ \t\v\f]+
76 integer                 [0-9]+
77 number                  [0-9]+[\.][0-9]*
78 little                  \.[0-9]+
79 bom             \xEF\xBB\xBF
80
81 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
82 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
83
84 hex             [0]x[0-9a-fA-F]+
85 oct             [0]o[0-7]+
86
87
88 utf2            ([\xC2-\xDF][\x80-\xBF])
89 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
90 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
91 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
92 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
93 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
94 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
95 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
96
97 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
98 utf4            ({utf41}|{utf42}|{utf43})
99
100 utf             ({utf2}|{utf3}|{utf4})
101 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
102
103
104 newline                 ("\r"|"\n"|"\r\n")
105 blankline               {spaces}+{newline}
106 emptyline       {newline}({spaces}|[,;])+{newline}
107 next                    \.\.+
108 char_in_line_comment    [^\r\n]*
109 char_in_comment         [^\r\n\/*]*
110
111 boolnot                 ("@"|"~")
112 booltrue                ("%t"|"%T")
113 boolfalse               ("%f"|"%F")
114 booland                 ("&")
115 boolandand              ("&&")
116 boolor                  ("|")
117 booloror                ("||")
118
119 lbrack                  "["
120 rbrack                  "]"
121
122 lparen                  "("
123 rparen                  ")"
124
125 lbrace                  "{"
126 rbrace                  "}"
127
128 dollar                  "$"
129
130 semicolon               ";"
131 comma                   ","
132 colon                   ":"
133
134 startcomment            "//"
135 startblockcomment       "/*"
136 endblockcomment         "*/"
137
138 dquote                  "\""
139 quote                   "'"
140 in_string               [^\"\'\r\n\.]*
141
142 dot             "."
143 dotquote                ".'"
144 dottimes                ".*"
145 dotrdivide              "./"
146 dotldivide              ".\\"
147 dotpower                (".^"|".**")
148
149 plus                    "+"
150 minus                   "-"
151 rdivide                 "/"
152 ldivide                 "\\"
153 times                   "*"
154 power                   ("^"|"**")
155
156 equal                   "="{spaces}*"="
157 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
158 lowerthan               "<"
159 greaterthan             ">"
160 lowerequal              "<"{spaces}*"="
161 greaterequal            ">"{spaces}*"="
162
163 krontimes               ".*."
164 kronrdivide             "./."
165 kronldivide             ".\\."
166
167 controltimes    ("*."[^0-9])
168 controlrdivide  ("/."[^0-9])
169 controlldivide  ("\\."[^0-9])
170
171 assign                  "="
172
173 %%
174 <INITIAL>{bom}/.* {
175 // BOM found ==> ignored
176 }
177
178 <INITIAL,BEGINID>{booltrue}     {
179     BEGIN(INITIAL);
180     return scan_throw(BOOLTRUE);
181 }
182 <INITIAL,BEGINID>{boolfalse}    {
183     BEGIN(INITIAL);
184     return scan_throw(BOOLFALSE);
185 }
186
187 <INITIAL,BEGINID>"if"            {
188         if (last_token != DOT)
189     {
190         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
191     }
192     DEBUG("BEGIN(INITIAL)");
193     BEGIN(INITIAL);
194     return scan_throw(IF);
195 }
196
197 <INITIAL,BEGINID>"then"          {
198     DEBUG("BEGIN(INITIAL)");
199     BEGIN(INITIAL);
200     return scan_throw(THEN);
201 }
202
203 <INITIAL,BEGINID>"else"          {
204         if (last_token != DOT)
205     {
206         // Pop to step out IF
207         ParserSingleInstance::popControlStatus();
208         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
209     }
210     DEBUG("BEGIN(INITIAL)");
211     BEGIN(INITIAL);
212         return scan_throw(ELSE);
213 }
214
215 <INITIAL,BEGINID>"elseif" {
216         if (last_token != DOT)
217     {
218         ParserSingleInstance::popControlStatus();
219         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
220     }
221     DEBUG("BEGIN(INITIAL)");
222     BEGIN(INITIAL);
223         return scan_throw(ELSEIF);
224 }
225
226 <INITIAL,BEGINID>"end"          {
227         if (last_token != DOT)
228     {
229         ParserSingleInstance::popControlStatus();
230     }
231     DEBUG("BEGIN(INITIAL)");
232     BEGIN(INITIAL);
233     return scan_throw(END);
234 }
235
236 <INITIAL,BEGINID>"select"       {
237         if (last_token != DOT)
238     {
239         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
240     }
241     DEBUG("BEGIN(INITIAL)");
242     BEGIN(INITIAL);
243     return scan_throw(SELECT);
244 }
245
246 <INITIAL,BEGINID>"switch"       {
247         if (last_token != DOT)
248     {
249         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
250     }
251     DEBUG("BEGIN(INITIAL)");
252     BEGIN(INITIAL);
253     return scan_throw(SWITCH);
254 }
255
256 <INITIAL,BEGINID>"otherwise" {
257         if (last_token != DOT)
258     {
259         ParserSingleInstance::popControlStatus();
260         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
261     }
262     DEBUG("BEGIN(INITIAL)");
263     BEGIN(INITIAL);
264         return scan_throw(OTHERWISE);
265 }
266
267 <INITIAL,BEGINID>"case"         {
268         if (last_token != DOT)
269     {
270         ParserSingleInstance::popControlStatus();
271         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
272     }
273     DEBUG("BEGIN(INITIAL)");
274     BEGIN(INITIAL);
275     return scan_throw(CASE);
276 }
277
278 <INITIAL,BEGINID>"function" {
279         if (last_token != DOT)
280     {
281         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
282     }
283     DEBUG("BEGIN(INITIAL)");
284     BEGIN(INITIAL);
285     return scan_throw(FUNCTION);
286 }
287
288 <INITIAL,BEGINID>"endfunction" {
289         if (last_token != DOT)
290     {
291         ParserSingleInstance::popControlStatus();
292     }
293     DEBUG("BEGIN(INITIAL)");
294     BEGIN(INITIAL);
295         return scan_throw(ENDFUNCTION);
296 }
297
298 <INITIAL,BEGINID>"for" {
299         if (last_token != DOT)
300     {
301         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
302     }
303     BEGIN(INITIAL);
304     return scan_throw(FOR);
305 }
306
307 <INITIAL,BEGINID>"while"        {
308         if (last_token != DOT)
309     {
310         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
311     }
312         BEGIN(INITIAL);
313         return scan_throw(WHILE);
314 }
315
316 <INITIAL,BEGINID>"do"           {
317         BEGIN(INITIAL);
318     return scan_throw(DO);
319 }
320
321 <INITIAL,BEGINID>"break"                {
322         BEGIN(INITIAL);
323         return scan_throw(BREAK);
324 }
325
326 <INITIAL,BEGINID>"continue"             {
327         BEGIN(INITIAL);
328         return scan_throw(CONTINUE);
329 }
330
331 <INITIAL,BEGINID>"try" {
332         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
333         BEGIN(INITIAL);
334         return scan_throw(TRY);
335 }
336
337 <INITIAL,BEGINID>"catch" {
338     // Pop to step out TRY
339         ParserSingleInstance::popControlStatus();
340         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
341         BEGIN(INITIAL);
342         return scan_throw(CATCH);
343 }
344
345 <INITIAL,BEGINID>"return"       {
346     BEGIN(INITIAL);
347     return scan_throw(RETURN);
348 }
349
350 <INITIAL,BEGINID>"resume"       {
351     BEGIN(INITIAL);
352     return scan_throw(RETURN);
353 }
354
355 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
356         BEGIN(BEGINID);
357 }
358
359 <BEGINID>
360 {
361     {id}                        {
362         wchar_t *pwText = to_wide_string(yytext);
363         if (yytext != NULL && pwText == NULL)
364         {
365             std::string str = "Can\'t convert \'";
366             str += yytext;
367             str += "\' to UTF-8";
368             BEGIN(INITIAL);
369             yyerror(str);
370             return scan_throw(FLEX_ERROR);
371         }
372         yylval.str = new std::wstring(pwText);
373         FREE(pwText);
374         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
375         if (pIT && pIT->isCallable())
376         {
377             scan_throw(ID);
378             BEGIN(SHELLMODE);
379         }
380         else
381         {
382             BEGIN(INITIAL);
383             return scan_throw(ID);
384         }
385     }
386
387 }
388
389 <INITIAL,MATRIX>{boolnot}               {
390   return scan_throw(NOT);
391 }
392 <INITIAL,MATRIX>{dollar}                {
393   return scan_throw(DOLLAR);
394 }
395 <INITIAL,MATRIX>{booltrue}              {
396   return scan_throw(BOOLTRUE);
397 }
398 <INITIAL,MATRIX>{boolfalse}             {
399   return scan_throw(BOOLFALSE);
400 }
401 <INITIAL,MATRIX>{booland}               {
402   return scan_throw(AND);
403 }
404 <INITIAL,MATRIX>{boolandand}    {
405   return scan_throw(ANDAND);
406 }
407 <INITIAL,MATRIX>{boolor}                {
408   return scan_throw(OR);
409 }
410 <INITIAL,MATRIX>{booloror}              {
411   return scan_throw(OROR);
412 }
413
414
415 <INITIAL>{lparen}                       {
416   return scan_throw(LPAREN);
417 }
418 <INITIAL>{rparen}                       {
419   return scan_throw(RPAREN);
420 }
421
422
423 <INITIAL,MATRIX>{semicolon}             {
424         scan_step();
425   return scan_throw(SEMI);
426 }
427
428 <INITIAL,MATRIX>{comma}                 {
429         scan_step();
430   return scan_throw(COMMA);
431 }
432
433 <INITIAL,MATRIX>{colon}                 {
434   return scan_throw(COLON);
435 }
436
437
438 <INITIAL,MATRIX>{lbrace}                {
439   yy_push_state(MATRIX);
440   paren_levels.push(0);
441   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
442   return scan_throw(LBRACE);
443 }
444
445 {rbrace}                        {
446   return scan_throw(RBRACE);
447 }
448
449
450 <INITIAL,MATRIX>{dotquote}              {
451   return scan_throw(DOTQUOTE);
452 }
453 <INITIAL,MATRIX>{dottimes}              {
454   return scan_throw(DOTTIMES);
455 }
456 <INITIAL,MATRIX>{dotrdivide}            {
457   return scan_throw(DOTRDIVIDE);
458 }
459 <INITIAL,MATRIX>{dotldivide}            {
460   return scan_throw(DOTLDIVIDE);
461 }
462 <INITIAL,MATRIX>{dotpower}              {
463   return scan_throw(DOTPOWER);
464 }
465
466
467 {minus}                                 {
468   return scan_throw(MINUS);
469 }
470 {plus}                                  {
471   return scan_throw(PLUS);
472 }
473 <INITIAL,MATRIX>{times}                 {
474   return scan_throw(TIMES);
475 }
476 <INITIAL,MATRIX>{rdivide}               {
477   return scan_throw(RDIVIDE);
478 }
479 <INITIAL,MATRIX>{ldivide}               {
480   return scan_throw(LDIVIDE);
481 }
482 <INITIAL,MATRIX>{power}                 {
483   return scan_throw(POWER);
484 }
485
486 <INITIAL,MATRIX>{krontimes}             {
487   return scan_throw(KRONTIMES);
488 }
489 <INITIAL,MATRIX>{kronrdivide}           {
490   return scan_throw(KRONRDIVIDE);
491 }
492 <INITIAL,MATRIX>{kronldivide}           {
493   return scan_throw(KRONLDIVIDE);
494 }
495
496
497 <INITIAL,MATRIX>{controltimes}          {
498     unput(yytext[yyleng - 1]);
499     return scan_throw(CONTROLTIMES);
500 }
501 <INITIAL,MATRIX>{controlrdivide}                {
502     unput(yytext[yyleng - 1]);
503     return scan_throw(CONTROLRDIVIDE);
504 }
505 <INITIAL,MATRIX>{controlldivide}                {
506     unput(yytext[yyleng - 1]);
507     return scan_throw(CONTROLLDIVIDE);
508 }
509
510
511 <INITIAL,MATRIX>{equal}                 {
512   return scan_throw(EQ);
513 }
514 <INITIAL,MATRIX>{notequal}              {
515   return scan_throw(NE);
516 }
517 <INITIAL,MATRIX>{lowerthan}             {
518   return scan_throw(LT);
519 }
520 <INITIAL,MATRIX>{greaterthan}           {
521   return scan_throw(GT);
522 }
523 <INITIAL,MATRIX>{lowerequal}            {
524   return scan_throw(LE);
525 }
526 <INITIAL,MATRIX>{greaterequal}          {
527   return scan_throw(GE);
528 }
529
530
531 <INITIAL,MATRIX>{assign}                {
532   return scan_throw(ASSIGN);
533  }
534
535
536 <INITIAL,MATRIX>{lbrack}                {
537   DEBUG("yy_push_state(MATRIX)");
538   yy_push_state(MATRIX);
539   paren_levels.push(0);
540   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
541   return scan_throw(LBRACK);
542 }
543
544 <INITIAL>{rbrack}                               {
545   return scan_throw(RBRACK);
546 }
547
548
549 <INITIAL,MATRIX>{dot}                   {
550   return scan_throw(DOT);
551 }
552
553 <INITIAL>{next}                 {
554     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
555     yy_push_state(LINEBREAK);
556 }
557
558 <INITIAL,MATRIX>{integer}               {
559   yylval.number = atof(yytext);
560 #ifdef TOKENDEV
561   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
562 #endif
563 //  scan_step();
564   return scan_throw(VARINT);
565 }
566
567
568 <INITIAL,MATRIX>{floating_D}            {
569   scan_exponent_convert(yytext);
570   yylval.number = atof(yytext);
571 #ifdef TOKENDEV
572   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
573 #endif
574   //scan_step();
575   return scan_throw(VARFLOAT);
576 }
577
578 <INITIAL,MATRIX>{floating_E}            {
579   yylval.number = atof(yytext);
580 #ifdef TOKENDEV
581   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
582 #endif
583   //scan_step();
584   return scan_throw(VARFLOAT);
585 }
586
587
588 <INITIAL,MATRIX>{number}                {
589   yylval.number = atof(yytext);
590 #ifdef TOKENDEV
591   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
592 #endif
593 //  scan_step();
594   return scan_throw(NUM);
595 }
596
597
598 <INITIAL,MATRIX>{little}                {
599   yylval.number = atof(yytext);
600 #ifdef TOKENDEV
601   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
602 #endif
603 //  scan_step();
604   return scan_throw(NUM);
605 }
606
607
608 <INITIAL,MATRIX>{id}                    {
609     wchar_t *pwText = to_wide_string(yytext);
610     if (yytext != NULL && pwText == NULL)
611     {
612         std::string str = "Can\'t convert \'";
613         str += yytext;
614         str += "\' to UTF-8";
615         BEGIN(INITIAL);
616         yyerror(str);
617         return scan_throw(FLEX_ERROR);
618     }
619     yylval.str = new std::wstring(pwText);
620     FREE(pwText);
621 #ifdef TOKENDEV
622   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
623 #endif
624 //  scan_step();
625   return scan_throw(ID);
626 }
627
628
629 <INITIAL,MATRIX>{startblockcomment}     {
630   yylval.comment = new std::wstring();
631   comment_level = 1;
632   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
633   yy_push_state(REGIONCOMMENT);
634 }
635
636
637 <INITIAL,MATRIX>{startcomment}          {
638   pstBuffer.clear();
639   yy_push_state(LINECOMMENT);
640 }
641
642
643 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
644   pstBuffer.clear();
645   str_opener_column = yylloc.first_column;
646   yy_push_state(DOUBLESTRING);
647 }
648
649
650 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
651   /*
652   ** Matrix Transposition special behaviour
653   ** ID' []' toto()' are transposition call
654   */
655   if (last_token == ID
656       || last_token == RBRACK
657       || last_token == RPAREN
658       || last_token == RBRACE
659       || last_token == VARINT
660       || last_token == VARFLOAT
661       || last_token == NUM
662       || last_token == BOOLTRUE
663       || last_token == BOOLFALSE)
664   {
665       return scan_throw(QUOTE);
666   }
667   else
668   {
669       pstBuffer.clear();
670       str_opener_column = yylloc.first_column;
671       yy_push_state(SIMPLESTRING);
672   }
673 }
674
675
676 <INITIAL,MATRIX>{spaces}                {
677         scan_step();
678         scan_throw(SPACES);
679 }
680
681
682 <INITIAL>{newline}              {
683   yylloc.last_line += 1;
684   yylloc.last_column = 1;
685   scan_step();
686   if (last_token != EOL) {
687       return scan_throw(EOL);
688   }
689
690 }
691
692
693 <INITIAL,MATRIX>{blankline}             {
694   yylloc.last_line += 1;
695   yylloc.last_column = 1;
696   scan_step();
697   if (last_token != EOL)
698   {
699       return scan_throw(EOL);
700   }
701   scan_throw(EOL);
702 }
703
704 <INITIAL,MATRIX>{emptyline}             {
705   yylloc.last_line += 2;
706   yylloc.last_column = 1;
707   scan_step();
708   if (last_token != EOL)
709   {
710       return scan_throw(EOL);
711   }
712   scan_throw(EOL);
713 }
714 .                                       {
715     std::string str = "Unexpected token \'";
716     str += yytext;
717     str += "\'";
718     BEGIN(INITIAL);
719     yyerror(str);
720     return scan_throw(FLEX_ERROR);
721 }
722
723
724 <MATRIX>
725 {
726   {lparen} {
727     ++paren_levels.top();
728     return scan_throw(LPAREN);
729   }
730
731   {rparen} {
732     --paren_levels.top();
733     return scan_throw(RPAREN);
734   }
735
736   {spaces}*{lparen} {
737       unput(yytext[yyleng -1]);
738       if (last_token == ID
739           || last_token == RPAREN
740           || last_token == QUOTE
741           || last_token == VARINT
742           || last_token == VARFLOAT
743           || last_token == NUM)
744       {
745           return scan_throw(COMMA);
746       }
747   }
748
749   {spaces}*{colon}{spaces}* {
750       return scan_throw(COLON);
751   }
752
753   {newline} {
754       yylloc.last_line += 1;
755       yylloc.last_column = 1;
756       if(last_token != DOTS && last_token != EOL)
757       {
758           return scan_throw(EOL);
759       }
760       scan_throw(EOL);
761   }
762
763   {rbrack}                              {
764     DEBUG("yy_pop_state()");
765     yy_pop_state();
766     paren_levels.pop();
767     ParserSingleInstance::popControlStatus();
768     return scan_throw(RBRACK);
769   }
770
771   {rbrace}                              {
772     yy_pop_state();
773     paren_levels.pop();
774     ParserSingleInstance::popControlStatus();
775     return scan_throw(RBRACE);
776   }
777
778   {plus}                                |
779   {spaces}{plus}{spaces}                {
780     return scan_throw(PLUS);
781   }
782
783   {minus}                               |
784   {spaces}{minus}{spaces}               {
785     return scan_throw(MINUS);
786   }
787
788   {spaces}{plus}                        {
789     // no need to unput the '+'
790     if (last_token != LBRACK
791        && last_token != EOL
792        && last_token != SEMI
793        && last_token != COMMA
794        && last_token != DOTTIMES
795        && last_token != DOTRDIVIDE
796        && last_token != DOTLDIVIDE
797        && last_token != DOTPOWER
798        && last_token != MINUS
799        && last_token != PLUS
800        && last_token != TIMES
801        && last_token != RDIVIDE
802        && last_token != LDIVIDE
803        && last_token != POWER
804        && last_token != KRONTIMES
805        && last_token != KRONRDIVIDE
806        && last_token != KRONLDIVIDE
807        && last_token != EQ
808        && last_token != NE
809        && last_token != LT
810        && last_token != GT
811        && last_token != LE
812        && last_token != GE
813       && paren_levels.top() == 0)
814    {
815        return scan_throw(COMMA);
816    }
817    else
818    {
819        unput('+');
820        yylloc.last_column--;
821    }
822   }
823
824   {spaces}{minus}                       {
825     unput('-');
826     yylloc.last_column--;
827     if (last_token != LBRACK
828        && last_token != EOL
829        && last_token != SEMI
830        && last_token != COMMA
831        && last_token != DOTTIMES
832        && last_token != DOTRDIVIDE
833        && last_token != DOTLDIVIDE
834        && last_token != DOTPOWER
835        && last_token != MINUS
836        && last_token != PLUS
837        && last_token != TIMES
838        && last_token != RDIVIDE
839        && last_token != LDIVIDE
840        && last_token != POWER
841        && last_token != KRONTIMES
842        && last_token != KRONRDIVIDE
843        && last_token != KRONLDIVIDE
844        && last_token != EQ
845        && last_token != NE
846        && last_token != LT
847        && last_token != GT
848        && last_token != LE
849        && last_token != GE
850        && paren_levels.top() == 0)
851    {
852        return scan_throw(COMMA);
853    }
854   }
855
856   .                                     {
857     std::string str = "Unexpected token \'";
858     str += yytext;
859     str += "\' within a matrix.";
860     BEGIN(INITIAL);
861     yyerror(str);
862     return scan_throw(FLEX_ERROR);
863   }
864
865   {next}{spaces}*{newline}          {
866       yylloc.last_line += 1;
867       yylloc.last_column = 1;
868       scan_step();
869   }
870
871   {next}{spaces}*{startcomment}.*{newline}          {
872       yylloc.last_line += 1;
873       yylloc.last_column = 1;
874       scan_step();
875   }
876
877   {spaces}{next}{spaces}*{newline}          {
878       yylloc.last_line += 1;
879       yylloc.last_column = 1;
880       scan_step();
881       unput(' ');
882       yylloc.last_column--;
883   }
884
885   {spaces}{next}{spaces}*{startcomment}.*{newline}          {
886       yylloc.last_line += 1;
887       yylloc.last_column = 1;
888       scan_step();
889       unput(' ');
890       yylloc.last_column--;
891   }
892
893   <<EOF>>       {
894       yy_pop_state();
895       paren_levels.pop();
896   }
897 }
898
899 <LINEBREAK>
900 {
901   {newline}                             {
902     yylloc.last_line += 1;
903     yylloc.last_column = 1;
904     scan_step();
905     yy_pop_state();
906     ParserSingleInstance::popControlStatus();
907   }
908
909   {startblockcomment}                   {
910     ++comment_level;
911     yy_push_state(REGIONCOMMENT);
912   }
913
914   {startcomment}                        {
915     scan_throw(DOTS);
916     pstBuffer.clear();
917     yy_push_state(LINECOMMENT);
918   }
919
920   {spaces}                              {
921       /* Do nothing... */
922   }
923
924   <<EOF>>       {
925       yy_pop_state();
926   }
927   .                                     {
928     std::string str = "Unexpected token \'";
929     str += yytext;
930     str += "\' after line break with .. or ...";
931     BEGIN(INITIAL);
932     yyerror(str);
933     return scan_throw(FLEX_ERROR);
934   }
935 }
936
937
938 <LINECOMMENT>
939 {
940   {newline}     {
941     //yylloc.last_line += 1;
942     //yylloc.last_column = 1;
943     //scan_step();
944     yy_pop_state();
945     for (int i = yyleng - 1 ; i >= 0 ; --i)
946     {
947         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
948         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
949         unput(yytext[i]);
950         yylloc.last_column--;
951     }
952     /*
953     ** To forgot comments after lines break
954     */
955     if (last_token != DOTS)
956     {
957         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
958         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
959         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
960         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
961         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
962         {
963             pstBuffer.clear();
964             std::string str = "Can\'t convert \'";
965             str += pstBuffer.c_str();
966             str += "\' to UTF-8";
967             BEGIN(INITIAL);
968             yyerror(str);
969             return scan_throw(FLEX_ERROR);
970         }
971         yylval.comment = new std::wstring(pwstBuffer);
972         pstBuffer.clear();
973         FREE (pwstBuffer);
974         return scan_throw(COMMENT);
975     }
976     else
977     {
978         pstBuffer.clear();
979     }
980   }
981
982   <<EOF>>       {
983     yy_pop_state();
984     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
985     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
986     {
987         pstBuffer.clear();
988         std::string str = "Can\'t convert \'";
989         str += pstBuffer.c_str();
990         str += "\' to UTF-8";
991         BEGIN(INITIAL);
992         yyerror(str);
993         return scan_throw(FLEX_ERROR);
994     }
995     yylval.comment = new std::wstring(pwstBuffer);
996     pstBuffer.clear();
997     FREE (pwstBuffer);
998     return scan_throw(COMMENT);
999   }
1000
1001   {char_in_line_comment}         {
1002       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1003       // only translate to WCHAR_T when popping state.
1004       pstBuffer += yytext;
1005   }
1006 }
1007
1008
1009 <REGIONCOMMENT>
1010 {
1011   {endblockcomment}                             {
1012     --comment_level;
1013     if (comment_level == 0) {
1014       ParserSingleInstance::popControlStatus();
1015       yy_pop_state();
1016       //return scan_throw(BLOCKCOMMENT);
1017     }
1018   }
1019
1020   {startblockcomment}                           {
1021     ++comment_level;
1022     yy_push_state(REGIONCOMMENT);
1023   }
1024
1025   {newline}                                     {
1026     yylloc.last_line += 1;
1027     yylloc.last_column = 1;
1028     scan_step();
1029     *yylval.comment += L"\n//";
1030   }
1031
1032   {char_in_comment}                             |
1033   .                                             {
1034       wchar_t *pwText = to_wide_string(yytext);
1035       *yylval.comment += std::wstring(pwText);
1036       FREE(pwText);
1037   }
1038
1039  <<EOF>>                                        {
1040       yy_pop_state();
1041 //    std::string str = "unexpected end of file in a comment";
1042 //    scan_error(str);
1043   }
1044 }
1045
1046
1047 <SIMPLESTRING>
1048 {
1049   {dquote}{dquote}                              {
1050     pstBuffer += "\"";
1051   }
1052
1053   {dquote}{quote}                               {
1054     pstBuffer += "'";
1055   }
1056
1057   {quote}{dquote}                               {
1058     pstBuffer += "\"";
1059   }
1060
1061   {quote}{quote}                                {
1062     pstBuffer += "'";
1063   }
1064
1065   {quote}                                       {
1066     yy_pop_state();
1067     //scan_step();
1068     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1069     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1070     {
1071         pstBuffer.clear();
1072         std::string str = "Can\'t convert \'";
1073         str += pstBuffer.c_str();
1074         str += "\' to UTF-8";
1075         BEGIN(INITIAL);
1076         yyerror(str);
1077         return scan_throw(FLEX_ERROR);
1078     }
1079     yylval.str = new std::wstring(pwstBuffer);
1080     pstBuffer.clear();
1081     FREE(pwstBuffer);
1082     yylloc.first_column = str_opener_column;
1083     return scan_throw(STR);
1084   }
1085
1086   {dquote}                  {
1087     pstBuffer.clear();
1088     BEGIN(INITIAL);
1089     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1090     return scan_throw(FLEX_ERROR);
1091   }
1092
1093   {next}{newline}           {
1094       yylloc.last_line += 1;
1095       yylloc.last_column = 1;
1096       /* Do nothing... Just skip */
1097   }
1098
1099   {newline}                                     {
1100     pstBuffer.clear();
1101     yylloc.last_line += 1;
1102     yylloc.last_column = 1;
1103     BEGIN(INITIAL);
1104     yyerror("Unexpected end of line in a string.");
1105     return scan_throw(FLEX_ERROR);
1106   }
1107
1108   <<EOF>>                                       {
1109     pstBuffer.clear();
1110     BEGIN(INITIAL);
1111     yyerror("Unexpected end of file in a string.");
1112     return scan_throw(FLEX_ERROR);
1113   }
1114
1115   {in_string}                                           |
1116   .                                                     {
1117     //scan_step();
1118     pstBuffer += yytext;
1119   }
1120 }
1121
1122
1123 <DOUBLESTRING>
1124 {
1125   {dquote}{dquote}                              {
1126     pstBuffer += "\"";
1127   }
1128
1129   {dquote}{quote}                               {
1130     pstBuffer += "'";
1131   }
1132
1133   {quote}{dquote}               {
1134     pstBuffer += "\"";
1135   }
1136
1137   {quote}{quote}                                {
1138     pstBuffer += "'";
1139   }
1140
1141   {dquote}                      {
1142     yy_pop_state();
1143     //scan_step();
1144     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1145     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1146     {
1147         pstBuffer.clear();
1148         std::string str = "Can\'t convert \'";
1149         str += pstBuffer.c_str();
1150         str += "\' to UTF-8";
1151         BEGIN(INITIAL);
1152         yyerror(str);
1153         return scan_throw(FLEX_ERROR);
1154     }
1155     yylval.str = new std::wstring(pwstBuffer);
1156     pstBuffer.clear();
1157     FREE(pwstBuffer);
1158     yylloc.first_column = str_opener_column;
1159     return scan_throw(STR);
1160   }
1161
1162   {quote}                  {
1163     pstBuffer.clear();
1164     BEGIN(INITIAL);
1165     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1166     return scan_throw(FLEX_ERROR);
1167   }
1168
1169   {next}{newline}           {
1170       yylloc.last_line += 1;
1171       yylloc.last_column = 1;
1172       /* Do nothing... Just skip */
1173   }
1174
1175   {newline} {
1176     pstBuffer.clear();
1177     yylloc.last_line += 1;
1178     yylloc.last_column = 1;
1179     BEGIN(INITIAL);
1180     yyerror("Unexpected end of line in a string.");
1181     return scan_throw(FLEX_ERROR);
1182   }
1183
1184   <<EOF>>   {
1185     pstBuffer.clear();
1186     BEGIN(INITIAL);
1187     yyerror("Unexpected end of file in a string.");
1188     return scan_throw(FLEX_ERROR);
1189   }
1190
1191   {in_string}         |
1192   .                   {
1193    //scan_step();
1194    pstBuffer += yytext;
1195   }
1196 }
1197
1198
1199 <SHELLMODE>
1200 {
1201     {spaces}                    {
1202         if (last_token == ID)
1203         {
1204             scan_throw(SPACES);
1205             return ID;
1206         }
1207     }
1208
1209     {semicolon}                 {
1210         BEGIN(INITIAL);
1211         scan_step();
1212         return scan_throw(SEMI);
1213     }
1214
1215     {comma}                     {
1216         BEGIN(INITIAL);
1217         scan_step();
1218         return scan_throw(COMMA);
1219     }
1220
1221     {newline}                   {
1222         BEGIN(INITIAL);
1223         yylloc.last_line += 1;
1224         yylloc.last_column = 1;
1225         scan_step();
1226         return scan_throw(EOL);
1227     }
1228
1229     {assign} {
1230         if (last_token == STR || last_token == SPACES)
1231         {
1232             wchar_t *pwText = to_wide_string(yytext);
1233             yylval.str = new std::wstring(pwText);
1234             FREE(pwText);
1235             return scan_throw(STR);
1236         }
1237         else
1238         {
1239             BEGIN(INITIAL);
1240             return scan_throw(ASSIGN);
1241         }
1242     }
1243
1244     {lparen} {
1245         if (last_token == STR || last_token == SPACES)
1246         {
1247             wchar_t *pwText = to_wide_string(yytext);
1248             yylval.str = new std::wstring(pwText);
1249             FREE(pwText);
1250             return scan_throw(STR);
1251         }
1252         else
1253         {
1254             BEGIN(INITIAL);
1255             return scan_throw(LPAREN);
1256         }
1257     }
1258
1259     {lowerthan} {
1260         if (last_token == STR || last_token == SPACES)
1261         {
1262             wchar_t *pwText = to_wide_string(yytext);
1263             yylval.str = new std::wstring(pwText);
1264             FREE(pwText);
1265             return scan_throw(STR);
1266         }
1267         else
1268         {
1269             BEGIN(INITIAL);
1270             return scan_throw(LT);
1271         }
1272     }
1273
1274     {greaterthan} {
1275         if (last_token == STR || last_token == SPACES)
1276         {
1277             wchar_t *pwText = to_wide_string(yytext);
1278             yylval.str = new std::wstring(pwText);
1279             FREE(pwText);
1280             return scan_throw(STR);
1281         }
1282         else
1283         {
1284             BEGIN(INITIAL);
1285             return scan_throw(GT);
1286         }
1287     }
1288
1289     {boolnot} {
1290         if (last_token == STR || last_token == SPACES)
1291         {
1292             wchar_t *pwText = to_wide_string(yytext);
1293             yylval.str = new std::wstring(pwText);
1294             FREE(pwText);
1295             return scan_throw(STR);
1296         }
1297         else
1298         {
1299             BEGIN(INITIAL);
1300             return scan_throw(NOT);
1301         }
1302     }
1303
1304
1305     [^ \t\v\f\r\n,;'"]+               {
1306         wchar_t *pwText = to_wide_string(yytext);
1307         yylval.str = new std::wstring(pwText);
1308         FREE(pwText);
1309         return scan_throw(STR);
1310     }
1311
1312     <<EOF>>                     {
1313         BEGIN(INITIAL);
1314     }
1315
1316 }
1317
1318 %%
1319
1320 int scan_throw(int token) {
1321   last_token = token;
1322 #ifdef DEV
1323   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1324 #endif
1325   return token;
1326 }
1327
1328 int get_last_token() {
1329     return last_token;
1330 }
1331
1332 void scan_step() {
1333   yylloc.first_line = yylloc.last_line;
1334   yylloc.first_column = yylloc.last_column;
1335 }
1336
1337 /*
1338 ** convert floating numbers to C standard
1339 ** 1.2d-3 -> 1.2e-3
1340 ** 1.2D-3 -> 1.2e-3
1341 */
1342 void scan_exponent_convert(char *in)
1343 {
1344   for (; *in != 'd' && *in != 'D'; ++in);
1345   *in = 'e';
1346 }
1347
1348 #ifdef _MSC_VER
1349 int isatty (int desc)
1350 {
1351   return 0;
1352 }
1353 #endif