Bug 12716 fixed: In SciNotes, RTL languages were not correctly displayed
[scilab.git] / scilab / modules / scinotes / src / java / jflex / scilab.jflex
1 //CHECKSTYLE:OFF
2
3 package org.scilab.modules.scinotes;
4
5 import java.util.Arrays;
6 import java.util.HashSet;
7 import java.util.Iterator;
8 import java.util.ArrayList;
9 import java.util.List;
10 import java.util.Set;
11 import java.io.IOException;
12 import javax.swing.text.BadLocationException;
13 import javax.swing.text.Element;
14
15 import org.scilab.modules.commons.ScilabCommonsUtils;
16
17 @javax.annotation.Generated("JFlex")
18 @SuppressWarnings("fallthrough")
19
20 %%
21
22 %public
23 %class ScilabLexer
24 %extends ScilabLexerConstants
25 %final
26 %unicode
27 %char
28 %type int
29 %switch
30 %pack
31
32 %{
33     public int start;
34     public int end;
35     public int beginString;
36     public static Set<String> commands = new HashSet<String>();
37     public static Set<String> macros = new HashSet<String>();
38     public static Set<String> variables = new HashSet<String>();
39     public Set<String> infile;
40
41     private ScilabDocument doc;
42     private boolean transposable;
43     private Element elem;
44     private boolean breakstring;
45
46     public ScilabLexer(ScilabDocument doc) {
47         this(doc, true);
48     }
49
50     public ScilabLexer(ScilabDocument doc, boolean update) {
51         this.doc = doc;
52         this.elem = doc.getDefaultRootElement();
53         this.infile = doc.getFunctionsInDoc();
54         if (update) {
55            update();
56         }
57     }      
58
59     public static void update() {
60         if (ScilabCommonsUtils.isScilabThread()) {
61             String[] vars = ScilabKeywords.GetVariablesName();
62             String[] funs = ScilabKeywords.GetFunctionsName();
63             String[] macs = ScilabKeywords.GetMacrosName();
64             variables.clear();
65             commands.clear();
66             macros.clear();
67             if (vars != null) {
68                 variables.addAll(Arrays.asList(vars));
69             }
70             if (funs != null) {
71                 commands.addAll(Arrays.asList(funs));
72             }
73             if (macs != null) {
74                 macros.addAll(Arrays.asList(macs));
75             }
76         }
77     }
78
79     public void setRange(int p0, int p1) {
80         start = p0;
81         end = p1;
82         transposable = false;
83         breakstring = false;
84         yyreset(new ScilabDocumentReader(doc, p0, p1));
85         int currentLine = elem.getElementIndex(start);
86         if (currentLine != 0 && ((ScilabDocument.ScilabLeafElement) elem.getElement(currentLine - 1)).isBrokenString()) {
87            yybegin(QSTRING);
88         }
89     }
90
91     public int yychar() {
92         return yychar;
93     }
94
95     public int scan() throws IOException {
96         int ret = yylex();
97         if (start + yychar + yylength() == end - 1) {
98            ((ScilabDocument.ScilabLeafElement) elem.getElement(elem.getElementIndex(start))).setBrokenString(breakstring);
99            breakstring = false;
100         }
101         return ret;
102     }
103
104     public int getKeyword(int pos, boolean strict) {
105         Element line = elem.getElement(elem.getElementIndex(pos));
106         int end = line.getEndOffset();
107         int tok = -1;
108         start = line.getStartOffset();
109         int startL = start;
110         int s = -1;
111
112         try {
113            yyreset(new ScilabDocumentReader(doc, start, end));
114            if (!strict) {
115               pos++;
116            }
117
118            while (startL < pos && (s != startL || yystate() == BREAKSTRING)) {
119                s = startL;
120                tok = yylex();
121                startL = start + yychar + yylength();
122            }
123
124            return tok;
125         } catch (IOException e) {
126            return ScilabLexerConstants.DEFAULT;
127         }
128      }
129
130      public static ScilabTokens getScilabTokens(String str) {
131          ScilabDocument doc = new ScilabDocument(false);
132          try {
133              doc.insertString(0, str, null);
134          } catch (BadLocationException e) { }
135          return getScilabTokens(doc);
136      }
137
138      public static ScilabTokens getScilabTokens(ScilabDocument doc) {
139          ScilabLexer lexer = new ScilabLexer(doc);
140          lexer.yyreset(new ScilabDocumentReader(doc, 0, doc.getLength()));
141          ScilabTokens tokens = new ScilabTokens();
142          int tok = -1;
143          try {
144             while (tok != ScilabLexerConstants.EOF) {
145                tok = lexer.yylex();
146                tokens.add(tok, lexer.yychar + lexer.yylength());
147             }
148          } catch (IOException e) { }
149  
150          return tokens;
151      }
152
153      public static class ScilabTokens {
154         private List<Integer> tokenType = new ArrayList<Integer>();
155         private List<Integer> tokenPos = new ArrayList<Integer>();
156
157         ScilabTokens() { }
158
159         void add(final int type, final int pos) {
160            tokenType.add(type);
161            tokenPos.add(pos);
162         }
163
164         public final List<Integer> getTokenType() {
165            return tokenType;
166         }
167
168         public final List<Integer> getTokenPos() {
169            return tokenPos;
170         }
171      }
172 %}
173
174 /* main character classes */
175 eol = \n
176
177 open = "[" | "(" | "{"
178 close = "]" | ")" | "}"
179
180 comment = "//"
181
182 quote = "'"
183
184 dquote = "\""
185
186 cstes = "%t" | "%T" | "%f" | "%F" | "%e" | "%pi" | "%inf" | "%i" | "%z" | "%s" | "%nan" | "%eps" | "SCI" | "WSCI" | "SCIHOME" | "TMPDIR"
187
188 operator = ".'" | ".*" | "./" | ".\\" | ".^" | ".**" | "+" | "-" | "/" | "\\" | "*" | "^" | "**" | "==" | "~=" | "<>" | "<" | ">" | "<=" | ">=" | ".*." | "./." | ".\\." | "/." | "=" | "&" | "|" | "@" | "@=" | "~"
189
190 functionKwds = "function" | "endfunction"
191
192 structureKwds = "then" | "do" | "catch" | "case"
193
194 elseif = "elseif" | "else"
195
196 openCloseStructureKwds = "if" | "for" | "while" | "try" | "select" | "end"
197
198 controlKwds = "abort" | "break" | "quit" | "return" | "resume" | "pause" | "continue" | "exit"
199
200 authors = "Calixte Denizet" | "Calixte DENIZET" | "Sylvestre Ledru" | "Sylvestre LEDRU" | "Antoine Elias" | "Antoine ELIAS" | "Bruno Jofret" | "Bruno JOFRET" | "Claude Gomez" | "Claude GOMEZ" | "Clement David" | "Clement DAVID" | "Manuel Juliachs" | "Manuel JULIACHS" | "Sheldon Cooper" | "Leonard Hofstadter" | "Serge Steer" | "Serge STEER" | "Vincent Couvert" | "Vincent COUVERT" | "Adeline Carnis" | "Adeline CARNIS" | "Charlotte Hecquet" | "Charlotte HECQUET" | "Paul Bignier" | "Paul BIGNIER" | "Alexandre Herisse" | "Alexandre HERISSE" | "Simon Marchetto" | "Simon MARCHETTO" | "Vladislav Trubkin" | "Vladislav TRUBKIN" | "Cedric Delamarre" | "Cedric DELAMARRE" | "Inria" | "INRIA" | "DIGITEO" | "Digiteo" | "Scilab Enterprises" | "ENPC"
201
202 error = "Scilab Entreprises" | "Scilab Entreprise" | "Scilab Enterprise"
203 todo = ("TODO" | "todo" | "Todo")[ \t]*:[^\n]*
204
205 break = ".."(".")*
206 breakinstring = {break}[ \t]*({comment} | {eol})
207
208 special = "$" | ":" | {break}
209
210 string = (([^ \t\'\"\r\n\.]*)|([\'\"]{2}))+
211
212 id = ([a-zA-Z%_#!?][a-zA-Z0-9_#!$?]*)|("$"[a-zA-Z0-9_#!$?]+)
213
214 badid = ([0-9$][a-zA-Z0-9_#!$?]+)
215 whitabs = (" "+"\t" | "\t"+" ")[ \t]*
216 badop = [+-]([\*\/\\\^] | "."[\*\+\-\/\\\^]) | ":=" | "->" | " !=" | "&&" | "||" | ([*+-/\\\^]"=")
217
218 dot = "."
219
220 url = ("http://"|"https://"|"ftp://"|"dav://"|"davs://"|"sftp://"|"ftps://"|"smb:///"|"file://")[^ \t\f\n\r\'\"]+
221 mailaddr = [ \t]*[a-zA-Z0-9_\.\-]+"@"([a-zA-Z0-9\-]+".")+[a-zA-Z]{2,5}[ \t]*
222 mail = ("<" {mailaddr} ">") | ("mailto:" {mailaddr})
223
224 latex = "$$"(([^$]*|"\\$")+)"$$"
225 latexinstring = (\"|\')"$"(([^$\'\"]*|"\\$"|([\'\"]{2}))+)"$"(\"|\')
226
227 digit = [0-9]
228 exp = [dDeE][+-]?{digit}*
229 number = ({digit}+"."?{digit}*{exp}?)|("."{digit}+{exp}?)
230
231 arabic_char = [\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]
232 hebrew_char = [\u0590-\u05FF\uFB1D-\uFB4F]
233 nko_char = [\u07C0-\u07FF]
234 thaana_char = [\u0780-\u07BF]
235 rtl_char = {arabic_char}|{hebrew_char}|{nko_char}|{thaana_char}
236 rtl_comment = {rtl_char}[^\n]*
237 rtl_in_string = {rtl_char}(([^\'\"\r\n\.]*)|([\'\"]{2}))+
238
239 %x QSTRING, COMMENT, FIELD, COMMANDS, COMMANDSWHITE, BREAKSTRING
240
241 %%
242
243 <YYINITIAL> {
244   {comment}                      {
245                                    transposable = false;
246                                    yypushback(2);
247                                    yybegin(COMMENT);
248                                  }
249
250   {operator}                     {
251                                    transposable = false;
252                                    return ScilabLexerConstants.OPERATOR;
253                                  }
254
255   {functionKwds}                 {
256                                    transposable = false;
257                                    return ScilabLexerConstants.FKEYWORD;
258                                  }
259
260   {openCloseStructureKwds}       {
261                                    transposable = false;
262                                    return ScilabLexerConstants.OSKEYWORD;
263                                  }
264
265   {structureKwds}                {
266                                    transposable = false;
267                                    return ScilabLexerConstants.SKEYWORD;
268                                  }
269
270   {elseif}                       {
271                                    transposable = false;
272                                    return ScilabLexerConstants.ELSEIF;
273                                  }
274
275   {controlKwds}                  {
276                                    transposable = false;
277                                    return ScilabLexerConstants.CKEYWORD;
278                                  }
279
280   {cstes}                        {
281                                    transposable = true;
282                                    return ScilabLexerConstants.CONSTANTES;
283                                  }
284
285   {id}                           {
286                                    transposable = true;
287                                    String str = yytext();
288                                    if (commands.contains(str)) {
289                                        yybegin(COMMANDS);
290                                        return ScilabLexerConstants.COMMANDS;
291                                    } else if (macros.contains(str)) {
292                                        yybegin(COMMANDS);
293                                        return ScilabLexerConstants.MACROS;
294                                    } else if (infile.contains(str)) {
295                                        yybegin(COMMANDS);
296                                        return ScilabLexerConstants.MACROINFILE;
297                                    } else {
298                                        List<String>[] arr = doc.getInOutArgs(start + yychar);
299                                        if (arr != null && (arr[0].contains(str) || arr[1].contains(str))) {
300                                            return ScilabLexerConstants.INPUTOUTPUTARGS;
301                                        } else if (variables.contains(str)) {
302                                            return ScilabLexerConstants.VARIABLES;
303                                        }
304                                    }
305                                    return ScilabLexerConstants.ID;
306                                  }
307
308   {number}                       {
309                                    transposable = true;
310                                    return ScilabLexerConstants.NUMBER;
311                                  }
312
313   {special}                      {
314                                    transposable = false;
315                                    return ScilabLexerConstants.SPECIAL;
316                                  }
317
318   {dot}                          {
319                                    transposable = false;
320                                    yybegin(FIELD);
321                                    return ScilabLexerConstants.OPERATOR;
322                                  }
323
324   {latexinstring}                {
325                                    return ScilabLexerConstants.LATEXINSTRING;
326                                  }
327
328   {quote}                        {
329                                     if (transposable) {
330                                        return ScilabLexerConstants.TRANSP;
331                                     } else {
332                                        beginString = zzStartRead;
333                                        yybegin(QSTRING);
334                                        return ScilabLexerConstants.STRING;
335                                     }
336                                  }
337
338   {open}                         {
339                                    transposable = false;
340                                    return ScilabLexerConstants.OPENCLOSE;
341                                  }
342
343   {close}                        {
344                                    transposable = true;
345                                    return ScilabLexerConstants.OPENCLOSE;
346                                  }
347
348   {dquote}                       {
349                                    transposable = false;
350                                    beginString = zzStartRead;
351                                    yybegin(QSTRING);
352                                    return ScilabLexerConstants.STRING;
353                                  }
354
355   {badid}                        |
356   {badop}                        |
357   {whitabs}                      {
358                                    return ScilabLexerConstants.ERROR;
359                                  }
360
361   " "                            {
362                                    transposable = false;
363                                    return ScilabLexerConstants.WHITE;
364                                  }
365
366   "\t"                           {
367                                    transposable = false;
368                                    return ScilabLexerConstants.TAB;
369                                  }
370
371   .                              |
372   {eol}                          {
373                                    transposable = false;
374                                    return ScilabLexerConstants.DEFAULT;
375                                  }
376 }
377
378 <COMMANDS> {
379   [ \t]*"("                      {
380                                    yypushback(yylength());
381                                    yybegin(YYINITIAL);
382                                  }
383
384   " "                            {
385                                    yybegin(COMMANDSWHITE);
386                                    return ScilabLexerConstants.WHITE;
387                                  }
388
389   "\t"                           {
390                                    yybegin(COMMANDSWHITE);
391                                    return ScilabLexerConstants.TAB;
392                                  }
393   .
394                                  {
395                                    yypushback(1);
396                                    yybegin(YYINITIAL);
397                                  }
398
399   {eol}                          { }
400 }
401
402 <COMMANDSWHITE> {
403   {comment}                      {
404                                    transposable = false;
405                                    yypushback(2);
406                                    yybegin(COMMENT);
407                                  }
408
409   ([^ \t,;/]*) | ("/"[^ /]*)     {
410                                    return ScilabLexerConstants.STRING;
411                                  }
412
413   " "                            {
414                                    return ScilabLexerConstants.WHITE;
415                                  }
416
417   "\t"                           {
418                                    return ScilabLexerConstants.TAB;
419                                  }
420
421   .                              |
422   {eol}                          {
423                                    yypushback(1);
424                                    yybegin(YYINITIAL);
425                                  }
426 }
427
428 <FIELD> {
429   {id}                           {
430                                    transposable = true;
431                                    return ScilabLexerConstants.FIELD;
432                                  }
433
434   .                              |
435   {eol}                          {
436                                    yypushback(1);
437                                    yybegin(YYINITIAL);
438                                  }
439 }
440
441 <QSTRING> {
442   {breakinstring}                {
443                                    yypushback(yylength());
444                                    yybegin(BREAKSTRING);
445                                    transposable = false;
446                                    return ScilabLexerConstants.STRING;
447                                  }
448
449   " "                            {
450                                    return ScilabLexerConstants.WHITE_STRING;
451                                  }
452
453   "\t"                           {
454                                    return ScilabLexerConstants.TAB_STRING;
455                                  }
456
457   {rtl_in_string}                |
458   {string}                       |
459   "."                            {
460                                    return ScilabLexerConstants.STRING;
461                                  }
462
463   (\'|\")                        {
464                                    transposable = false;
465                                    yybegin(YYINITIAL);
466                                    return ScilabLexerConstants.STRING;
467                                  }
468
469   .                              |
470   {eol}                          {
471                                    return ScilabLexerConstants.DEFAULT;
472                                  }
473 }
474
475 <COMMENT> {
476   {todo}                         {
477                                    return ScilabLexerConstants.TODO;
478                                  }
479
480   {error}                        {
481                                    return ScilabLexerConstants.ERROR;
482                                  }
483
484
485   {authors}                      {
486                                    return ScilabLexerConstants.AUTHORS;
487                                  }
488
489   {url}                          {
490                                    return ScilabLexerConstants.URL;
491                                  }
492
493   {mail}                         {
494                                    return ScilabLexerConstants.MAIL;
495                                  }
496
497   {latex}                        {
498                                    return ScilabLexerConstants.LATEX;
499                                  }
500
501   " "                            {
502                                    return ScilabLexerConstants.WHITE_COMMENT;
503                                  }
504
505   "\t"                           {
506                                    return ScilabLexerConstants.TAB_COMMENT;
507                                  }
508  
509   {rtl_comment}                  {
510                                     return ScilabLexerConstants.COMMENT;
511                                  }
512
513   [^ \t\n]+                      {
514                                    return ScilabLexerConstants.COMMENT;
515                                  }
516
517   {eol}                          {
518                                    yybegin(YYINITIAL);
519                                    return ScilabLexerConstants.DEFAULT;
520                                  }
521 }
522
523 <BREAKSTRING> {
524   {break}                        {
525                                    breakstring = true;
526                                    return ScilabLexerConstants.SPECIAL;
527                                  }
528
529   " "                            {
530                                    return ScilabLexerConstants.WHITE;
531                                  }
532
533   "\t"                           {
534                                    return ScilabLexerConstants.TAB;
535                                  }
536
537   {comment}                      {
538                                    transposable = false;
539                                    yypushback(2);
540                                    yybegin(COMMENT);
541                                  }
542
543   .                              |
544   {eol}                          {
545                                    return ScilabLexerConstants.DEFAULT;
546                                  }
547 }
548
549 <<EOF>>                          {
550                                    return ScilabLexerConstants.EOF;
551                                  }