JavaParser/com/github/javaparser/JavaToken.java

1	/*
2	* Copyright (C) 2007-2010 Júlio Vilmar Gesser.
3	* Copyright (C) 2011, 2013-2020 The JavaParser Team.
4	*
5	* This file is part of JavaParser.
6	*
7	* JavaParser can be used either under the terms of
8	* a) the GNU Lesser General Public License as published by
9	* the Free Software Foundation, either version 3 of the License, or
10	* (at your option) any later version.
11	* b) the terms of the Apache License
12	*
13	* You should have received a copy of both licenses in LICENCE.LGPL and
14	* LICENCE.APACHE. Please refer to those files for details.
15	*
16	* JavaParser is distributed in the hope that it will be useful,
17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19	* GNU Lesser General Public License for more details.
20	*/
21	package com.github.javaparser;
22
23	import java.util.List;
24	import java.util.Optional;
25	import static com.github.javaparser.utils.CodeGenerationUtils.f;
26	import static com.github.javaparser.utils.Utils.SYSTEM_EOL;
27	import static com.github.javaparser.utils.Utils.assertNotNull;
28	import com.github.javaparser.ast.Generated;
29
30	/**
31	* A token from a parsed source file.
32	* (Awkwardly named "Java"Token since JavaCC already generates an internal class Token.)
33	* It is a node in a double linked list called token list.
34	*/
35	public class JavaToken {
36
37	public static final JavaToken INVALID = new JavaToken();
38
39	private Range range;
40
41	private int kind;
42
43	private String text;
44
45	private JavaToken previousToken = null;
46
47	private JavaToken nextToken = null;
48
49	private JavaToken() {
50	this(null, 0, "INVALID", null, null);
51	}
52
53	public JavaToken(int kind, String text) {
54	this(null, kind, text, null, null);
55	}
56
57	JavaToken(Token token, List<JavaToken> tokens) {
58	// You could be puzzled by the following lines
59	//
60	// The reason why these lines are necessary is the fact that Java is ambiguous. There are cases where the
61	// sequence of characters ">>>" and ">>" should be recognized as the single tokens ">>>" and ">>". In other
62	// cases however we want to split those characters in single GT tokens (">").
63	//
64	// For example, in expressions ">>" and ">>>" are valid, while when defining types we could have this:
65	//
66	// List<List<Set<String>>>>
67	//
68	// You can see that the sequence ">>>>" should be interpreted as four consecutive ">" tokens closing a type
69	// parameter list.
70	//
71	// The JavaCC handle this case by first recognizing always the longest token, and then depending on the context
72	// putting back the unused chars in the stream. However in those cases the token provided is invalid: it has an
73	// image corresponding to the text originally recognized, without considering that after some characters could
74	// have been put back into the stream.
75	//
76	// So in the case of:
77	//
78	// List<List<Set<String>>>>
79	// ___ -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class
80	// ___ -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class
81	// __ -> recognized as ">>", then ">" put back in the stream but Token(type=GT, image=">>") passed to this class
82	// _ -> Token(type=GT, image=">") good!
83	//
84	// So given the image could be wrong but the type is correct, we look at the type of the token and we fix
85	// the image. Everybody is happy and we can keep this horrible thing as our little secret.
86	Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn);
87	String text = token.image;
88	if (token.kind == GeneratedJavaParserConstants.GT) {
89	range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn);
90	text = ">";
91	} else if (token.kind == GeneratedJavaParserConstants.RSIGNEDSHIFT) {
92	range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn + 1);
93	text = ">>";
94	}
95	this.range = range;
96	this.kind = token.kind;
97	this.text = text;
98	if (!tokens.isEmpty()) {
99	final JavaToken previousToken = tokens.get(tokens.size() - 1);
100	this.previousToken = previousToken;
101	previousToken.nextToken = this;
102	} else {
103	previousToken = null;
104	}
105	}
106
107	/**
108	* Create a token of a certain kind.
109	*/
110	public JavaToken(int kind) {
111	String content = GeneratedJavaParserConstants.tokenImage[kind];
112	if (content.startsWith("\"")) {
113	content = content.substring(1, content.length() - 1);
114	}
115	if (TokenTypes.isEndOfLineToken(kind)) {
116	content = SYSTEM_EOL;
117	} else if (TokenTypes.isWhitespace(kind)) {
118	content = " ";
119	}
120	this.kind = kind;
121	this.text = content;
122	}
123
124	public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) {
125	assertNotNull(text);
126	this.range = range;
127	this.kind = kind;
128	this.text = text;
129	this.previousToken = previousToken;
130	this.nextToken = nextToken;
131	}
132
133	public Optional<Range> getRange() {
134	return Optional.ofNullable(range);
135	}
136
137	public int getKind() {
138	return kind;
139	}
140
141	void setKind(int kind) {
142	this.kind = kind;
143	}
144
145	public String getText() {
146	return text;
147	}
148
149	public Optional<JavaToken> getNextToken() {
150	return Optional.ofNullable(nextToken);
151	}
152
153	public Optional<JavaToken> getPreviousToken() {
154	return Optional.ofNullable(previousToken);
155	}
156
157	public void setRange(Range range) {
158	this.range = range;
159	}
160
161	public void setText(String text) {
162	this.text = text;
163	}
164
165	public String asString() {
166	return text;
167	}
168
169	/**
170	* @return the token range that goes from the beginning to the end of the token list this token is a part of.
171	*/
172	public TokenRange toTokenRange() {
173	return new TokenRange(findFirstToken(), findLastToken());
174	}
175
176	@Override
177	public String toString() {
178	String text = getText().replace("\n", "\\n").replace("\r", "\\r").replace("\r\n", "\\r\\n").replace("\t", "\\t");
179	return f("\"%s\" <%s> %s", text, getKind(), getRange().map(Range::toString).orElse("(?)-(?)"));
180	}
181
182	/**
183	* Used by the parser while constructing nodes. No tokens should be invalid when the parser is done.
184	*/
185	public boolean valid() {
186	return !invalid();
187	}
188
189	/**
190	* Used by the parser while constructing nodes. No tokens should be invalid when the parser is done.
191	*/
192	public boolean invalid() {
193	return this == INVALID;
194	}
195
196	public enum Category {
197
198	WHITESPACE_NO_EOL,
199	EOL,
200	COMMENT,
201	IDENTIFIER,
202	KEYWORD,
203	LITERAL,
204	SEPARATOR,
205	OPERATOR;
206
207	public boolean isWhitespaceOrComment() {
208	return isWhitespace() \|\| this == COMMENT;
209	}
210
211	public boolean isWhitespace() {
212	return this == WHITESPACE_NO_EOL \|\| this == EOL;
213	}
214
215	public boolean isEndOfLine() {
216	return this == EOL;
217	}
218
219	public boolean isComment() {
220	return this == COMMENT;
221	}
222
223	public boolean isWhitespaceButNotEndOfLine() {
224	return this == WHITESPACE_NO_EOL;
225	}
226
227	public boolean isIdentifier() {
228	return this == IDENTIFIER;
229	}
230
231	public boolean isKeyword() {
232	return this == KEYWORD;
233	}
234
235	public boolean isLiteral() {
236	return this == LITERAL;
237	}
238
239	public boolean isSeparator() {
240	return this == SEPARATOR;
241	}
242
243	public boolean isOperator() {
244	return this == OPERATOR;
245	}
246	}
247
248	@Generated("com.github.javaparser.generator.core.other.TokenKindGenerator")
249	public enum Kind {
250
251	EOF(0),
252	SPACE(1),
253	WINDOWS_EOL(2),
254	UNIX_EOL(3),
255	OLD_MAC_EOL(4),
256	SINGLE_LINE_COMMENT(5),
257	ENTER_JAVADOC_COMMENT(6),
258	ENTER_MULTILINE_COMMENT(7),
259	JAVADOC_COMMENT(8),
260	MULTI_LINE_COMMENT(9),
261	COMMENT_CONTENT(10),
262	ABSTRACT(11),
263	ASSERT(12),
264	BOOLEAN(13),
265	BREAK(14),
266	BYTE(15),
267	CASE(16),
268	CATCH(17),
269	CHAR(18),
270	CLASS(19),
271	CONST(20),
272	CONTINUE(21),
273	_DEFAULT(22),
274	DO(23),
275	DOUBLE(24),
276	ELSE(25),
277	ENUM(26),
278	EXTENDS(27),
279	FALSE(28),
280	FINAL(29),
281	FINALLY(30),
282	FLOAT(31),
283	FOR(32),
284	GOTO(33),
285	IF(34),
286	IMPLEMENTS(35),
287	IMPORT(36),
288	INSTANCEOF(37),
289	INT(38),
290	INTERFACE(39),
291	LONG(40),
292	NATIVE(41),
293	NEW(42),
294	NULL(43),
295	PACKAGE(44),
296	PRIVATE(45),
297	PROTECTED(46),
298	PUBLIC(47),
299	RETURN(48),
300	SHORT(49),
301	STATIC(50),
302	STRICTFP(51),
303	SUPER(52),
304	SWITCH(53),
305	SYNCHRONIZED(54),
306	THIS(55),
307	THROW(56),
308	THROWS(57),
309	TRANSIENT(58),
310	TRUE(59),
311	TRY(60),
312	VOID(61),
313	VOLATILE(62),
314	WHILE(63),
315	YIELD(64),
316	REQUIRES(65),
317	TO(66),
318	WITH(67),
319	OPEN(68),
320	OPENS(69),
321	USES(70),
322	MODULE(71),
323	EXPORTS(72),
324	PROVIDES(73),
325	TRANSITIVE(74),
326	LONG_LITERAL(75),
327	INTEGER_LITERAL(76),
328	DECIMAL_LITERAL(77),
329	HEX_LITERAL(78),
330	OCTAL_LITERAL(79),
331	BINARY_LITERAL(80),
332	FLOATING_POINT_LITERAL(81),
333	DECIMAL_FLOATING_POINT_LITERAL(82),
334	DECIMAL_EXPONENT(83),
335	HEXADECIMAL_FLOATING_POINT_LITERAL(84),
336	HEXADECIMAL_EXPONENT(85),
337	HEX_DIGITS(86),
338	UNICODE_ESCAPE(87),
339	CHARACTER_LITERAL(88),
340	STRING_LITERAL(89),
341	ENTER_TEXT_BLOCK(90),
342	TEXT_BLOCK_LITERAL(91),
343	TEXT_BLOCK_CONTENT(92),
344	IDENTIFIER(93),
345	LETTER(94),
346	PART_LETTER(95),
347	LPAREN(96),
348	RPAREN(97),
349	LBRACE(98),
350	RBRACE(99),
351	LBRACKET(100),
352	RBRACKET(101),
353	SEMICOLON(102),
354	COMMA(103),
355	DOT(104),
356	AT(105),
357	ASSIGN(106),
358	LT(107),
359	BANG(108),
360	TILDE(109),
361	HOOK(110),
362	COLON(111),
363	EQ(112),
364	LE(113),
365	GE(114),
366	NE(115),
367	SC_OR(116),
368	SC_AND(117),
369	INCR(118),
370	DECR(119),
371	PLUS(120),
372	MINUS(121),
373	STAR(122),
374	SLASH(123),
375	BIT_AND(124),
376	BIT_OR(125),
377	XOR(126),
378	REM(127),
379	LSHIFT(128),
380	PLUSASSIGN(129),
381	MINUSASSIGN(130),
382	STARASSIGN(131),
383	SLASHASSIGN(132),
384	ANDASSIGN(133),
385	ORASSIGN(134),
386	XORASSIGN(135),
387	REMASSIGN(136),
388	LSHIFTASSIGN(137),
389	RSIGNEDSHIFTASSIGN(138),
390	RUNSIGNEDSHIFTASSIGN(139),
391	ELLIPSIS(140),
392	ARROW(141),
393	DOUBLECOLON(142),
394	RUNSIGNEDSHIFT(143),
395	RSIGNEDSHIFT(144),
396	GT(145),
397	CTRL_Z(146);
398
399	private final int kind;
400
401	Kind(int kind) {
402	this.kind = kind;
403	}
404
405	public static Kind valueOf(int kind) {
406	switch(kind) {
407	case 146:
408	return CTRL_Z;
409	case 145:
410	return GT;
411	case 144:
412	return RSIGNEDSHIFT;
413	case 143:
414	return RUNSIGNEDSHIFT;
415	case 142:
416	return DOUBLECOLON;
417	case 141:
418	return ARROW;
419	case 140:
420	return ELLIPSIS;
421	case 139:
422	return RUNSIGNEDSHIFTASSIGN;
423	case 138:
424	return RSIGNEDSHIFTASSIGN;
425	case 137:
426	return LSHIFTASSIGN;
427	case 136:
428	return REMASSIGN;
429	case 135:
430	return XORASSIGN;
431	case 134:
432	return ORASSIGN;
433	case 133:
434	return ANDASSIGN;
435	case 132:
436	return SLASHASSIGN;
437	case 131:
438	return STARASSIGN;
439	case 130:
440	return MINUSASSIGN;
441	case 129:
442	return PLUSASSIGN;
443	case 128:
444	return LSHIFT;
445	case 127:
446	return REM;
447	case 126:
448	return XOR;
449	case 125:
450	return BIT_OR;
451	case 124:
452	return BIT_AND;
453	case 123:
454	return SLASH;
455	case 122:
456	return STAR;
457	case 121:
458	return MINUS;
459	case 120:
460	return PLUS;
461	case 119:
462	return DECR;
463	case 118:
464	return INCR;
465	case 117:
466	return SC_AND;
467	case 116:
468	return SC_OR;
469	case 115:
470	return NE;
471	case 114:
472	return GE;
473	case 113:
474	return LE;
475	case 112:
476	return EQ;
477	case 111:
478	return COLON;
479	case 110:
480	return HOOK;
481	case 109:
482	return TILDE;
483	case 108:
484	return BANG;
485	case 107:
486	return LT;
487	case 106:
488	return ASSIGN;
489	case 105:
490	return AT;
491	case 104:
492	return DOT;
493	case 103:
494	return COMMA;
495	case 102:
496	return SEMICOLON;
497	case 101:
498	return RBRACKET;
499	case 100:
500	return LBRACKET;
501	case 99:
502	return RBRACE;
503	case 98:
504	return LBRACE;
505	case 97:
506	return RPAREN;
507	case 96:
508	return LPAREN;
509	case 95:
510	return PART_LETTER;
511	case 94:
512	return LETTER;
513	case 93:
514	return IDENTIFIER;
515	case 92:
516	return TEXT_BLOCK_CONTENT;
517	case 91:
518	return TEXT_BLOCK_LITERAL;
519	case 90:
520	return ENTER_TEXT_BLOCK;
521	case 89:
522	return STRING_LITERAL;
523	case 88:
524	return CHARACTER_LITERAL;
525	case 87:
526	return UNICODE_ESCAPE;
527	case 86:
528	return HEX_DIGITS;
529	case 85:
530	return HEXADECIMAL_EXPONENT;
531	case 84:
532	return HEXADECIMAL_FLOATING_POINT_LITERAL;
533	case 83:
534	return DECIMAL_EXPONENT;
535	case 82:
536	return DECIMAL_FLOATING_POINT_LITERAL;
537	case 81:
538	return FLOATING_POINT_LITERAL;
539	case 80:
540	return BINARY_LITERAL;
541	case 79:
542	return OCTAL_LITERAL;
543	case 78:
544	return HEX_LITERAL;
545	case 77:
546	return DECIMAL_LITERAL;
547	case 76:
548	return INTEGER_LITERAL;
549	case 75:
550	return LONG_LITERAL;
551	case 74:
552	return TRANSITIVE;
553	case 73:
554	return PROVIDES;
555	case 72:
556	return EXPORTS;
557	case 71:
558	return MODULE;
559	case 70:
560	return USES;
561	case 69:
562	return OPENS;
563	case 68:
564	return OPEN;
565	case 67:
566	return WITH;
567	case 66:
568	return TO;
569	case 65:
570	return REQUIRES;
571	case 64:
572	return YIELD;
573	case 63:
574	return WHILE;
575	case 62:
576	return VOLATILE;
577	case 61:
578	return VOID;
579	case 60:
580	return TRY;
581	case 59:
582	return TRUE;
583	case 58:
584	return TRANSIENT;
585	case 57:
586	return THROWS;
587	case 56:
588	return THROW;
589	case 55:
590	return THIS;
591	case 54:
592	return SYNCHRONIZED;
593	case 53:
594	return SWITCH;
595	case 52:
596	return SUPER;
597	case 51:
598	return STRICTFP;
599	case 50:
600	return STATIC;
601	case 49:
602	return SHORT;
603	case 48:
604	return RETURN;
605	case 47:
606	return PUBLIC;
607	case 46:
608	return PROTECTED;
609	case 45:
610	return PRIVATE;
611	case 44:
612	return PACKAGE;
613	case 43:
614	return NULL;
615	case 42:
616	return NEW;
617	case 41:
618	return NATIVE;
619	case 40:
620	return LONG;
621	case 39:
622	return INTERFACE;
623	case 38:
624	return INT;
625	case 37:
626	return INSTANCEOF;
627	case 36:
628	return IMPORT;
629	case 35:
630	return IMPLEMENTS;
631	case 34:
632	return IF;
633	case 33:
634	return GOTO;
635	case 32:
636	return FOR;
637	case 31:
638	return FLOAT;
639	case 30:
640	return FINALLY;
641	case 29:
642	return FINAL;
643	case 28:
644	return FALSE;
645	case 27:
646	return EXTENDS;
647	case 26:
648	return ENUM;
649	case 25:
650	return ELSE;
651	case 24:
652	return DOUBLE;
653	case 23:
654	return DO;
655	case 22:
656	return _DEFAULT;
657	case 21:
658	return CONTINUE;
659	case 20:
660	return CONST;
661	case 19:
662	return CLASS;
663	case 18:
664	return CHAR;
665	case 17:
666	return CATCH;
667	case 16:
668	return CASE;
669	case 15:
670	return BYTE;
671	case 14:
672	return BREAK;
673	case 13:
674	return BOOLEAN;
675	case 12:
676	return ASSERT;
677	case 11:
678	return ABSTRACT;
679	case 10:
680	return COMMENT_CONTENT;
681	case 9:
682	return MULTI_LINE_COMMENT;
683	case 8:
684	return JAVADOC_COMMENT;
685	case 7:
686	return ENTER_MULTILINE_COMMENT;
687	case 6:
688	return ENTER_JAVADOC_COMMENT;
689	case 5:
690	return SINGLE_LINE_COMMENT;
691	case 4:
692	return OLD_MAC_EOL;
693	case 3:
694	return UNIX_EOL;
695	case 2:
696	return WINDOWS_EOL;
697	case 1:
698	return SPACE;
699	case 0:
700	return EOF;
701	default:
702	throw new IllegalArgumentException(f("Token kind %i is unknown.", kind));
703	}
704	}
705
706	public boolean isPrimitive() {
707	return this == BYTE \|\| this == CHAR \|\| this == SHORT \|\| this == INT \|\| this == LONG \|\| this == FLOAT \|\| this == DOUBLE;
708	}
709
710	public int getKind() {
711	return kind;
712	}
713	}
714
715	public JavaToken.Category getCategory() {
716	return TokenTypes.getCategory(kind);
717	}
718
719	/**
720	* Inserts newToken into the token list just before this token.
721	*/
722	public void insert(JavaToken newToken) {
723	assertNotNull(newToken);
724	getPreviousToken().ifPresent(p -> {
725	p.nextToken = newToken;
726	newToken.previousToken = p;
727	});
728	previousToken = newToken;
729	newToken.nextToken = this;
730	}
731
732	/**
733	* Inserts newToken into the token list just after this token.
734	*/
735	public void insertAfter(JavaToken newToken) {
736	assertNotNull(newToken);
737	getNextToken().ifPresent(n -> {
738	n.previousToken = newToken;
739	newToken.nextToken = n;
740	});
741	nextToken = newToken;
742	newToken.previousToken = this;
743	}
744
745	/**
746	* Links the tokens around the current token together, making the current token disappear from the list.
747	*/
748	public void deleteToken() {
749	final Optional<JavaToken> nextToken = getNextToken();
750	final Optional<JavaToken> previousToken = getPreviousToken();
751	previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null));
752	nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null));
753	}
754
755	/**
756	* Replaces the current token with newToken.
757	*/
758	public void replaceToken(JavaToken newToken) {
759	assertNotNull(newToken);
760	getPreviousToken().ifPresent(p -> {
761	p.nextToken = newToken;
762	newToken.previousToken = p;
763	});
764	getNextToken().ifPresent(n -> {
765	n.previousToken = newToken;
766	newToken.nextToken = n;
767	});
768	}
769
770	/**
771	* @return the last token in the token list.
772	*/
773	public JavaToken findLastToken() {
774	JavaToken current = this;
775	while (current.getNextToken().isPresent()) {
776	current = current.getNextToken().get();
777	}
778	return current;
779	}
780
781	/**
782	* @return the first token in the token list.
783	*/
784	public JavaToken findFirstToken() {
785	JavaToken current = this;
786	while (current.getPreviousToken().isPresent()) {
787	current = current.getPreviousToken().get();
788	}
789	return current;
790	}
791
792	@Override
793	public int hashCode() {
794	int result = kind;
795	result = 31 * result + text.hashCode();
796	return result;
797	}
798
799	@Override
800	public boolean equals(Object o) {
801	if (this == o)
802	return true;
803	if (o == null \|\| getClass() != o.getClass())
804	return false;
805	JavaToken javaToken = (JavaToken) o;
806	if (kind != javaToken.kind)
807	return false;
808	if (!text.equals(javaToken.text))
809	return false;
810	return true;
811	}
812	}
813