[libclang] Fix issue with clang_tokenize and make sure it interprets CXSourceRange as half-open character range.

akyrtzi · akyrtzi · commit 3b61b92e0fdf · 2016-11-09T23:58:39.000Z
Patch provided by Emilio Cobos Álvarez! (https://reviews.llvm.org/D26446) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@286421 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/test/Index/annotate-tokens.c b/test/Index/annotate-tokens.c
@@ -239,3 +239,7 @@ void test() {
 // CHECK-RANGE2: Identifier: "reg" [68:3 - 68:6] DeclRefExpr=reg:67:7
 // CHECK-RANGE2: Punctuation: "." [68:6 - 68:7] MemberRefExpr=field:62:9
 // CHECK-RANGE2: Identifier: "field" [68:7 - 68:12] MemberRefExpr=field:62:9
+
+// RUN: c-index-test -test-annotate-tokens=%s:68:15:68:16 %s | FileCheck %s -check-prefix=CHECK-RANGE3
+// CHECK-RANGE3: Literal: "1" [68:15 - 68:16] IntegerLiteral=
+// CHECK-RANGE3-NOT: Punctuation: ";"
diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp
@@ -6156,7 +6156,7 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
     }
     CXTokens.push_back(CXTok);
     previousWasAt = Tok.is(tok::at);
-  } while (Lex.getBufferLocation() <= EffectiveBufferEnd);
+  } while (Lex.getBufferLocation() < EffectiveBufferEnd);
 }
 
 void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,

Original file line number	Diff line number	Diff line change
`@@ -6156,7 +6156,7 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range,`
`6156`	`6156`	`}`
`6157`	`6157`	`CXTokens.push_back(CXTok);`
`6158`	`6158`	`previousWasAt = Tok.is(tok::at);`
`6159`		`- } while (Lex.getBufferLocation() <= EffectiveBufferEnd);`
	`6159`	`+ } while (Lex.getBufferLocation() < EffectiveBufferEnd);`
`6160`	`6160`	`}`
`6161`	`6161`
`6162`	`6162`	`void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,`