[clang] Clear NeedsCleaning flag after ExpandBuiltinMacro (#133574)

After builtin macro expansion in `Preprocessor::ExpandBuiltinMacro` the
result token may have the `Token::NeedsCleaning` flag set which causes
an assertion failure later on when the lexer retrieves the spelling of
the token in `getSpellingSlow`.

This commit adds an `Tok.clearFlag(Token::NeedsCleaning)` call to the
end of `ExpandBuiltinMacro`.

Closes #128384

Author: mariusdr

clang/docs/ReleaseNotes.rst

@@ -404,6 +404,8 @@ Bug Fixes in This Version
 - Defining an integer literal suffix (e.g., ``LL``) before including
   ``<stdint.h>`` in a freestanding build no longer causes invalid token pasting
   when using the ``INTn_C`` macros. (#GH85995)
+- Fixed an assertion failure in the expansion of builtin macros like ``__has_embed()`` with line breaks before the
+  closing paren. (#GH133574)
 - Clang no longer accepts invalid integer constants which are too large to fit
   into any (standard or extended) integer type when the constant is unevaluated.
   Merely forming the token is sufficient to render the program invalid. Code

clang/lib/Lex/PPMacroExpansion.cpp

@@ -2089,6 +2089,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   CreateString(OS.str(), Tok, Tok.getLocation(), Tok.getLocation());
   Tok.setFlagValue(Token::StartOfLine, IsAtStartOfLine);
   Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+  Tok.clearFlag(Token::NeedsCleaning);
 }
 
 void Preprocessor::markMacroAsUsed(MacroInfo *MI) {

clang/test/Preprocessor/embed___has_embed.c

@@ -58,3 +58,22 @@ unsigned char buffer[] = {
 #else
 #error 17
 #endif
+
+#if __has_embed(__FILE__\
+)
+#else
+#error 18
+#endif
+
+#define F __FI\
+LE__
+#if __has_embed(F)
+#else
+#error 19
+#endif
+
+#if __has_embed(F\
+)
+#else 
+#error 20
+#endif

clang/test/Preprocessor/has_attribute.c

@@ -68,3 +68,23 @@ int has_no_volatile_attribute();
 int has_fallthrough;
 #endif
 // CHECK: int has_fallthrough;
+
+#if __has_attribute(F\
+)
+int has_fallthrough_2;
+#endif
+// CHECK: int has_fallthrough_2;
+
+#define F_2 fall\
+through
+
+#if __has_attribute(F_2)
+int has_fallthrough_3;
+#endif
+// CHECK: int has_fallthrough_3;
+
+#if __has_attribute(F_2\
+)
+int has_fallthrough_4;
+#endif
+// CHECK: int has_fallthrough_4;

clang/test/Preprocessor/has_attribute.cpp

@@ -116,6 +116,26 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_cpp_attribute(CF\
+)
+int has_clang_falthrough_5;
+#endif
+// CHECK: int has_clang_falthrough_5;
+
+#define CF_2 clang::\
+fallthrough
+
+#if __has_cpp_attribute(CF_2)
+int has_clang_falthrough_6;
+#endif
+// CHECK: int has_clang_falthrough_6;
+
+#if __has_cpp_attribute(CF_2\
+)
+int has_clang_falthrough_7;
+#endif
+// CHECK: int has_clang_falthrough_7;
 }
 
 // Test for Microsoft __declspec attributes

clang/test/Preprocessor/has_c_attribute.c

@@ -84,3 +84,23 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_c_attribute(CL\
+)
+int has_clang_likely_5;
+#endif
+// CHECK: int has_clang_likely_5;
+
+#define CL_2 clang::\
+likely
+
+#if __has_c_attribute(CL_2)
+int has_clang_likely_6;
+#endif
+// CHECK: int has_clang_likely_6;
+
+#if __has_c_attribute(CL_2\
+)
+int has_clang_likely_7;
+#endif
+// CHECK: int has_clang_likely_7;

clang/test/Preprocessor/has_include.c

@@ -197,3 +197,52 @@ __has_include
 #ifdef FOO
 #elif __has_include(<foo>)
 #endif
+
+#if __has_include(<stdint.h>\
+)
+#else
+  #error "__has_include failed (10)."
+#endif
+
+#define MACRO6 <stdint.h>
+#if __has_include(MACRO6\
+)
+#else
+  #error "__has_include failed (11)."
+#endif
+
+#if __has_include_next(<stdint.h>/*expected-warning {{#include_next in primary source file}}*/\
+)
+#else
+  #error "__has_include_next failed (9)."
+#endif
+
+#if __has_include_next(MACRO6/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (10)."
+#endif
+
+#define MACRO7 <std\
+int.h>
+#if __has_include(MACRO7)
+#else
+  #error "__has_include failed (12)."
+#endif
+
+#if __has_include(MACRO7\
+)
+#else
+  #error "__has_include failed (13)."
+#endif
+
+#if __has_include_next(MACRO7) //expected-warning {{#include_next in primary source file}}
+#else
+  #error "__has_include_next failed (11)."
+#endif
+
+#if __has_include_next(MACRO7/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (12)."
+#endif

clang/test/Preprocessor/pr133574.c

@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 -E -verify %s
+// expected-no-diagnostics
+
+#define DATE_LBR __D\
+ATE__
+
+const char* test1(void) {
+    return __DATE\
+__;
+}
+const char* test2(void) {
+    return DATE_LBR;
+}
+
+#define TIME_LBR __TIME_\
+_
+
+const char* test3(void) {
+    return __TIM\
+E__;
+}
+
+const char* test4(void) {
+    return TIME_LBR;
+}
+
+#define LINE_LBR __LI\
+NE__
+
+int test5(void) {
+    return _\
+_LINE__;
+}
+
+int test6(void) {
+    return LINE_LBR;
+}
+
+#define FILE_LBR __FI\
+LE__
+
+const char* test7(void) {
+    return __\
+FILE__;
+}
+
+const char* test8(void) {
+    return FILE_LBR;
+}
+
+#define FILE_NAME_LBR __FILE_NA\
+ME__
+
+const char* test9(void) {
+    return __FILE_NAM\
+E__;
+}
+
+const char* test10(void) {
+    return FILE_NAME_LBR;
+}
+
+#define BASE_FILE_LBR __BASE_FIL\
+E__
+
+const char* test11(void) {
+    return __BASE_\
+FILE__;
+}
+
+const char* test12(void) {
+    return BASE_FILE_LBR;
+}
+
+#define INCLUDE_LEVEL_LBR __INCLUDE\
+_LEVEL__
+
+int test13(void) {
+    return __IN\
+CLUDE_LEVEL__;
+}
+
+int test14(void) {
+    return INCLUDE_LEVEL_LBR;
+}
+
+#define TIMESTAMP_LBR __TIMESTA\
+MP__
+
+const char* test15(void) {
+    return __TIMESTA\
+MP__;
+}
+
+const char* test16(void) {
+    return TIMESTAMP_LBR;
+}
+
+#define FLT_EVAL_METHOD_LBR __FLT_EVAL_METH\
+OD__
+
+int test17(void) {
+    return __FL\
+T_EVAL_METHOD__;
+}
+
+int test18(void) {
+    return FLT_EVAL_METHOD_LBR;
+}
+
+#define COUNTER_LBR __COUNTE\
+R__
+
+int test19(void) {
+    return _\
+_COUNTER__;
+}
+
+int test20(void) {
+    return COUNTER_LBR;
+}