Fix #680: Add option to keep comments in preprocessed code (#681)

Author: ludviggunne

main.cpp

@@ -242,7 +242,8 @@ int main(int argc, char **argv)
             f.close();
             rawtokens = new simplecpp::TokenList(filename,files,dui,&outputList);
         }
-        rawtokens->removeComments();
+        if (dui.removeComments)
+            rawtokens->removeComments();
         simplecpp::FileDataCache filedata;
         simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList);
         simplecpp::cleanup(filedata);

simplecpp.cpp

@@ -3411,7 +3411,7 @@ simplecpp::FileDataCache simplecpp::load(const simplecpp::TokenList &rawtokens,
     return cache;
 }
 
-static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token *&tok1, simplecpp::MacroMap &macros, std::vector<std::string> &files, simplecpp::OutputList *outputList)
+static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token *&tok1, simplecpp::MacroMap &macros, std::vector<std::string> &files, simplecpp::OutputList *outputList, const simplecpp::DUI &dui)
 {
     const simplecpp::Token * const tok = tok1;
     const simplecpp::MacroMap::const_iterator it = tok->name ? macros.find(tok->str()) : macros.end();
@@ -3442,7 +3442,7 @@ static bool preprocessToken(simplecpp::TokenList &output, const simplecpp::Token
         }
         output.takeTokens(value);
     } else {
-        if (!tok->comment)
+        if (!tok->comment || !dui.removeComments)
             output.push_back(new simplecpp::Token(*tok));
         tok1 = tok->next;
     }
@@ -3717,7 +3717,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
                 TokenList inc2(files);
                 if (!inc1.empty() && inc1.cfront()->name) {
                     const Token *inctok = inc1.cfront();
-                    if (!preprocessToken(inc2, inctok, macros, files, outputList)) {
+                    if (!preprocessToken(inc2, inctok, macros, files, outputList, dui)) {
                         output.clear();
                         return;
                     }
@@ -3899,7 +3899,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
                         maybeUsedMacros[rawtok->next->str()].emplace_back(rawtok->next->location);
 
                         const Token *tmp = tok;
-                        if (!preprocessToken(expr, tmp, macros, files, outputList)) {
+                        if (!preprocessToken(expr, tmp, macros, files, outputList, dui)) {
                             output.clear();
                             return;
                         }
@@ -3997,7 +3997,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
         const Location loc(rawtok->location);
         TokenList tokens(files);
 
-        if (!preprocessToken(tokens, rawtok, macros, files, outputList)) {
+        if (!preprocessToken(tokens, rawtok, macros, files, outputList, dui)) {
             output.clear();
             return;
         }

test.cpp

@@ -533,12 +533,15 @@ static void combineOperators_ellipsis()
 
 static void comment()
 {
+    simplecpp::DUI dui;
+    dui.removeComments = true;
+
     ASSERT_EQUALS("// abc", readfile("// abc"));
-    ASSERT_EQUALS("", preprocess("// abc"));
+    ASSERT_EQUALS("", preprocess("// abc", dui));
     ASSERT_EQUALS("/*\n\n*/abc", readfile("/*\n\n*/abc"));
-    ASSERT_EQUALS("\n\nabc", preprocess("/*\n\n*/abc"));
+    ASSERT_EQUALS("\n\nabc", preprocess("/*\n\n*/abc", dui));
     ASSERT_EQUALS("* p = a / * b / * c ;", readfile("*p=a/ *b/ *c;"));
-    ASSERT_EQUALS("* p = a / * b / * c ;", preprocess("*p=a/ *b/ *c;"));
+    ASSERT_EQUALS("* p = a / * b / * c ;", preprocess("*p=a/ *b/ *c;", dui));
 }
 
 static void comment_multiline()
@@ -578,6 +581,27 @@ static void comment_multiline()
     ASSERT_EQUALS("// abc\ndef", readfile("// abc\\\ndef"));
 }
 
+static void keep_comments()
+{
+    simplecpp::DUI dui;
+    dui.removeComments = false;
+
+    {
+        const char code[] = "/* comment */\n";
+        ASSERT_EQUALS("/* comment */", preprocess(code,dui));
+    }
+
+    {
+        const char code[] = "// comment\n";
+        ASSERT_EQUALS("// comment", preprocess(code,dui));
+    }
+
+    {
+        const char code[] = "#define MACRO /* comment */\nMACRO\n";
+        ASSERT_EQUALS("\n/* comment */", preprocess(code,dui));
+    }
+}
+
 
 static void constFold()
 {
@@ -2584,13 +2608,16 @@ static void location11()
 
 static void location12()
 {
+    simplecpp::DUI dui;
+    dui.removeComments = true;
+
     const char code[] =
         "/**//**/#/**//**/line/**//**/3/**//**/\"file.c\"/**/\n"
         "__LINE__ __FILE__\n";
     ASSERT_EQUALS("\n"
                   "#line 3 \"file.c\"\n"
                   "3 \"file.c\"",
-                  preprocess(code));
+                  preprocess(code, dui));
 }
 
 
@@ -3244,6 +3271,7 @@ static void include9()
     simplecpp::TokenList out(files);
     simplecpp::DUI dui;
     dui.includePaths.emplace_back(".");
+    dui.removeComments = true;
     simplecpp::preprocess(out, rawtokens_c, files, cache, dui);
 
     ASSERT_EQUALS("\n#line 2 \"1.h\"\nx = 1 ;", out.stringify());
@@ -4383,6 +4411,7 @@ static void runTests(int argc, char **argv, Input input)
 
     TEST_CASE(comment);
     TEST_CASE(comment_multiline);
+    TEST_CASE(keep_comments);
 
     TEST_CASE(constFold);