Fix: Resolved deterministic memory leak and dangling pointer in SQLParser::tokenize

.github/workflows/ci.yml

@@ -14,6 +14,8 @@ jobs:
     env:
       CC: ${{matrix.cc}}
       CXX: ${{matrix.cxx}}
+      CXXFLAGS: ${{matrix.env_cxxflags}}
+      LDFLAGS: ${{matrix.env_ldflags}}
     defaults:
       run:
         shell: bash
@@ -46,9 +48,24 @@ jobs:
             cxx: clang++
             os: macos-latest
 
+          - name: clang-sanitizer-ubuntu
+            cc: clang-19
+            cxx: clang++-19
+            os: ubuntu-latest
+            container: ubuntu:24.04
+            env_cxxflags: "-fsanitize=address,undefined"
+            env_ldflags: "-fsanitize=address,undefined"
+
+          - name: clang-sanitizer-macOS
+            cc: clang
+            cxx: clang++
+            os: macos-latest
+            env_cxxflags: "-fsanitize=address,undefined"
+            env_ldflags: "-fsanitize=address,undefined"
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         if: matrix.name != 'gcc-6'
 
       - name: Checkout (Ubuntu 18.04)
@@ -67,17 +84,17 @@ jobs:
           git checkout $GITHUB_HEAD_REF
 
       - name: Setup (macOS)
-        if: matrix.name == 'clang-macOS'
+        if: matrix.os == 'macos-latest'
         run: |
           brew install bison flex
           echo "BISON=$(brew --prefix bison)/bin/bison" >> $GITHUB_ENV
           echo "FLEX=$(brew --prefix flex)/bin/flex" >> $GITHUB_ENV
 
       - name: Setup (Ubuntu)
-        if: matrix.name != 'clang-macOS'
+        if: matrix.os == 'ubuntu-latest'
         run: |
           apt-get update
-          apt-get install --no-install-recommends -y bison flex ${CC} ${CXX} make valgrind
+          apt-get install --no-install-recommends -y bison flex ${CC} ${CXX} make valgrind ${{ matrix.name == 'clang-sanitizer-ubuntu' && 'libclang-rt-19-dev' || '' }}
           echo "BISON=bison" >> $GITHUB_ENV
           echo "FLEX=flex" >> $GITHUB_ENV
 

Makefile

@@ -36,7 +36,9 @@ GMAKE = make mode=$(mode)
 NAME := sqlparser
 PARSER_CPP = $(SRCPARSER)/bison_parser.cpp  $(SRCPARSER)/flex_lexer.cpp
 PARSER_H   = $(SRCPARSER)/bison_parser.h    $(SRCPARSER)/flex_lexer.h
-LIB_CFLAGS = -std=c++17 $(OPT_FLAG)
+LIB_CFLAGS = -std=c++17 $(OPT_FLAG) $(CXXFLAGS)
+LIB_LFLAGS = $(LDFLAGS)
+
 
 relaxed_build ?= "off"
 ifeq ($(relaxed_build), on)
@@ -54,12 +56,12 @@ static ?= no
 ifeq ($(static), yes)
 	LIB_BUILD  = lib$(NAME).a
 	LIBLINKER  = $(AR)
-	LIB_LFLAGS = rs
+	LIB_LFLAGS += rs
 else
 	LIB_BUILD   = lib$(NAME).so
 	LIBLINKER   = $(CXX)
 	LIB_CFLAGS += -fPIC
-	LIB_LFLAGS  = -shared -o
+	LIB_LFLAGS += -shared -o
 endif
 LIB_CPP = $(sort $(shell find $(SRC) -name '*.cpp' -not -path "$(SRCPARSER)/*") $(PARSER_CPP))
 LIB_H   = $(shell find $(SRC) -name '*.h' -not -path "$(SRCPARSER)/*") $(PARSER_H)

src/SQLParser.cpp

@@ -59,11 +59,13 @@ bool SQLParser::tokenize(const std::string& sql, std::vector<int16_t>* tokens) {
   int16_t token = hsql_lex(&yylval, &yylloc, scanner);
   while (token != 0) {
     tokens->push_back(token);
-    token = hsql_lex(&yylval, &yylloc, scanner);
 
     if (token == SQL_IDENTIFIER || token == SQL_STRING) {
       free(yylval.sval);
+      yylval.sval = nullptr;
     }
+    token = hsql_lex(&yylval, &yylloc, scanner);
+
   }
 
   hsql__delete_buffer(state, scanner);

test/sql_parser.cpp

@@ -42,3 +42,18 @@ TEST(SQLParserTokenizeStringifyTest) {
   ASSERT(query == cache[token_string]);
   ASSERT(&query != &cache[token_string]);
 }
+
+// Regression test for the memory leak reported in issue #261.
+TEST(SQLParserTokenizeLeakRegressionTest) {
+
+  const std::string query = "'string_1' 'string_2' 'string_3';";
+  std::vector<int16_t> tokens;
+
+  ASSERT(SQLParser::tokenize(query, &tokens));
+
+  ASSERT_EQ(tokens.size(), 4);
+  ASSERT_EQ(tokens[0], SQL_STRING);
+  ASSERT_EQ(tokens[1], SQL_STRING);
+  ASSERT_EQ(tokens[2], SQL_STRING);
+  ASSERT_EQ(tokens[3], ';');
+}