sd: sync to master-506-c9cd497 (#1991)

2026-09-18 00:35:04 +02:00 · 2026-02-23 06:35:59 -03:00
parent 06c0ffaead
commit a6c0a224b2
2 changed files with 14 additions and 6 deletions
@@ -919,15 +919,21 @@ std::vector<std::string> token_split(const std::string& text) {

        // `\s*[\r\n]+|\s+(?!\S)|\s+`
        if (is_space(cp)) {
-            std::string token = codepoint_to_utf8(cp);
-            ++i;
+            std::string token;
+            bool saw_new_line = false;

            while (i < cps.size() && is_space(cps[i])) {
                token += codepoint_to_utf8(cps[i]);
-                ++i;
+
                if (cps[i] == U'\r' || cps[i] == U'\n') {
-                    break;
+                    saw_new_line = true;
+                } else {
+                    if (saw_new_line) {
+                        break;
+                    }
                }
+
+                ++i;
            }

            tokens.push_back(token);