preproc: (shared) COMPOUND_STRING (#9086)

2026-03-21 18:04:50 -05:00 · 2026-03-04 16:09:12 +00:00 · 2026-03-04 16:09:12 +00:00 · 44e9991e8b
commit 44e9991e8b
parent fc9427b1bf
9 changed files with 186 additions and 78 deletions
--- a/include/global.h
+++ b/include/global.h
@ -42,6 +42,7 @@
 // We define these when using certain IDEs to fool preproc
 #define _(x)        {x}
 #define __(x)       {x}
+#define COMPOUND_STRING(x) 0
 #define INCBIN(...) {0}
 #define INCBIN_U8   INCBIN
 #define INCBIN_U16  INCBIN
--- a/include/metaprogram.h
+++ b/include/metaprogram.h
@ -31,8 +31,7 @@
 #define APPEND_COMMA(a) a,

 /* Converts a string to a compound literal, essentially making it a pointer to const u8 */
-#define COMPOUND_STRING(str) (const u8[]) _(str)
-#define COMPOUND_STRING_SIZE_LIMIT(str, limit) (const u8[COMPOUND_STRING_CHECK_SIZE(str, limit)]) _(str)
+#define COMPOUND_STRING_SIZE_LIMIT(str, limit) ((sizeof(COMPOUND_STRING(str)) <= (limit)) ? COMPOUND_STRING(str) : (const u8 [COMPOUND_STRING_CHECK_SIZE(str, limit)]) _(str))

 /* Used for COMPOUND_STRING_SIZE_LIMIT. Stupid, but makes sure we only get
 * one error message regardless of how many characters over the limit we are.
--- a/include/test/battle.h
+++ b/include/test/battle.h
@ -1229,8 +1229,7 @@ void SendOut(u32 sourceLine, struct BattlePokemon *, u32 partyIndex);
 #define HP_BAR(battler, ...) QueueHP(__LINE__, battler, (struct HPEventContext) { R_APPEND_TRUE(__VA_ARGS__) })
 #define SUB_HIT(battler, ...) QueueSubHit(__LINE__, battler, (struct SubHitEventContext) { R_APPEND_TRUE(__VA_ARGS__) })
 #define EXPERIENCE_BAR(battler, ...) QueueExp(__LINE__, battler, (struct ExpEventContext) { R_APPEND_TRUE(__VA_ARGS__) })
-// Static const is needed to make the modern compiler put the pattern variable in the .rodata section, instead of putting it on stack(which can break the game).
-#define MESSAGE(pattern) do {static const u8 msg[] = _(pattern); QueueMessage(__LINE__, msg);} while (0)
+#define MESSAGE(pattern) QueueMessage(__LINE__, COMPOUND_STRING(pattern))
 #define STATUS_ICON(battler, status) QueueStatus(__LINE__, battler, (struct StatusEventContext) { status })
 #define CATCHING_CHANCE(address) QueueCatchingChance(__LINE__, address)
 #define FREEZE_OR_FROSTBURN_STATUS(battler, isFrostbite) \
--- a/ld_script_modern.ld
+++ b/ld_script_modern.ld
@ -73,6 +73,12 @@ SECTIONS {
 		data/*.o(script_data);
    } > ROM =0

+    .rodata.compound_string :
+    SUBALIGN(0)
+    {
+        *.o(.rodata.compound_string.*);
+    } > ROM =0
+
    .rodata :
    ALIGN(4)
    {
--- a/ld_script_test.ld
+++ b/ld_script_test.ld
@ -96,6 +96,12 @@ SECTIONS {
        *libnosys.a:*.o(.text*);
    } > ROM =0

+    .rodata.compound_string :
+    SUBALIGN(0)
+    {
+        *.o(.rodata.compound_string.*);
+    } > ROM =0
+
    .rodata :
    ALIGN(4)
    {
--- a/src/contest.c
+++ b/src/contest.c
@ -361,29 +361,29 @@ EWRAM_DATA u8 gCurContestWinnerSaveIdx = 0;
 COMMON_DATA rng_value_t gContestRngValue = {0};

 //Text
-const u8 gText_LinkStandby4[] = COMPOUND_STRING("Link standby!");
+const u8 gText_LinkStandby4[] = _("Link standby!");

-const u8 gText_AppealNumWhichMoveWillBePlayed[] = COMPOUND_STRING("Appeal no. {STR_VAR_1}!\nWhich move will be played?");
-const u8 gText_AppealNumButItCantParticipate[] = COMPOUND_STRING("Appeal no. {STR_VAR_1}!\nBut it can't participate!");
-const u8 gText_MonAppealedWithMove[] = COMPOUND_STRING("{STR_VAR_1} appealed with\n{STR_VAR_2}!");
-const u8 gText_MonWasWatchingOthers[] = COMPOUND_STRING("{STR_VAR_1} was watching\nthe others.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_AllOutOfAppealTime[] = COMPOUND_STRING("We're all out of\nAppeal Time!{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_JudgeLookedAtMonExpectantly[] = COMPOUND_STRING("The JUDGE looked at\n{STR_VAR_1} expectantly.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_AppealComboWentOverWell[] = COMPOUND_STRING("The appeal combo went\nover well.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_AppealComboWentOverVeryWell[] = COMPOUND_STRING("The appeal combo went\nover very well.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_AppealComboWentOverExcellently[] = COMPOUND_STRING("The appeal combo went\nover excellently.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonWasTooNervousToMove[] = COMPOUND_STRING("{STR_VAR_1} was too\nnervous to move.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_CouldntImproveItsCondition[] = COMPOUND_STRING("But it couldn't improve\nits condition…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_BadConditionResultedInWeakAppeal[] = COMPOUND_STRING("Its bad condition\nresulted in a weak appeal.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonWasUnaffected[] = COMPOUND_STRING("{STR_VAR_1} was\nunaffected.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_RepeatedAppeal[] = COMPOUND_STRING("{STR_VAR_1} disappointed\nby repeating an appeal.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonsXWentOverGreat[] = COMPOUND_STRING("{STR_VAR_1}'s {STR_VAR_3}\nwent over great.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonsXDidntGoOverWell[] = COMPOUND_STRING("{STR_VAR_1}'s {STR_VAR_3}\ndidn't go over well here…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonsXGotTheCrowdGoing[] = COMPOUND_STRING("{STR_VAR_1}'s {STR_VAR_3}\ngot the crowd going.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonCantAppealNextTurn[] = COMPOUND_STRING("{STR_VAR_1} can't appeal\nnext turn…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_AttractedCrowdsAttention[] = COMPOUND_STRING("It attracted the crowd's\nattention.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_CrowdContinuesToWatchMon[] = COMPOUND_STRING("The crowd continues to\nwatch {STR_VAR_3}.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
-const u8 gText_MonsMoveIsIgnored[] = COMPOUND_STRING("{STR_VAR_1}'s\n{STR_VAR_2} is ignored.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AppealNumWhichMoveWillBePlayed[] = _("Appeal no. {STR_VAR_1}!\nWhich move will be played?");
+const u8 gText_AppealNumButItCantParticipate[] = _("Appeal no. {STR_VAR_1}!\nBut it can't participate!");
+const u8 gText_MonAppealedWithMove[] = _("{STR_VAR_1} appealed with\n{STR_VAR_2}!");
+const u8 gText_MonWasWatchingOthers[] = _("{STR_VAR_1} was watching\nthe others.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AllOutOfAppealTime[] = _("We're all out of\nAppeal Time!{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_JudgeLookedAtMonExpectantly[] = _("The JUDGE looked at\n{STR_VAR_1} expectantly.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AppealComboWentOverWell[] = _("The appeal combo went\nover well.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AppealComboWentOverVeryWell[] = _("The appeal combo went\nover very well.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AppealComboWentOverExcellently[] = _("The appeal combo went\nover excellently.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonWasTooNervousToMove[] = _("{STR_VAR_1} was too\nnervous to move.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_CouldntImproveItsCondition[] = _("But it couldn't improve\nits condition…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_BadConditionResultedInWeakAppeal[] = _("Its bad condition\nresulted in a weak appeal.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonWasUnaffected[] = _("{STR_VAR_1} was\nunaffected.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_RepeatedAppeal[] = _("{STR_VAR_1} disappointed\nby repeating an appeal.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonsXWentOverGreat[] = _("{STR_VAR_1}'s {STR_VAR_3}\nwent over great.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonsXDidntGoOverWell[] = _("{STR_VAR_1}'s {STR_VAR_3}\ndidn't go over well here…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonsXGotTheCrowdGoing[] = _("{STR_VAR_1}'s {STR_VAR_3}\ngot the crowd going.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonCantAppealNextTurn[] = _("{STR_VAR_1} can't appeal\nnext turn…{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_AttractedCrowdsAttention[] = _("It attracted the crowd's\nattention.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_CrowdContinuesToWatchMon[] = _("The crowd continues to\nwatch {STR_VAR_3}.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");
+const u8 gText_MonsMoveIsIgnored[] = _("{STR_VAR_1}'s\n{STR_VAR_2} is ignored.{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}{PAUSE 0x0F}");

 static const u8 sSliderHeartYPositions[CONTESTANT_COUNT] =
 {
--- a/src/contest_painting.c
+++ b/src/contest_painting.c
@ -42,7 +42,7 @@ static void PrintContestPaintingCaption(u8, u8);
 static void VBlankCB_ContestPainting(void);
 static void _InitContestMonPixels(u8 *spriteGfx, u16 *palette, u16 (*destPixels)[64][64]);

-const u8 gContestHallPaintingCaption[] = COMPOUND_STRING("{STR_VAR_1}\n{STR_VAR_2}'s {STR_VAR_3}");
+const u8 gContestHallPaintingCaption[] = _("{STR_VAR_1}\n{STR_VAR_2}'s {STR_VAR_3}");

 static const u16 sPictureFramePalettes[]          = INCBIN_U16("graphics/picture_frame/bg.gbapal");
 static const u32 sPictureFrameTiles_Cool[]        = INCBIN_U32("graphics/picture_frame/cool.4bpp.smol");
--- a/tools/preproc/c_file.cpp
+++ b/tools/preproc/c_file.cpp
@ -62,6 +62,26 @@ CFile::~CFile()
    free(m_buffer);
 }

+void CFile::printf(const char *format, ...)
+{
+    std::va_list args;
+    va_start(args, format);
+    int n = vsnprintf(NULL, 0, format, args);
+    va_end(args);
+
+    char *s = new char[n + 1];
+    va_start(args, format);
+    std::vsnprintf(s, n + 1, format, args);
+    va_end(args);
+    m_output.append(s);
+    delete[] s;
+}
+
+void CFile::putchar(char c)
+{
+    m_output.append(1, c);
+}
+
 void CFile::Preproc()
 {
    char stringChar = 0;
@ -72,27 +92,28 @@ void CFile::Preproc()
        {
            if (m_buffer[m_pos] == stringChar)
            {
-                std::putchar(stringChar);
+                putchar(stringChar);
                m_pos++;
                stringChar = 0;
            }
            else if (m_buffer[m_pos] == '\\' && m_buffer[m_pos + 1] == stringChar)
            {
-                std::putchar('\\');
-                std::putchar(stringChar);
+                putchar('\\');
+                putchar(stringChar);
                m_pos += 2;
            }
            else
            {
                if (m_buffer[m_pos] == '\n')
                    m_lineNum++;
-                std::putchar(m_buffer[m_pos]);
+                putchar(m_buffer[m_pos]);
                m_pos++;
            }
        }
        else
        {
            TryConvertString();
+            TryConvertCompoundString();
            TryConvertIncbin();

            if (m_pos >= m_size)
@ -100,7 +121,7 @@ void CFile::Preproc()

            char c = m_buffer[m_pos++];

-            std::putchar(c);
+            putchar(c);

            if (c == '\n')
                m_lineNum++;
@ -110,6 +131,28 @@ void CFile::Preproc()
                stringChar = '\'';
        }
    }
+
+    // TODO: Share the tails of compound strings where possible (like the
+    // 'ld' optimization for 'SHF_MERGE | SHF_STRINGS').
+    for (auto it : m_compoundStrings)
+    {
+        // HINT: GCC puts the section name into the assembly without any
+        // validation, so we're able to apply the 'SHF_MERGE' flag and
+        // our own 'sh_entsize' by using '@' to comment out what GCC
+        // would have put after the section name.
+        // This will not work with Clang, see
+        // https://discourse.llvm.org/t/creating-shf-merge-shf-strings-section/86399
+        std::printf("static const unsigned char __attribute__((section(\".rodata.compound_string.%016lx,\\\"aM\\\",%%progbits,%ld @\"))) sCompoundString_%016lx[] = {", it.second, it.first.size(), it.second);
+        if (it.first.size() > 0)
+        {
+            std::printf(" 0x%02X", it.first[0]);
+            for (std::size_t i = 1; i < it.first.size(); i++)
+                std::printf(", 0x%02X", it.first[i]);
+        }
+        std::printf(" };\n");
+    }
+
+    std::puts(m_output.c_str());
 }

 bool CFile::ConsumeHorizontalWhitespace()
@ -129,7 +172,7 @@ bool CFile::ConsumeNewline()
    {
        m_pos += 2;
        m_lineNum++;
-        std::putchar('\n');
+        putchar('\n');
        return true;
    }

@ -137,7 +180,7 @@ bool CFile::ConsumeNewline()
    {
        m_pos++;
        m_lineNum++;
-        std::putchar('\n');
+        putchar('\n');
        return true;
    }

@ -150,6 +193,48 @@ void CFile::SkipWhitespace()
        ;
 }

+std::vector<unsigned char> CFile::ConvertString()
+{
+    std::vector<unsigned char> converted;
+
+    while (true)
+    {
+        SkipWhitespace();
+
+        if (m_buffer[m_pos] == '"')
+        {
+            unsigned char s[kMaxStringLength];
+            int length = 0;
+            StringParser stringParser(m_buffer, m_size);
+            try
+            {
+                m_pos += stringParser.ParseString(m_pos, s, length);
+            }
+            catch (std::runtime_error& e)
+            {
+                RaiseError(e.what());
+            }
+            converted.insert(converted.end(), s, s + length);
+        }
+        else if (m_buffer[m_pos] == ')')
+        {
+            m_pos++;
+            break;
+        }
+        else
+        {
+            if (m_pos >= m_size)
+                RaiseError("unexpected EOF");
+            if (IsAsciiPrintable(m_buffer[m_pos]))
+                RaiseError("unexpected character '%c'", m_buffer[m_pos]);
+            else
+                RaiseError("unexpected character '\\x%02X'", m_buffer[m_pos]);
+        }
+    }
+
+    return converted;
+}
+
 void CFile::TryConvertString()
 {
    long oldPos = m_pos;
@ -180,50 +265,55 @@ void CFile::TryConvertString()

    SkipWhitespace();

-    std::printf("{ ");
+    printf("{ ");

-    while (1)
-    {
-        SkipWhitespace();
-
-        if (m_buffer[m_pos] == '"')
-        {
-            unsigned char s[kMaxStringLength];
-            int length;
-            StringParser stringParser(m_buffer, m_size);
-
-            try
-            {
-                m_pos += stringParser.ParseString(m_pos, s, length);
-            }
-            catch (std::runtime_error& e)
-            {
-                RaiseError(e.what());
-            }
-
-            for (int i = 0; i < length; i++)
-                printf("0x%02X, ", s[i]);
-        }
-        else if (m_buffer[m_pos] == ')')
-        {
-            m_pos++;
-            break;
-        }
-        else
-        {
-            if (m_pos >= m_size)
-                RaiseError("unexpected EOF");
-            if (IsAsciiPrintable(m_buffer[m_pos]))
-                RaiseError("unexpected character '%c'", m_buffer[m_pos]);
-            else
-                RaiseError("unexpected character '\\x%02X'", m_buffer[m_pos]);
-        }
-    }
+    std::vector<unsigned char> converted = ConvertString();
+    for (std::size_t i = 0; i < converted.size(); i++)
+        printf("0x%02X, ", converted[i]);

    if (noTerminator)
-        std::printf(" }");
+        printf(" }");
    else
-        std::printf("0xFF }");
+        printf("0xFF }");
+}
+
+static std::uint64_t fnv1a(const std::vector<unsigned char>& bytes)
+{
+    std::uint64_t hash = 0xcbf29ce484222325UL;
+    for (auto b : bytes)
+        hash = (hash ^ b) * 0x00000100000001b3UL;
+    return hash;
+}
+
+void CFile::TryConvertCompoundString()
+{
+    long oldPos = m_pos;
+    long oldLineNum = m_lineNum;
+    std::string ident = "COMPOUND_STRING";
+
+    if ((m_pos > 0 && IsIdentifierChar(m_buffer[m_pos - 1])) || !CheckIdentifier(ident))
+        return;
+
+    m_pos += ident.length();
+
+    SkipWhitespace();
+
+    if (m_buffer[m_pos] != '(')
+    {
+        m_pos = oldPos;
+        m_lineNum = oldLineNum;
+        return;
+    }
+
+    m_pos++;
+    std::vector<unsigned char> converted = ConvertString();
+    converted.push_back(0xFF);
+
+    std::uint64_t hash = fnv1a(converted);
+    m_compoundStrings[converted] = hash;
+    // WARNING: Assumes no collisions.
+    // TODO: Incorporate filename to prevent cross-TU collisions.
+    printf("sCompoundString_%016lx", hash);
 }

 bool CFile::CheckIdentifier(const std::string& ident)
@ -317,7 +407,7 @@ void CFile::TryConvertIncbin()

    m_pos++;

-    std::printf("{");
+    printf("{");

    while (true)
    {
@ -372,9 +462,9 @@ void CFile::TryConvertIncbin()
            offset += size;

            if (isSigned)
-                std::printf("%d,", data);
+                printf("%d,", data);
            else
-                std::printf("%uu,", data);
+                printf("%uu,", data);
        }

        SkipWhitespace();
@ -390,7 +480,7 @@ void CFile::TryConvertIncbin()

    m_pos++;

-    std::printf("}");
+    printf("}");
 }

 // Reports a diagnostic message.
--- a/tools/preproc/c_file.h
+++ b/tools/preproc/c_file.h
@ -25,6 +25,7 @@
 #include <cstdint>
 #include <string>
 #include <memory>
+#include <map>
 #include "preproc.h"

 class CFile
@ -43,17 +44,23 @@ private:
    long m_lineNum;
    std::string m_filename;
    bool m_isStdin;
+    std::map<std::vector<unsigned char>, std::uint64_t> m_compoundStrings;
+    std::string m_output;

    bool ConsumeHorizontalWhitespace();
    bool ConsumeNewline();
    void SkipWhitespace();
+    std::vector<unsigned char> ConvertString();
    void TryConvertString();
+    void TryConvertCompoundString();
    std::unique_ptr<unsigned char[]> ReadWholeFile(const std::string& path, int& size);
    bool CheckIdentifier(const std::string& ident);
    void TryConvertIncbin();
    void ReportDiagnostic(const char* type, const char* format, std::va_list args);
    void RaiseError(const char* format, ...);
    void RaiseWarning(const char* format, ...);
+    void printf(const char *format, ...);
+    void putchar(char c);
 };

 #endif // C_FILE_H