LEX (File Format)/C example

While the specifications for LEX files have always been publicly available here on Tockdom some people expressed concerns about implementation differences when implemening LEX themselves, so we're making the LEX-related code parts from LE-CODE available here. All the code on this particular Tockdom page is under the MIT license "Copyright 2023 Wiimm & Leseratte" so there should be no licensing issues putting it into any kind of distribution, open-source or closed-source.

Structures

lex_header_t

typedef struct lex_header_t
{
    char	magic[4];	// always LEX_BIN_MAGIC
    u16		major_version;	// usually LEX_MAJOR_VERSION
    u16		minor_version;	// usually LEX_MINOR_VERSION
    u32		size;		// size of this file (header+streams)
    u32		element_off;	// offset of first lex_element_t, 32-bit aligned
}
__attribute__ ((packed)) lex_header_t;

lex_element_t

typedef struct lex_element_t
{
    u32		magic;		// identification of section
    u32		size;		// size of 'data' (this header excluded)
    u8		data[];		// section data, 32-bit aligned
}
__attribute__ ((packed)) lex_element_t;

Section-independant code

The main hook for LEX files is at 80512820, just before a track's KMP file is getting loaded. In that place we open the LEX file (if present), reset all modifications previously done (lex_file_reset), then start parsing the lex file (lex_file_parse).

    lex_ptr = open_szs_subfile(file_open_data_addr, 1, "course.lex", 0); // call to 805411fc
    lex_file_reset();
    lex_file_parse();
    set_online_time_limit_by_lex(lex_set1_data.apply_online_sec);

The lex_file_parse sets up various global variables that are then checked by other hooks around the code whenever needed (and has special handling for the cannon subsection CANN because of legacy reasons):

void lex_file_parse()

void lex_file_parse()
{
    if ( !lex_ptr || isModeWorldwide() )
        return;

    if (lex_ptr->magic != 0x4c452d58) {
        LEReport("This is not a valid course.lex file\n");
        return;
    }

    if (lex_ptr->major_version != 1) {
        LEReport("course.lex file has invalid version (%d), need %d\n", lex_ptr->major_version, 1);
        return;
    }

    lex_element_t * lex_element_ptr = (lex_element_t *)((int)((int)lex_ptr + (int)lex_ptr->element_off));

    void * object_data_ptr;

    while (lex_element_ptr->magic != 0)
    {
        object_data_ptr = (void *)((int)lex_element_ptr + 8);

        switch (lex_element_ptr->magic)
        {
            case 0x43414e4e: // "CANN": Cannon settings.
                // object data ptr contains offset to cannon data
                // The game stores cannon speeds in a float[4] for each cannon type which is at PAL 808b5ce8.
                // This array is loaded / used at PAL 805850a4, in a set of lis/addi instructions. 
                // We're just patching these, so cannon data is loaded from the temporary LEX buffer instead.
                object_data_ptr = (void *)(((int)object_data_ptr) + 4); // skip size
                code_patcher(&mkwfun_lex_cannon_type_patch_addr, 0, 0x3c600000 + (((unsigned int)object_data_ptr >> 16) & 0xffff));
                code_patcher(&mkwfun_lex_cannon_type_patch_addr, 4, 0x60630000 + ((unsigned int)object_data_ptr & 0xffff));
                break;

            case 0x48495054: // "HIPT": HIde Position Tracker
                lex_hipt_len = lex_element_ptr->size / sizeof(*lex_hipt_list);
                lex_hipt_list = object_data_ptr;
                LEDebugReport("XPF: load LEX/HIPT, size=%d -> n=%d\n", lex_element_ptr->size, lex_hipt_len);
                break;

            case 0x52495450: // "RITP": Random ITem Points
                lex_ritp_len = lex_element_ptr->size / sizeof(*lex_ritp_list);
                lex_ritp_list = object_data_ptr;
                LEDebugReport("XPF: load LEX/RITP, size=%d -> n=%d\n", lex_element_ptr->size, lex_ritp_len);
                break;


            case 0x53455431: // "SET1": Settings #1
		{
		    unsigned size = lex_element_ptr->size;
		    LEDebugReport("load LEX/SET1, size=%d, max-size=%d\n",
				size, sizeof(lex_set1_t));
		    if (size > sizeof(lex_set1_t))
			size = sizeof(lex_set1_t);
		    memcpy(&lex_set1_data, object_data_ptr, size);
		 }
		 break;

            case 0x54455354: // "TEST": Settings to tests.
		{
		    unsigned size = lex_element_ptr->size;
		    LEDebugReport("XPF: load LEX/TEST, size=%d, max-size=%d\n", size, sizeof(lex_test_t));
		    if (size > sizeof(lex_test_t))
			size = sizeof(lex_test_t);
		    memcpy(&lex_test_data, object_data_ptr, size);
		}
		break;

            // case 0x2d2d2d2d:	// "----", invalidated section, ignore.
            default: // ignore unknown sections
                break;
        }

        // go to next element:
        lex_element_ptr = (lex_element_t *)((int)lex_element_ptr + lex_element_ptr->size + 8);
    }

    lex_ptr = 0;
}

The lex_file_reset function is called every time a new track loads as well, and resets all modifications done by a LEX file so the next track starts with a clean state. The cannon speed modifications will be directly reset here, any other modifications just set a bunch of global variables to their proper state and the functions hooking into the game code will react accordingly:

void lex_file_reset()

const lex_dev1_t lex_dev1_data_reset = { 0 };
const lex_set1_t lex_set1_data_reset = { { 1.0, 1.0, 1.0 }, 0, 0, 0 };
const lex_test_t lex_test_data_reset = { 0, 0, 0, -1, 0, 0 };

void lex_file_reset()
{
        // reset cannon param (PAL 805850a4)
        #if REGION_LETTER == 'P'
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 0, 0x3c60808b);
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 4, 0x38635ce8);
        #elif REGION_LETTER == 'E'
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 0, 0x3c60808b);
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 4, 0x38631428);
        #elif REGION_LETTER == 'J'
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 0, 0x3c60808b);
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 4, 0x38634e48);
        #elif REGION_LETTER == 'K'
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 0, 0x3c60808a);
            code_patcher(&mkwfun_lex_cannon_type_patch_addr, 4, 0x38634160);
        #else
            #error "no region"
        #endif

            // reset LEX data
        // [[lex-dev]]
            lex_dev1_data = lex_dev1_data_reset;
            lex_set1_data = lex_set1_data_reset;
            lex_test_data = lex_test_data_reset;
            lex_hipt_len = 0;
            lex_ritp_len = 0;
            *lex_ritp_log = 0;
}

Section-dependant code

Nearly every LEX section has dedicated hooks and code in the LE-CODE.

Dedicated code for the CANN, FEAT and CTDN sections

The FEAT section of a LEX file is a pure metadata section. LE-CODE does not contain any implementation code for this section. This section was requested by MrBean as a kind of signal to CTGP (or other distributions) which particular special features (LEX or otherwise) a track uses. The full spec of this section will eventually be put on Tockdom as well when it's fully finished; the state in progress can be found in the source code of the SZS tools.

The CANN section also has no dedicated code elsewhere, the code for the cannon speeds is already included in the parser above.

The CTDN section is not used at all in the LE-CODE, it's only useful for distributions that support the Countdown race mode. It was requested by Kazuki and you may find a proper implementation at Countdown Mode Beta8 Mod.

Dedicated code for the SET1 section

The SET1 section of a LEX file contains multiple short settings that don't really need their own section. This section will probably be extended in the future, but only in a compatible way.

As for the currently defined parameters, `START-ITEM` is only metadata that can be used by external tools, and `ITEM-POS-FACTOR` and `APPLY-ONLINE-SEC` are implemented in the LE-CODE.

ITEM-POS-FACTOR

In online races, items will only work within the coordinates of around ±131000, because there's only so many bits in the network protocol for the item position. With this setting, a track creator can define the coordinate stretch factors. By setting that to 2, each coordinate will be divided by 2 prior to sending it over the network, and multiplied by 2 after receiving it from the network. This reduces the accuracy of item coordinates, but it makes them work on the whole map.

The code for that is written in Assembly:

.extern lex_set1_data;
.globl itemrange_receive_X_mod

itemrange_receive_X_mod: 
    lis r5, lex_set1_data@ha;
    lfs f1, lex_set1_data@l(r5);	// load factor
    fmuls f0, f0, f1;			// multiply f0 with factor
    stfs f0, 0(r4);			// store new value
    blr; 

.globl itemrange_send_X_mod
itemrange_send_X_mod:
    lis r5, lex_set1_data@ha;
    lfs f1, lex_set1_data@l(r5);	// load factor
    lfs f0, 0(r4);			// load original value to send
    fdivs f0, f0, f1;			// divide by factor
    blr; 

.globl itemrange_receive_Y_mod
itemrange_receive_Y_mod: 
    lis r5, (lex_set1_data+4)@ha;
    lfs f1, (lex_set1_data+4)@l(r5);	// load factor
    fmuls f0, f0, f1;			// multiply f0 with factor
    stfs f0, 0(r4);			// store new value
    blr; 
    
.globl itemrange_send_Y_mod
itemrange_send_Y_mod:
    lis r5, (lex_set1_data+4)@ha;
    lfs f1, (lex_set1_data+4)@l(r5);	// load factor
    lfs f0, 0(r4);			// load original value to send
    fdivs f0, f0, f1;			// divide by factor
    blr; 
    
    
.globl itemrange_receive_Z_mod
itemrange_receive_Z_mod: 
    lis r5, (lex_set1_data+8)@ha;
    lfs f1, (lex_set1_data+8)@l(r5);	// load factor
    fmuls f0, f0, f1;			// multiply f0 with factor
    stfs f0, 0(r4);			// store new value
    blr; 


.globl itemrange_send_Z_mod
itemrange_send_Z_mod:
    lis r5, (lex_set1_data+8)@ha;
    lfs f1, (lex_set1_data+8)@l(r5);	// load factor
    lfs f0, 0(r4);			// load original value to send
    fdivs f0, f0, f1;			// divide by factor
    blr; 

These are the patches necessary to trigger the above ASM code:

    // fix for receiving items:
    // shot_drop is at PAL 0x8079b4ac
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x1ac, &itemrange_receive_X_mod); // X
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x2b0, &itemrange_receive_Y_mod); // Y
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x3b0, &itemrange_receive_Z_mod); // Z

    // tail_destroy is at PAL 0x8079c960
    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x1ac, &itemrange_receive_X_mod); // X
    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x2b0, &itemrange_receive_Y_mod); // Y
    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x3b0, &itemrange_receive_Z_mod); // Z


    // fix for sending items:
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x13c, &itemrange_send_X_mod); // X
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x23c, &itemrange_send_Y_mod); // Y
    code_patcher_BL(&EVENTDATA_item_position_SHOT_DROP, 0x33c, &itemrange_send_Z_mod); // Z

    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x13c, &itemrange_send_X_mod); // X
    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x23c, &itemrange_send_Y_mod); // Y
    code_patcher_BL(&EVENTDATA_item_position_TAIL_DESTROY, 0x33c, &itemrange_send_Z_mod); // Z

APPLY-ONLINE-SEC

The time limit for online races is set at PAL 8053f3b8 - r3 ends up being the time limit in milliseconds. The `set_online_time_limit_by_lex` that's called after parsing the LEX file. It modifes the existing ASM instructions at this address to modify the timeout value to the value set in the LEX file data:

    // Convert to milliseconds and split to hi and lo word
    const unsigned int time = seconds * 1000;
    const unsigned short hi = (time >> 16) & 0xffff;
    const unsigned short lo = (time & 0xffff);

    // Create ASM instructions
    const unsigned int asm_hi = 0x3c600000 | hi; // lis r3, time@h;
    const unsigned int asm_lo = 0x60640000 | lo; // ori r4, r3, time@l;

    // Patch
    code_patcher(&mod_online_time_limit_addr, 0, asm_hi);
    code_patcher(&mod_online_time_limit_addr, 4, asm_lo);

If necessary, the patch will also modify the timer watchdog at PAL 8053f474 (this is set to the time limit plus 30s). This prevents hard disconnections after a timer of 5:56 is reached.

Dedicated code for other sections

The LE-CODE also has dedicated code for the HIPT section (rules on when to hide the position tracker in a race), RITP section (rules to randomize which item routes are taken by Bullet Bills and Red shells) and TEST/DEV1 sections.

Implementations for the HIPT and RITP section in LE-CODE will be added later - these are more deeply integrated with the rest of the code than I had though so that'll take some more work to extract these.

Implementations of the TEST section aren't really useful without support for Extended presence flags, so that code will also be added later.

The DEV1 section has no proper implementation - it's the section type we are using when adding new features (before defining an actual section) or for quick testing. It randomly changes whenever we need it to, and no actual released track should contain such a section.