Memory leak updating CIMultiDict from mapping (present in far back as 5.2.0, likely introduced in 4.4.0) fixed in 6.4.0

[bdraco]

Leak 2 (old leak)

This version seems to leak as far back as 5.2.0, maybe earlier

leak happens when updating a CIMultiDict from a dict or CIMultiDict

EDIT: I have not found a version that doesn't leak that I can still compile

import os
import gc
import psutil
import ctypes
from multidict import CIMultiDict

def trim_memory() -> int:
    gc.collect()
    # libc = ctypes.CDLL("libc.so.6")
    # return libc.malloc_trim(0)


def get_memory_usage():
    process = psutil.Process(os.getpid())
    memory_info = process.memory_info()
    return memory_info.rss / (1024 * 1024)


def main():
    for _ in range(10_000):
        for _ in range(1000):
            result = CIMultiDict()
            headers = {f"X-Any-{i}": str(i) for i in range(1000)}
            result.update(headers)
        del result
        del headers
        trim_memory()
        print(f"Memory usage: {get_memory_usage():.2f} MB")


if __name__ == "__main__":
    main()

% python3.9 leak2.py                                 
multidict version: 6.3.2
Memory usage: 37.47 MB
Memory usage: 61.06 MB
Memory usage: 84.12 MB

% python3.9 leak2.py                                 
multidict version: 6.0.0
Memory usage: 36.91 MB
Memory usage: 60.98 MB
Memory usage: 84.89 MB
Memory usage: 108.28 MB
Memory usage: 131.38 MB

Originally posted by @bdraco in #1131

[bdraco]

bisecting might takes few hours due to having to go back to v4.0.0

test_leak2.sh

#!/bin/bash
make clean
pip3.11 install --upgrade .
python3.11 -I leak2.py

leak2.py

import os
import gc
import psutil
import ctypes
from multidict import CIMultiDict, MultiDict

from multidict import __version__ as multidict_version

print(f"multidict version: {multidict_version}")


def trim_memory() -> int:
    gc.collect()
    # libc = ctypes.CDLL("libc.so.6")
    # return libc.malloc_trim(0)


def get_memory_usage():
    process = psutil.Process(os.getpid())
    memory_info = process.memory_info()
    return memory_info.rss / (1024 * 1024)


headers = {f"X-Any-{i}": str(i) for i in range(1000)}


def main():
    for _ in range(10):
        for _ in range(1000):
            result = MultiDict()

            result.update(headers)
        del result
        usage = get_memory_usage()
        print(f"Memory usage: {usage:.2f} MB")
        if usage > 100:
            raise MemoryError("Memory usage exceeded 100 MB")


if __name__ == "__main__":
    main()

[bdraco]

I couldn't get versions earlier than 5.2.0 to build so thats as far back as I could test

[bdraco]

extend is not affected, only update

[bdraco]

leak seems to happen in pair_list_update_from_dict -> _pair_list_update

likely in

    if (!found) {
        if (_pair_list_add_with_hash(list, identity, key, value, hash) < 0) {
            return -1;
        }
        if (_dict_set_number(used, identity, list->size) < 0) {
            return -1;
        }
    }