rt-4d/research/strings_extract.py

import re, sys, json

DUMP = r"C:/Users/vikto/Documents/Claude/rt-4d/radio-spi-dump.bin"
data = open(DUMP, "rb").read()
assert len(data) == 4194304

REGIONS = [
    ("calibration",   0x000000, 0x001000),
    ("main_settings", 0x002000, 0x001000),
    ("channels",      0x004000, 0x00C000),
    ("zones",         0x01C000, 0x020000),
    ("contacts",      0x05C000, 0x010000),
    ("groups",        0x07C000, 0x003000),
    ("dmr_keys",      0x082000, 0x003000),
    ("call_log",      0x088000, 0x00C000),
    ("default_sms",   0x094000, 0x001000),
    ("msg_drafts",    0x095000, 0x010000),
    ("msg_inbox",     0x0A5000, 0x010000),
    ("msg_outbox",    0x0B5000, 0x010000),
    ("schedules",     0x0C6000, 0x008000),
    ("dtmf_names",    0x0C7000, 0x000100),
    ("fm_settings",   0x0D6000, 0x001000),
]
def region_of(addr):
    for name, a, s in REGIONS:
        if a <= addr < a+s:
            return name
    return "UNMAPPED"

# printable set for ascii
def is_print(b):
    return 0x20 <= b <= 0x7E

def looks_real(s):
    """Filter out punctuation-run / repeated-char noise."""
    s2 = s.strip()
    if len(s2) < 3:
        return False
    # must contain at least one letter or digit
    if not re.search(r"[A-Za-z0-9一-鿿]", s2):
        return False
    # count alnum ratio
    alnum = sum(1 for c in s2 if c.isalnum() or ord(c) > 0x2000)
    if alnum / len(s2) < 0.4:
        return False
    # reject if single char repeated (e.g. "7777", "((((")
    uniq = set(s2)
    if len(uniq) <= 2 and len(s2) >= 4:
        # allow things like "10" no; but "AA" reject; require >2 unique for long
        return False
    # reject runs of same punctuation
    if re.fullmatch(r"(.)\1*", s2):
        return False
    return True

results = []  # (addr, encoding, region, text)

# ---- ASCII runs (min 3) ----
run_start = None
for i, b in enumerate(data):
    if is_print(b):
        if run_start is None:
            run_start = i
    else:
        if run_start is not None:
            s = data[run_start:i].decode("latin-1")
            if len(s) >= 3 and looks_real(s):
                results.append((run_start, "ascii", region_of(run_start), s))
            run_start = None
if run_start is not None:
    s = data[run_start:len(data)].decode("latin-1")
    if len(s) >= 3 and looks_real(s):
        results.append((run_start, "ascii", region_of(run_start), s))

# ---- UTF-16LE runs ----
# pattern: printable-ascii byte followed by 0x00, repeated
i = 0
n = len(data)
while i < n-1:
    if is_print(data[i]) and data[i+1] == 0x00:
        j = i
        chars = []
        while j < n-1 and data[j+1] == 0x00 and (is_print(data[j]) or data[j]==0x00):
            if data[j] == 0x00:
                break
            chars.append(chr(data[j]))
            j += 2
        s = "".join(chars)
        if len(s) >= 3 and looks_real(s):
            results.append((i, "utf16le", region_of(i), s))
        i = j + 2 if j > i else i+2
    else:
        i += 1

# ---- GBK / chinese: try decode ascii runs that had high bytes? separate scan ----
# Scan for GBK multibyte sequences (lead 0x81-0xFE, trail 0x40-0xFE)
def gbk_scan():
    out=[]
    i=0
    while i < n:
        b=data[i]
        # start a candidate if ascii-print or gbk lead
        if is_print(b) or (0x81<=b<=0xFE and i+1<n and 0x40<=data[i+1]<=0xFE and data[i+1]!=0x7F):
            start=i
            raw=bytearray()
            while i<n:
                b=data[i]
                if is_print(b):
                    raw.append(b); i+=1
                elif 0x81<=b<=0xFE and i+1<n and 0x40<=data[i+1]<=0xFE and data[i+1]!=0x7F:
                    raw.append(b); raw.append(data[i+1]); i+=2
                else:
                    break
            try:
                s=bytes(raw).decode("gbk")
            except Exception:
                s=None
            if s and re.search(r"[一-鿿]", s):  # only keep if real chinese char present
                if looks_real(s):
                    out.append((start,"gbk",region_of(start),s))
        else:
            i+=1
    return out
results += gbk_scan()

# dedup: prefer utf16/gbk over ascii at same-ish addr
results.sort(key=lambda r:(r[0], r[1]))

# Remove ascii substrings that are actually part of utf16 (every other byte) -- keep both but mark
for addr, enc, reg, text in results:
    pass

# Print
with open(r"C:/Users/vikto/Documents/Claude/rt-4d/analyze/strings_out.txt","w",encoding="utf-8") as f:
    for addr, enc, reg, text in results:
        f.write(f"0x{addr:06X}\t{enc}\t{reg}\t{repr(text)}\n")

print(f"TOTAL candidate real strings: {len(results)}")
from collections import Counter
print("By region:", dict(Counter(r[2] for r in results)))
print("By encoding:", dict(Counter(r[1] for r in results)))