@inproceedings{zhang-etal-2026-rule,
title = "{RULE}-{EDIT}: Benchmarking Rule-Level Knowledge Editing in Large Language Models",
author = "Zhang, Jizhi and
Hou, Dylan Xinming and
Li, Zhaoyi and
Yang, Qishun and
Ihsani, Mahardika Krisna and
Ali, Muhammad Asif and
Xia, Gus and
Hu, Lijie",
editor = "Lee, Hung-Yi and
Dev, Sunipa and
Li, Jessy and
Elliott, Desmond",
booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2026",
month = oct,
year = "2026",
address = "Budapest, Hungary",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2026.findings-emnlp.1924/",
pages = "41008--41027",
ISBN = "979-8-89176-462-0",
abstract = "Knowledge editing seeks to update language models without full retraining. While most prior work focuses on isolated factual or instance-level edits, we study a more structured setting: mathematical rules as a controlled stress test for abstract symbolic rule editing. We introduce RULE-EDIT, the first benchmark explicitly designed for editing and evaluating rule-level knowledge in LLMs. Our benchmark investigates two key questions: (1) whether edits to abstract rules propagate to derived instances, and (2) whether edits remain consistent across different representations of the same rule. To evaluate these aspects, we propose two new metrics: Instance Portability and Rule Understanding. Experiments on strong open-source LLMs with representative editing methods show that, although models can often overwrite formula-level knowledge, they struggle to transfer edits to rule-derived instances and to maintain consistency across symbolic and natural-language forms. For example, several methods achieve nearly 100{\%} reliability on direct rule queries, while Instance Portability never exceeds 52{\%} and Rule Understanding remains below 26{\%}. Overall, RULE-EDIT provides a controlled benchmark for studying whether current knowledge editing approaches truly modify abstract symbolic rules in LLMs, rather than merely altering surface outputs."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="zhang-etal-2026-rule">
<titleInfo>
<title>RULE-EDIT: Benchmarking Rule-Level Knowledge Editing in Large Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Jizhi</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Dylan</namePart>
<namePart type="given">Xinming</namePart>
<namePart type="family">Hou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Zhaoyi</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Qishun</namePart>
<namePart type="family">Yang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mahardika</namePart>
<namePart type="given">Krisna</namePart>
<namePart type="family">Ihsani</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Muhammad</namePart>
<namePart type="given">Asif</namePart>
<namePart type="family">Ali</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Gus</namePart>
<namePart type="family">Xia</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lijie</namePart>
<namePart type="family">Hu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2026-10</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EMNLP 2026</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hung-Yi</namePart>
<namePart type="family">Lee</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sunipa</namePart>
<namePart type="family">Dev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jessy</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Desmond</namePart>
<namePart type="family">Elliott</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Budapest, Hungary</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
<identifier type="isbn">979-8-89176-462-0</identifier>
</relatedItem>
<abstract>Knowledge editing seeks to update language models without full retraining. While most prior work focuses on isolated factual or instance-level edits, we study a more structured setting: mathematical rules as a controlled stress test for abstract symbolic rule editing. We introduce RULE-EDIT, the first benchmark explicitly designed for editing and evaluating rule-level knowledge in LLMs. Our benchmark investigates two key questions: (1) whether edits to abstract rules propagate to derived instances, and (2) whether edits remain consistent across different representations of the same rule. To evaluate these aspects, we propose two new metrics: Instance Portability and Rule Understanding. Experiments on strong open-source LLMs with representative editing methods show that, although models can often overwrite formula-level knowledge, they struggle to transfer edits to rule-derived instances and to maintain consistency across symbolic and natural-language forms. For example, several methods achieve nearly 100% reliability on direct rule queries, while Instance Portability never exceeds 52% and Rule Understanding remains below 26%. Overall, RULE-EDIT provides a controlled benchmark for studying whether current knowledge editing approaches truly modify abstract symbolic rules in LLMs, rather than merely altering surface outputs.</abstract>
<identifier type="citekey">zhang-etal-2026-rule</identifier>
<location>
<url>https://aclanthology.org/2026.findings-emnlp.1924/</url>
</location>
<part>
<date>2026-10</date>
<extent unit="page">
<start>41008</start>
<end>41027</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T RULE-EDIT: Benchmarking Rule-Level Knowledge Editing in Large Language Models
%A Zhang, Jizhi
%A Hou, Dylan Xinming
%A Li, Zhaoyi
%A Yang, Qishun
%A Ihsani, Mahardika Krisna
%A Ali, Muhammad Asif
%A Xia, Gus
%A Hu, Lijie
%Y Lee, Hung-Yi
%Y Dev, Sunipa
%Y Li, Jessy
%Y Elliott, Desmond
%S Findings of the Association for Computational Linguistics: EMNLP 2026
%D 2026
%8 October
%I Association for Computational Linguistics
%C Budapest, Hungary
%@ 979-8-89176-462-0
%F zhang-etal-2026-rule
%X Knowledge editing seeks to update language models without full retraining. While most prior work focuses on isolated factual or instance-level edits, we study a more structured setting: mathematical rules as a controlled stress test for abstract symbolic rule editing. We introduce RULE-EDIT, the first benchmark explicitly designed for editing and evaluating rule-level knowledge in LLMs. Our benchmark investigates two key questions: (1) whether edits to abstract rules propagate to derived instances, and (2) whether edits remain consistent across different representations of the same rule. To evaluate these aspects, we propose two new metrics: Instance Portability and Rule Understanding. Experiments on strong open-source LLMs with representative editing methods show that, although models can often overwrite formula-level knowledge, they struggle to transfer edits to rule-derived instances and to maintain consistency across symbolic and natural-language forms. For example, several methods achieve nearly 100% reliability on direct rule queries, while Instance Portability never exceeds 52% and Rule Understanding remains below 26%. Overall, RULE-EDIT provides a controlled benchmark for studying whether current knowledge editing approaches truly modify abstract symbolic rules in LLMs, rather than merely altering surface outputs.
%U https://aclanthology.org/2026.findings-emnlp.1924/
%P 41008-41027
Markdown (Informal)
[RULE-EDIT: Benchmarking Rule-Level Knowledge Editing in Large Language Models](https://aclanthology.org/2026.findings-emnlp.1924/) (Zhang et al., Findings 2026)
ACL
- Jizhi Zhang, Dylan Xinming Hou, Zhaoyi Li, Qishun Yang, Mahardika Krisna Ihsani, Muhammad Asif Ali, Gus Xia, and Lijie Hu. 2026. RULE-EDIT: Benchmarking Rule-Level Knowledge Editing in Large Language Models. In Findings of the Association for Computational Linguistics: EMNLP 2026, pages 41008–41027, Budapest, Hungary. Association for Computational Linguistics.