rickub
Sign in Get started
nandi/rustnimpublic⑂ Fork 0
⑂ d4592812fae6b652d94be442f5b3a408d3ab21b6
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/rustnim.git
git clone ssh://git@rickub.com/nandi/rustnim.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

mrules.rs · 460 lines · 18.6 KBRust Blame HistoryRaw
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago1//! A `macro_rules!` expander.
2//!
3//! The alternative was translating each `macro_rules!` into a Nim `template`,
4//! which is the shape-level correspondence: 36% of the 1,833 definitions in a
5//! 400-crate sample are a single rule with no repetition, which is exactly a
6//! template, and another 40% map to a Nim `macro` over `varargs`.
7//!
8//! That correspondence is real but it does not survive contact with this
9//! project's approach. A Nim template body is *untyped*: it is substituted and
10//! only then type-checked. Our lowering is type-directed throughout — it needs
11//! a type to choose `div` over `/`, to size a `cast`, to pick an integer
12//! literal's width. Translating a macro body would mean lowering Rust with no
13//! type information, which is precisely the guessing the project refuses.
14//!
15//! Expanding at the call site does not have that problem: the expansion is
16//! ordinary Rust in a context where types are known, so it lowers like any
17//! other code. Same applicability, faithful output.
18//!
19//! Implemented here: a single rule, no repetition, no `:tt`. That is the 36%.
20
21use proc_macro2::{Delimiter, Group, Ident, TokenStream, TokenTree};
22use std::collections::HashMap;
23
24/// One element of a matcher: a captured fragment, or a token to match exactly.
25#[derive(Debug, Clone)]
26enum Pat {
27 Frag(String),
28 Tok(String),
29 /// A delimited group, matched recursively.
30 Group(Delimiter, Vec<Pat>),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago31 /// `$( .. ) sep? op` — a repetition. `op` is `*`, `+` or `?`.
32 Rep { inner: Vec<Pat>, sep: Option<String>, op: char },
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago33}
34
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago35/// What a fragment captured: once, or once per repetition.
36#[derive(Debug, Clone)]
37enum Cap {
38 One(TokenStream),
39 Seq(Vec<TokenStream>),
40}
41
42type Binds = HashMap<String, Cap>;
43
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago44#[derive(Debug, Clone)]
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago45struct Rule {
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago46 pattern: Vec<Pat>,
47 body: TokenStream,
48}
49
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago50#[derive(Debug, Clone)]
51pub struct MacroDef {
52 /// Rules in source order. Rust tries them top to bottom and the first
53 /// whose matcher matches wins, so order is semantics, not style.
54 rules: Vec<Rule>,
55}
56
57/// Parse `macro_rules!`'s body: a `;`-separated list of
58/// `( $matcher ) => { $transcriber }`.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago59pub fn parse(tokens: TokenStream) -> Result<MacroDef, String> {
60 let t: Vec<TokenTree> = tokens.into_iter().collect();
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago61 let mut rules = Vec::new();
62 let mut i = 0;
63 while i < t.len() {
64 let matcher = match &t[i] {
65 TokenTree::Group(g) => g.clone(),
66 other => return Err(format!("expected a matcher group, found `{other}`")),
67 };
68 let arrow: String = t[i + 1..].iter().take(2).map(|t| t.to_string()).collect();
69 if arrow != "=>" {
70 return Err("expected `=>` after a matcher".into());
71 }
72 let body = match t.get(i + 3) {
73 Some(TokenTree::Group(g)) => g.stream(),
74 _ => return Err("expected a transcriber group".into()),
75 };
76 rules.push(Rule { pattern: parse_pattern(matcher.stream())?, body });
77 i += 4;
78 // The separator is optional after the last rule.
79 if i < t.len() && t[i].to_string() == ";" {
80 i += 1;
81 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago82 }
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago83 if rules.is_empty() {
84 return Err("a macro with no rules".into());
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago85 }
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago86 Ok(MacroDef { rules })
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago87}
88
89fn parse_pattern(ts: TokenStream) -> Result<Vec<Pat>, String> {
90 let mut out = Vec::new();
91 let mut it = ts.into_iter().peekable();
92 while let Some(t) = it.next() {
93 match t {
94 TokenTree::Punct(p) if p.as_char() == '$' => {
95 match it.next() {
96 Some(TokenTree::Ident(name)) => {
97 // `$name:fragment`
98 match it.next() {
99 Some(TokenTree::Punct(c)) if c.as_char() == ':' => {}
100 _ => return Err("expected `:` after a fragment name".into()),
101 }
102 let kind = match it.next() {
103 Some(TokenTree::Ident(k)) => k.to_string(),
104 _ => return Err("expected a fragment specifier".into()),
105 };
106 if kind == "tt" {
107 return Err(
108 "`:tt` makes a macro a token-tree interpreter, which \
109 has no mechanical translation"
110 .into(),
111 );
112 }
113 out.push(Pat::Frag(name.to_string()));
114 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago115 Some(TokenTree::Group(g)) => {
116 let inner = parse_pattern(g.stream())?;
117 if inner.iter().any(|p| matches!(p, Pat::Rep { .. })) {
118 return Err("nested `$(..)` repetition is not implemented yet".into());
119 }
120 // `$( .. ) sep? op`: an optional separator token, then
121 // the operator.
122 let mut sep = None;
123 let op = loop {
124 match it.next() {
125 Some(TokenTree::Punct(p))
126 if matches!(p.as_char(), '*' | '+' | '?') =>
127 {
128 break p.as_char()
129 }
130 Some(t) => {
131 if sep.is_some() {
132 return Err(
133 "a repetition separator must be one token".into()
134 );
135 }
136 sep = Some(t.to_string());
137 }
138 None => return Err("a repetition needs `*`, `+` or `?`".into()),
139 }
140 };
141 out.push(Pat::Rep { inner, sep, op });
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago142 }
143 _ => return Err("unexpected token after `$`".into()),
144 }
145 }
146 TokenTree::Group(g) => {
147 out.push(Pat::Group(g.delimiter(), parse_pattern(g.stream())?))
148 }
149 other => out.push(Pat::Tok(other.to_string())),
150 }
151 }
152 Ok(out)
153}
154
155impl MacroDef {
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago156 /// Try each rule in order and substitute using the first that matches,
157 /// which is what Rust does.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago158 pub fn expand(&self, input: TokenStream) -> Result<TokenStream, String> {
159 let toks: Vec<TokenTree> = input.into_iter().collect();
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago160 let mut why = Vec::new();
161 for (n, rule) in self.rules.iter().enumerate() {
162 let mut binds = Binds::new();
163 match match_seq(&rule.pattern, &toks, &mut binds, None) {
164 Ok(used) if used == toks.len() => {
165 return Ok(substitute(rule.body.clone(), &binds))
166 }
167 Ok(_) => why.push(format!(
168 "rule {}: matched, but left tokens over",
169 n + 1
170 )),
171 Err(e) => why.push(format!("rule {}: {e}", n + 1)),
172 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago173 }
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago174 Err(format!("no rule matched ({})", why.join("; ")))
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago175 }
176}
177
178/// Match `pats` against the front of `toks`, returning how many were consumed.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago179/// `outer_stop` is where a trailing fragment must stop when the pattern
180/// itself does not say: inside `$(, $rest:expr)*` that is the repetition's own
181/// leading comma, which is where the next iteration begins.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago182fn match_seq(
183 pats: &[Pat],
184 toks: &[TokenTree],
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago185 binds: &mut Binds,
186 outer_stop: Option<&str>,
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago187) -> Result<usize, String> {
188 let mut i = 0;
189 for (k, p) in pats.iter().enumerate() {
190 match p {
191 Pat::Tok(s) => {
192 let t = toks.get(i).ok_or("the invocation ends before the matcher does")?;
193 if t.to_string() != *s {
194 return Err(format!("expected `{s}`, found `{t}`"));
195 }
196 i += 1;
197 }
198 Pat::Group(d, inner) => {
199 let Some(TokenTree::Group(g)) = toks.get(i) else {
200 return Err("expected a delimited group".into());
201 };
202 if g.delimiter() != *d {
203 return Err("mismatched delimiter".into());
204 }
205 let sub: Vec<TokenTree> = g.stream().into_iter().collect();
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago206 let n = match_seq(inner, &sub, binds, None)?;
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago207 if n != sub.len() {
208 return Err("group has tokens the matcher does not consume".into());
209 }
210 i += 1;
211 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago212 Pat::Rep { inner, sep, op } => {
213 // Everything after the repetition that is a literal token
214 // marks where it has to stop.
215 let stop = next_literal(&pats[k + 1..]).or_else(|| outer_stop.map(str::to_string));
216 // A trailing fragment inside the repetition stops at the
217 // separator if there is one, or otherwise at whatever starts
218 // the next iteration.
219 let inner_stop = sep.clone().or_else(|| next_literal(inner));
220 let names = frag_names(inner);
221 let mut count = 0usize;
222 for n in &names {
223 binds.insert(n.clone(), Cap::Seq(Vec::new()));
224 }
225 while i < toks.len() {
226 if let Some(stop) = &stop {
227 if toks[i].to_string() == *stop {
228 break;
229 }
230 }
231 let mut one = Binds::new();
232 let used = match_seq(inner, &toks[i..], &mut one, inner_stop.as_deref())?;
233 if used == 0 {
234 break;
235 }
236 for n in &names {
237 let v = match one.remove(n) {
238 Some(Cap::One(ts)) => ts,
239 _ => return Err(format!("`${n}` did not capture in a repetition")),
240 };
241 match binds.get_mut(n) {
242 Some(Cap::Seq(v0)) => v0.push(v),
243 _ => unreachable!(),
244 }
245 }
246 count += 1;
247 i += used;
248 match sep {
249 Some(sp) if i < toks.len() && toks[i].to_string() == *sp => i += 1,
250 Some(_) => break,
251 None => {}
252 }
253 if *op == '?' {
254 break;
255 }
256 }
257 if *op == '+' && count == 0 {
258 return Err("`$(..)+` needs at least one repetition".into());
259 }
260 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago261 Pat::Frag(name) => {
262 // A fragment runs to the next literal token in the matcher, or
263 // to the end. That is what makes `$a:expr, $b:expr` split on
264 // the comma rather than swallowing it.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago265 let stop = next_literal(&pats[k + 1..])
266 .or_else(|| outer_stop.map(str::to_string));
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago267 let start = i;
268 let mut depth = 0i32;
269 while i < toks.len() {
270 let s = toks[i].to_string();
Expand multi-rule and recursive macro_rules d459281 nandithebull 7h ago271 if depth == 0 {
272 if let Some(stop) = &stop {
273 if s == *stop {
274 break;
275 }
276 }
277 // A fragment never spans a top-level `,` or `;`: an
278 // `expr` is one expression, and a comma at this level
279 // separates arguments rather than belonging to one.
280 // A comma inside brackets is within a `Group` token,
281 // so it is not at this level at all.
282 if s == "," || s == ";" {
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago283 break;
284 }
285 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago286 // `<`/`>` are counted so a generic argument list is not
287 // split, but `>` also closes `=>` and `->`, so the depth
288 // never goes negative.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago289 match &toks[i] {
290 TokenTree::Punct(p) if p.as_char() == '<' => depth += 1,
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago291 TokenTree::Punct(p) if p.as_char() == '>' => depth = (depth - 1).max(0),
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago292 _ => {}
293 }
294 i += 1;
295 }
296 if i == start {
297 return Err(format!("nothing matched fragment `${name}`"));
298 }
299 binds.insert(
300 name.clone(),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago301 Cap::One(toks[start..i].iter().cloned().collect::<TokenStream>()),
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago302 );
303 }
304 }
305 }
306 Ok(i)
307}
308
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago309/// The first literal token that can follow, looking through a repetition:
310/// after `$first:expr` in `($first:expr $(, $rest:expr)*)` the next literal is
311/// the repetition's leading comma, which is where `$first` has to stop.
312fn next_literal(pats: &[Pat]) -> Option<String> {
313 for p in pats {
314 match p {
315 Pat::Tok(s) => return Some(s.clone()),
316 Pat::Rep { inner, sep, .. } => {
317 if let Some(s) = next_literal(inner) {
318 return Some(s);
319 }
320 if let Some(s) = sep {
321 return Some(s.clone());
322 }
323 }
324 Pat::Group(..) | Pat::Frag(_) => return None,
325 }
326 }
327 None
328}
329
330/// The fragment names a pattern captures.
331fn frag_names(pats: &[Pat]) -> Vec<String> {
332 let mut out = Vec::new();
333 for p in pats {
334 match p {
335 Pat::Frag(n) => out.push(n.clone()),
336 Pat::Group(_, inner) => out.extend(frag_names(inner)),
337 Pat::Rep { inner, .. } => out.extend(frag_names(inner)),
338 Pat::Tok(_) => {}
339 }
340 }
341 out
342}
343
344/// Replace every `$name` in the body with what it captured, and expand every
345/// `$( .. ) sep? op` once per repetition.
346fn substitute(ts: TokenStream, binds: &Binds) -> TokenStream {
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago347 let mut out = Vec::new();
348 let mut it = ts.into_iter().peekable();
349 while let Some(t) = it.next() {
350 match t {
351 TokenTree::Punct(p) if p.as_char() == '$' => {
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago352 // `$( .. ) sep? op` in the transcriber.
353 if let Some(TokenTree::Group(g)) = it.peek() {
354 let g = g.clone();
355 it.next();
356 let mut sep = None;
357 loop {
358 match it.peek() {
359 Some(TokenTree::Punct(p))
360 if matches!(p.as_char(), '*' | '+' | '?') =>
361 {
362 it.next();
363 break;
364 }
365 Some(t) => {
366 sep = Some(t.clone());
367 it.next();
368 }
369 None => break,
370 }
371 }
372 let names: Vec<String> = fragments_in(g.stream())
373 .into_iter()
374 .filter(|n| matches!(binds.get(n), Some(Cap::Seq(_))))
375 .collect();
376 let n = names
377 .iter()
378 .filter_map(|n| match binds.get(n) {
379 Some(Cap::Seq(v)) => Some(v.len()),
380 _ => None,
381 })
382 .max()
383 .unwrap_or(0);
384 for idx in 0..n {
385 let mut one: Binds = binds.clone();
386 for nm in &names {
387 if let Some(Cap::Seq(v)) = binds.get(nm) {
388 if let Some(x) = v.get(idx) {
389 one.insert(nm.clone(), Cap::One(x.clone()));
390 }
391 }
392 }
393 if idx > 0 {
394 if let Some(s) = &sep {
395 out.push(s.clone());
396 }
397 }
398 out.extend(substitute(g.stream(), &one));
399 }
400 continue;
401 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago402 match it.peek() {
403 Some(TokenTree::Ident(name)) => {
404 let n = name.to_string();
405 it.next();
406 match binds.get(&n) {
407 // Parenthesised so that a captured expression keeps
408 // its own precedence, as Rust's `expr` fragments do.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago409 Some(Cap::One(v)) => out.push(TokenTree::Group(Group::new(
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago410 Delimiter::Parenthesis,
411 v.clone(),
412 ))),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago413 Some(Cap::Seq(_)) => {
414 // Used outside a `$(..)`; Rust rejects this too.
415 out.push(TokenTree::Punct(p));
416 out.push(TokenTree::Ident(Ident::new(
417 &n,
418 proc_macro2::Span::call_site(),
419 )));
420 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago421 None => {
422 out.push(TokenTree::Punct(p));
423 out.push(TokenTree::Ident(Ident::new(
424 &n,
425 proc_macro2::Span::call_site(),
426 )));
427 }
428 }
429 }
430 _ => out.push(TokenTree::Punct(p)),
431 }
432 }
433 TokenTree::Group(g) => {
434 let inner = substitute(g.stream(), binds);
435 out.push(TokenTree::Group(Group::new(g.delimiter(), inner)));
436 }
437 other => out.push(other),
438 }
439 }
440 out.into_iter().collect()
441}
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago442
443/// The `$name` references appearing in a token stream.
444fn fragments_in(ts: TokenStream) -> Vec<String> {
445 let mut out = Vec::new();
446 let mut it = ts.into_iter().peekable();
447 while let Some(t) = it.next() {
448 match t {
449 TokenTree::Punct(p) if p.as_char() == '$' => {
450 if let Some(TokenTree::Ident(n)) = it.peek() {
451 out.push(n.to_string());
452 it.next();
453 }
454 }
455 TokenTree::Group(g) => out.extend(fragments_in(g.stream())),
456 _ => {}
457 }
458 }
459 out
460}