rickub
Sign in Get started
nandi/rustnimpublic⑂ Fork 0
⑂ 3d4a7d283b83d3a5305dfc6654c290bd4ee162ae
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/rustnim.git
git clone ssh://git@rickub.com/nandi/rustnim.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

mrules.rs · 423 lines · 17.3 KBRust Blame HistoryRaw
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago1//! A `macro_rules!` expander.
2//!
3//! The alternative was translating each `macro_rules!` into a Nim `template`,
4//! which is the shape-level correspondence: 36% of the 1,833 definitions in a
5//! 400-crate sample are a single rule with no repetition, which is exactly a
6//! template, and another 40% map to a Nim `macro` over `varargs`.
7//!
8//! That correspondence is real but it does not survive contact with this
9//! project's approach. A Nim template body is *untyped*: it is substituted and
10//! only then type-checked. Our lowering is type-directed throughout — it needs
11//! a type to choose `div` over `/`, to size a `cast`, to pick an integer
12//! literal's width. Translating a macro body would mean lowering Rust with no
13//! type information, which is precisely the guessing the project refuses.
14//!
15//! Expanding at the call site does not have that problem: the expansion is
16//! ordinary Rust in a context where types are known, so it lowers like any
17//! other code. Same applicability, faithful output.
18//!
19//! Implemented here: a single rule, no repetition, no `:tt`. That is the 36%.
20
21use proc_macro2::{Delimiter, Group, Ident, TokenStream, TokenTree};
22use std::collections::HashMap;
23
24/// One element of a matcher: a captured fragment, or a token to match exactly.
25#[derive(Debug, Clone)]
26enum Pat {
27 Frag(String),
28 Tok(String),
29 /// A delimited group, matched recursively.
30 Group(Delimiter, Vec<Pat>),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago31 /// `$( .. ) sep? op` — a repetition. `op` is `*`, `+` or `?`.
32 Rep { inner: Vec<Pat>, sep: Option<String>, op: char },
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago33}
34
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago35/// What a fragment captured: once, or once per repetition.
36#[derive(Debug, Clone)]
37enum Cap {
38 One(TokenStream),
39 Seq(Vec<TokenStream>),
40}
41
42type Binds = HashMap<String, Cap>;
43
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago44#[derive(Debug, Clone)]
45pub struct MacroDef {
46 pattern: Vec<Pat>,
47 body: TokenStream,
48}
49
50/// Parse `macro_rules!`'s body: `( $matcher ) => { $transcriber };`.
51pub fn parse(tokens: TokenStream) -> Result<MacroDef, String> {
52 let t: Vec<TokenTree> = tokens.into_iter().collect();
53 // matcher group, `=`, `>`, transcriber group, optional `;`
54 let (matcher, rest) = match t.split_first() {
55 Some((TokenTree::Group(g), rest)) => (g.clone(), rest),
56 _ => return Err("expected a matcher group".into()),
57 };
58 let arrow: String = rest.iter().take(2).map(|t| t.to_string()).collect();
59 if arrow != "=>" {
60 return Err("expected `=>`".into());
61 }
62 let body = match rest.get(2) {
63 Some(TokenTree::Group(g)) => g.stream(),
64 _ => return Err("expected a transcriber group".into()),
65 };
66 if rest.len() > 4 || (rest.len() == 4 && rest[3].to_string() != ";") {
67 return Err("more than one rule is not implemented yet".into());
68 }
69 Ok(MacroDef { pattern: parse_pattern(matcher.stream())?, body })
70}
71
72fn parse_pattern(ts: TokenStream) -> Result<Vec<Pat>, String> {
73 let mut out = Vec::new();
74 let mut it = ts.into_iter().peekable();
75 while let Some(t) = it.next() {
76 match t {
77 TokenTree::Punct(p) if p.as_char() == '$' => {
78 match it.next() {
79 Some(TokenTree::Ident(name)) => {
80 // `$name:fragment`
81 match it.next() {
82 Some(TokenTree::Punct(c)) if c.as_char() == ':' => {}
83 _ => return Err("expected `:` after a fragment name".into()),
84 }
85 let kind = match it.next() {
86 Some(TokenTree::Ident(k)) => k.to_string(),
87 _ => return Err("expected a fragment specifier".into()),
88 };
89 if kind == "tt" {
90 return Err(
91 "`:tt` makes a macro a token-tree interpreter, which \
92 has no mechanical translation"
93 .into(),
94 );
95 }
96 out.push(Pat::Frag(name.to_string()));
97 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago98 Some(TokenTree::Group(g)) => {
99 let inner = parse_pattern(g.stream())?;
100 if inner.iter().any(|p| matches!(p, Pat::Rep { .. })) {
101 return Err("nested `$(..)` repetition is not implemented yet".into());
102 }
103 // `$( .. ) sep? op`: an optional separator token, then
104 // the operator.
105 let mut sep = None;
106 let op = loop {
107 match it.next() {
108 Some(TokenTree::Punct(p))
109 if matches!(p.as_char(), '*' | '+' | '?') =>
110 {
111 break p.as_char()
112 }
113 Some(t) => {
114 if sep.is_some() {
115 return Err(
116 "a repetition separator must be one token".into()
117 );
118 }
119 sep = Some(t.to_string());
120 }
121 None => return Err("a repetition needs `*`, `+` or `?`".into()),
122 }
123 };
124 out.push(Pat::Rep { inner, sep, op });
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago125 }
126 _ => return Err("unexpected token after `$`".into()),
127 }
128 }
129 TokenTree::Group(g) => {
130 out.push(Pat::Group(g.delimiter(), parse_pattern(g.stream())?))
131 }
132 other => out.push(Pat::Tok(other.to_string())),
133 }
134 }
135 Ok(out)
136}
137
138impl MacroDef {
139 /// Match an invocation's tokens and substitute them into the body.
140 pub fn expand(&self, input: TokenStream) -> Result<TokenStream, String> {
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago141 let mut binds = Binds::new();
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago142 let toks: Vec<TokenTree> = input.into_iter().collect();
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago143 let used = match_seq(&self.pattern, &toks, &mut binds, None)?;
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago144 if used != toks.len() {
145 return Err("the invocation has tokens the matcher does not consume".into());
146 }
147 Ok(substitute(self.body.clone(), &binds))
148 }
149}
150
151/// Match `pats` against the front of `toks`, returning how many were consumed.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago152/// `outer_stop` is where a trailing fragment must stop when the pattern
153/// itself does not say: inside `$(, $rest:expr)*` that is the repetition's own
154/// leading comma, which is where the next iteration begins.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago155fn match_seq(
156 pats: &[Pat],
157 toks: &[TokenTree],
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago158 binds: &mut Binds,
159 outer_stop: Option<&str>,
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago160) -> Result<usize, String> {
161 let mut i = 0;
162 for (k, p) in pats.iter().enumerate() {
163 match p {
164 Pat::Tok(s) => {
165 let t = toks.get(i).ok_or("the invocation ends before the matcher does")?;
166 if t.to_string() != *s {
167 return Err(format!("expected `{s}`, found `{t}`"));
168 }
169 i += 1;
170 }
171 Pat::Group(d, inner) => {
172 let Some(TokenTree::Group(g)) = toks.get(i) else {
173 return Err("expected a delimited group".into());
174 };
175 if g.delimiter() != *d {
176 return Err("mismatched delimiter".into());
177 }
178 let sub: Vec<TokenTree> = g.stream().into_iter().collect();
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago179 let n = match_seq(inner, &sub, binds, None)?;
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago180 if n != sub.len() {
181 return Err("group has tokens the matcher does not consume".into());
182 }
183 i += 1;
184 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago185 Pat::Rep { inner, sep, op } => {
186 // Everything after the repetition that is a literal token
187 // marks where it has to stop.
188 let stop = next_literal(&pats[k + 1..]).or_else(|| outer_stop.map(str::to_string));
189 // A trailing fragment inside the repetition stops at the
190 // separator if there is one, or otherwise at whatever starts
191 // the next iteration.
192 let inner_stop = sep.clone().or_else(|| next_literal(inner));
193 let names = frag_names(inner);
194 let mut count = 0usize;
195 for n in &names {
196 binds.insert(n.clone(), Cap::Seq(Vec::new()));
197 }
198 while i < toks.len() {
199 if let Some(stop) = &stop {
200 if toks[i].to_string() == *stop {
201 break;
202 }
203 }
204 let mut one = Binds::new();
205 let used = match_seq(inner, &toks[i..], &mut one, inner_stop.as_deref())?;
206 if used == 0 {
207 break;
208 }
209 for n in &names {
210 let v = match one.remove(n) {
211 Some(Cap::One(ts)) => ts,
212 _ => return Err(format!("`${n}` did not capture in a repetition")),
213 };
214 match binds.get_mut(n) {
215 Some(Cap::Seq(v0)) => v0.push(v),
216 _ => unreachable!(),
217 }
218 }
219 count += 1;
220 i += used;
221 match sep {
222 Some(sp) if i < toks.len() && toks[i].to_string() == *sp => i += 1,
223 Some(_) => break,
224 None => {}
225 }
226 if *op == '?' {
227 break;
228 }
229 }
230 if *op == '+' && count == 0 {
231 return Err("`$(..)+` needs at least one repetition".into());
232 }
233 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago234 Pat::Frag(name) => {
235 // A fragment runs to the next literal token in the matcher, or
236 // to the end. That is what makes `$a:expr, $b:expr` split on
237 // the comma rather than swallowing it.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago238 let stop = next_literal(&pats[k + 1..])
239 .or_else(|| outer_stop.map(str::to_string));
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago240 let start = i;
241 let mut depth = 0i32;
242 while i < toks.len() {
243 let s = toks[i].to_string();
244 if let Some(stop) = &stop {
245 if depth == 0 && s == *stop {
246 break;
247 }
248 }
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago249 // `<`/`>` are counted so a generic argument list is not
250 // split, but `>` also closes `=>` and `->`, so the depth
251 // never goes negative.
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago252 match &toks[i] {
253 TokenTree::Punct(p) if p.as_char() == '<' => depth += 1,
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago254 TokenTree::Punct(p) if p.as_char() == '>' => depth = (depth - 1).max(0),
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago255 _ => {}
256 }
257 i += 1;
258 }
259 if i == start {
260 return Err(format!("nothing matched fragment `${name}`"));
261 }
262 binds.insert(
263 name.clone(),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago264 Cap::One(toks[start..i].iter().cloned().collect::<TokenStream>()),
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago265 );
266 }
267 }
268 }
269 Ok(i)
270}
271
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago272/// The first literal token that can follow, looking through a repetition:
273/// after `$first:expr` in `($first:expr $(, $rest:expr)*)` the next literal is
274/// the repetition's leading comma, which is where `$first` has to stop.
275fn next_literal(pats: &[Pat]) -> Option<String> {
276 for p in pats {
277 match p {
278 Pat::Tok(s) => return Some(s.clone()),
279 Pat::Rep { inner, sep, .. } => {
280 if let Some(s) = next_literal(inner) {
281 return Some(s);
282 }
283 if let Some(s) = sep {
284 return Some(s.clone());
285 }
286 }
287 Pat::Group(..) | Pat::Frag(_) => return None,
288 }
289 }
290 None
291}
292
293/// The fragment names a pattern captures.
294fn frag_names(pats: &[Pat]) -> Vec<String> {
295 let mut out = Vec::new();
296 for p in pats {
297 match p {
298 Pat::Frag(n) => out.push(n.clone()),
299 Pat::Group(_, inner) => out.extend(frag_names(inner)),
300 Pat::Rep { inner, .. } => out.extend(frag_names(inner)),
301 Pat::Tok(_) => {}
302 }
303 }
304 out
305}
306
307/// Replace every `$name` in the body with what it captured, and expand every
308/// `$( .. ) sep? op` once per repetition.
309fn substitute(ts: TokenStream, binds: &Binds) -> TokenStream {
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago310 let mut out = Vec::new();
311 let mut it = ts.into_iter().peekable();
312 while let Some(t) = it.next() {
313 match t {
314 TokenTree::Punct(p) if p.as_char() == '$' => {
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago315 // `$( .. ) sep? op` in the transcriber.
316 if let Some(TokenTree::Group(g)) = it.peek() {
317 let g = g.clone();
318 it.next();
319 let mut sep = None;
320 loop {
321 match it.peek() {
322 Some(TokenTree::Punct(p))
323 if matches!(p.as_char(), '*' | '+' | '?') =>
324 {
325 it.next();
326 break;
327 }
328 Some(t) => {
329 sep = Some(t.clone());
330 it.next();
331 }
332 None => break,
333 }
334 }
335 let names: Vec<String> = fragments_in(g.stream())
336 .into_iter()
337 .filter(|n| matches!(binds.get(n), Some(Cap::Seq(_))))
338 .collect();
339 let n = names
340 .iter()
341 .filter_map(|n| match binds.get(n) {
342 Some(Cap::Seq(v)) => Some(v.len()),
343 _ => None,
344 })
345 .max()
346 .unwrap_or(0);
347 for idx in 0..n {
348 let mut one: Binds = binds.clone();
349 for nm in &names {
350 if let Some(Cap::Seq(v)) = binds.get(nm) {
351 if let Some(x) = v.get(idx) {
352 one.insert(nm.clone(), Cap::One(x.clone()));
353 }
354 }
355 }
356 if idx > 0 {
357 if let Some(s) = &sep {
358 out.push(s.clone());
359 }
360 }
361 out.extend(substitute(g.stream(), &one));
362 }
363 continue;
364 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago365 match it.peek() {
366 Some(TokenTree::Ident(name)) => {
367 let n = name.to_string();
368 it.next();
369 match binds.get(&n) {
370 // Parenthesised so that a captured expression keeps
371 // its own precedence, as Rust's `expr` fragments do.
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago372 Some(Cap::One(v)) => out.push(TokenTree::Group(Group::new(
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago373 Delimiter::Parenthesis,
374 v.clone(),
375 ))),
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago376 Some(Cap::Seq(_)) => {
377 // Used outside a `$(..)`; Rust rejects this too.
378 out.push(TokenTree::Punct(p));
379 out.push(TokenTree::Ident(Ident::new(
380 &n,
381 proc_macro2::Span::call_site(),
382 )));
383 }
Expand `macro_rules!` rather than translating it to a Nim template 04eb29f nandithebull 7h ago384 None => {
385 out.push(TokenTree::Punct(p));
386 out.push(TokenTree::Ident(Ident::new(
387 &n,
388 proc_macro2::Span::call_site(),
389 )));
390 }
391 }
392 }
393 _ => out.push(TokenTree::Punct(p)),
394 }
395 }
396 TokenTree::Group(g) => {
397 let inner = substitute(g.stream(), binds);
398 out.push(TokenTree::Group(Group::new(g.delimiter(), inner)));
399 }
400 other => out.push(other),
401 }
402 }
403 out.into_iter().collect()
404}
Expand `$(..)` repetition in macro_rules 3d4a7d2 nandithebull 7h ago405
406/// The `$name` references appearing in a token stream.
407fn fragments_in(ts: TokenStream) -> Vec<String> {
408 let mut out = Vec::new();
409 let mut it = ts.into_iter().peekable();
410 while let Some(t) = it.next() {
411 match t {
412 TokenTree::Punct(p) if p.as_char() == '$' => {
413 if let Some(TokenTree::Ident(n)) = it.peek() {
414 out.push(n.to_string());
415 it.next();
416 }
417 }
418 TokenTree::Group(g) => out.extend(fragments_in(g.stream())),
419 _ => {}
420 }
421 }
422 out
423}