diff --git a/rust/candid_parser/src/grammar.lalrpop b/rust/candid_parser/src/grammar.lalrpop index d1579e0b..11c3ad8d 100644 --- a/rust/candid_parser/src/grammar.lalrpop +++ b/rust/candid_parser/src/grammar.lalrpop @@ -47,6 +47,7 @@ extern { ":" => Token::Colon, "->" => Token::Arrow, "line_comment" => Token::LineComment(>), + "block_comment" => Token::BlockComment(>), "\n" => Token::Newline, } } @@ -314,8 +315,11 @@ Name: String = { Text => <>, } + +// Prioritize the last comment Comment: Comment = { - "line_comment" => Comment { lines: <> }, + => Comment { lines: bc.unwrap_or(lc) }, + => Comment { lines: lc.unwrap_or(bc) }, } // Also allows trailing separator diff --git a/rust/candid_parser/src/token.rs b/rust/candid_parser/src/token.rs index dc655081..9954d0c0 100644 --- a/rust/candid_parser/src/token.rs +++ b/rust/candid_parser/src/token.rs @@ -4,12 +4,16 @@ use logos::{Lexer, Logos}; #[derive(Logos, Debug, Clone, PartialEq, Eq, Ord, PartialOrd)] #[logos(skip r"[ \t\r]+")] #[logos(skip r"([ \t]*//[^\n]*\n)+\n")] // ignore line comments that are followed by an empty line +#[logos(skip r"[ \t]*/\*([^(\*/)]*)\*/\n\n")] // ignore block comments that are followed by an empty line pub enum Token { #[token("/*")] StartComment, // catch line comments at any indentation level, thanks to the `[ \t]*` prefix #[regex(r"([ \t]*//[^\n]*\n)+", parse_comment_lines)] LineComment(Vec), + // catch block comments at any indentation level, thanks to the `[ \t]*` prefix + #[regex(r"[ \t]*/\*([^(\*/)]*)\*/\n", parse_block_comment_lines)] + BlockComment(Vec), #[token("=")] Equals, #[token("(")] @@ -131,6 +135,20 @@ fn parse_comment_lines(lex: &mut Lexer) -> Vec { .collect() } +fn parse_block_comment_lines(lex: &mut Lexer) -> Vec { + lex.slice() + .lines() + .map(|s| { + s.trim() + .trim_start_matches("/*") + .trim_end_matches("*/") + .trim() + .to_string() + }) + .filter(|s| !s.is_empty()) + .collect() +} + pub struct Tokenizer<'input> { lex: Lexer<'input, Token>, } @@ -217,31 +235,21 @@ impl Iterator for Tokenizer<'_> { self.next() } Ok(Token::LineComment(mut lines)) => { - let span = self.lex.span(); - let source = self.lex.source(); - - // Check the char before the span: if it's NOT a newline, it means that - // the comment is at the end of a line and therefore it must be ignored. - // If it's at the start of the source (prev_char_index == 0), we don't have to check anything. - let prev_char_index = span.start.saturating_sub(1); - if prev_char_index > 0 { - let is_end_of_line_comment = source - .chars() - .nth(prev_char_index) - .map(|c| c != '\n') - .unwrap_or(false); - - if is_end_of_line_comment { - lines.remove(0); - } - } - + let span = skip_isolated_comments(&self.lex, &mut lines); // Ignore the comment if it's empty if lines.is_empty() { return self.next(); } Some(Ok((span.start, Token::LineComment(lines), span.end))) } + Ok(Token::BlockComment(mut lines)) => { + let span = skip_isolated_comments(&self.lex, &mut lines); + // Ignore the comment if it's empty + if lines.is_empty() { + return self.next(); + } + Some(Ok((span.start, Token::BlockComment(lines), span.end))) + } Ok(Token::StartString) => { let mut result = String::new(); let mut lex = self.lex.to_owned().morph::(); @@ -322,3 +330,26 @@ impl Iterator for Tokenizer<'_> { } } } + +fn skip_isolated_comments(lex: &Lexer, comment_lines: &mut Vec) -> Span { + let span = lex.span(); + let source = lex.source(); + + // Check the char before the span: if it's NOT a newline or space, it means that + // the comment is at the end of a line and therefore it must be ignored. + // If it's at the start of the source (prev_char_index == 0), we don't have to check anything. + let prev_char_index = span.start.saturating_sub(1); + if prev_char_index > 0 { + let is_end_of_line_comment = source + .chars() + .nth(prev_char_index) + .map(|c| c != '\n' && c != ' ') + .unwrap_or(false); + + if is_end_of_line_comment { + comment_lines.remove(0); + } + } + + span +} diff --git a/rust/candid_parser/tests/assets/actor.did b/rust/candid_parser/tests/assets/actor.did index 8d392d72..b1a21362 100644 --- a/rust/candid_parser/tests/assets/actor.did +++ b/rust/candid_parser/tests/assets/actor.did @@ -2,7 +2,9 @@ type f = func (int8) -> (int8); type g = f; type h = func (f) -> (f); type o = opt o; -// This is a service comment +/* +This is a service block comment +*/ service : { f : (nat) -> (h); diff --git a/rust/candid_parser/tests/assets/example.did b/rust/candid_parser/tests/assets/example.did index 028c03dd..dc2b4406 100644 --- a/rust/candid_parser/tests/assets/example.did +++ b/rust/candid_parser/tests/assets/example.did @@ -9,7 +9,13 @@ type my_type = principal; // This is an ignored comment +/* +This is a type block comment +*/ type List = opt record { + /* + This is a field block comment + */ head: int; // This is a field comment tail: List; @@ -17,11 +23,19 @@ type List = opt record { type f = func (List, func (int32) -> (int64)) -> (opt List, res); // This comment is ignored // This is another type comment type broker = service { + /* This is a service method block comment */ find : (name: text) -> (service {up:() -> (); current:() -> (nat32)}); // This comment is ignored }; type nested = record { nat; nat; record {nat;int;}; record { nat; 0x2a:nat; nat8; }; 42:nat; 40:nat; variant{ A; 0x2a; B; C }; }; -type res = variant { Ok: record{int;nat}; Err: record{ error: text } }; // This comment is ignored +type res = variant { + /* + This is a block comment for variant Ok + */ + Ok: record{int;nat}; /* This comment is ignored */ + /* This comment is a block comment for variant Err */ + Err: record{ error: text }; /* This comment is ignored */ +}; // This comment is ignored type nested_res = variant { // This comment is ignored // This is a variant comment Ok : variant { Ok; Err }; // This comment is ignored @@ -32,6 +46,17 @@ type nested_res = variant { // This comment is ignored // This is another ignored comment +/* + This is an ignored block comment +*/ + +/* This is another ignored block comment */ + +/* +This is a block comment +that spans multiple lines +*/ + // This is a service comment // that spans multiple lines for services service server : { @@ -41,7 +66,15 @@ service server : { // This is a method comment g1 : (my_type, List, opt List, nested) -> (int, broker, nested_res) query; // This comment is ignored + /* + This is a block comment for a method + */ h : (vec opt text, variant { A: nat; B: opt text }, opt List) -> (record { id: nat; 0x2a: record {} }); // This comment is ignored + /* + This is a block comment for a method + that spans multiple lines, +even with wrong indentation + */ i : f;// This comment is ignored // This is another method comment // that spans multiple lines for methods diff --git a/rust/candid_parser/tests/assets/ok/actor.d.ts b/rust/candid_parser/tests/assets/ok/actor.d.ts index 38873aef..4a8ff0ce 100644 --- a/rust/candid_parser/tests/assets/ok/actor.d.ts +++ b/rust/candid_parser/tests/assets/ok/actor.d.ts @@ -7,7 +7,7 @@ export type g = f; export type h = ActorMethod<[[Principal, string]], [Principal, string]>; export type o = [] | [o]; /** - * This is a service comment + * This is a service block comment */ export interface _SERVICE { 'f' : ActorMethod<[bigint], [Principal, string]>, diff --git a/rust/candid_parser/tests/assets/ok/actor.did b/rust/candid_parser/tests/assets/ok/actor.did index 2be70b17..35900533 100644 --- a/rust/candid_parser/tests/assets/ok/actor.did +++ b/rust/candid_parser/tests/assets/ok/actor.did @@ -2,5 +2,5 @@ type f = func (int8) -> (int8); type g = f; type h = func (f) -> (f); type o = opt o; -// This is a service comment +// This is a service block comment service : { f : (nat) -> (h); g : f; h : g; o : (o) -> (o) } diff --git a/rust/candid_parser/tests/assets/ok/actor.mo b/rust/candid_parser/tests/assets/ok/actor.mo index cb7318c5..86ff124c 100644 --- a/rust/candid_parser/tests/assets/ok/actor.mo +++ b/rust/candid_parser/tests/assets/ok/actor.mo @@ -6,7 +6,7 @@ module { public type g = f; public type h = shared f -> async f; public type o = ?o; - /// This is a service comment + /// This is a service block comment public type Self = actor { f : shared Nat -> async h; g : f; diff --git a/rust/candid_parser/tests/assets/ok/actor.rs b/rust/candid_parser/tests/assets/ok/actor.rs index 3434fc54..2b52d06e 100644 --- a/rust/candid_parser/tests/assets/ok/actor.rs +++ b/rust/candid_parser/tests/assets/ok/actor.rs @@ -10,7 +10,7 @@ pub type G = F; #[derive(CandidType, Deserialize)] pub struct O(pub Option>); -/// This is a service comment +/// This is a service block comment pub struct Service(pub Principal); impl Service { pub async fn f(&self, arg0: &candid::Nat) -> Result<(H,)> { @@ -28,6 +28,6 @@ impl Service { } /// Canister ID: `aaaaa-aa` pub const CANISTER_ID : Principal = Principal::from_slice(&[]); -/// This is a service comment +/// This is a service block comment pub const service : Service = Service(CANISTER_ID); diff --git a/rust/candid_parser/tests/assets/ok/comment.d.ts b/rust/candid_parser/tests/assets/ok/comment.d.ts index e7f9de08..a2f9ff0e 100644 --- a/rust/candid_parser/tests/assets/ok/comment.d.ts +++ b/rust/candid_parser/tests/assets/ok/comment.d.ts @@ -8,5 +8,9 @@ export type a = { 'a' : null } | * This is a type comment */ export type b = [bigint, bigint]; +/** + * gjkf + * jgfkg + */ export type id = number; diff --git a/rust/candid_parser/tests/assets/ok/comment.did b/rust/candid_parser/tests/assets/ok/comment.did index 5b849fc6..235758db 100644 --- a/rust/candid_parser/tests/assets/ok/comment.did +++ b/rust/candid_parser/tests/assets/ok/comment.did @@ -1,5 +1,7 @@ type a = variant { a; b : b }; // This is a type comment type b = record { int; nat }; +// gjkf +// jgfkg type id = nat8; diff --git a/rust/candid_parser/tests/assets/ok/comment.mo b/rust/candid_parser/tests/assets/ok/comment.mo index 4ea8ce67..294dbb4e 100644 --- a/rust/candid_parser/tests/assets/ok/comment.mo +++ b/rust/candid_parser/tests/assets/ok/comment.mo @@ -5,6 +5,8 @@ module { public type a = { #a; #b : b }; /// This is a type comment public type b = (Int, Nat); + /// gjkf + /// jgfkg public type id = Nat8; } diff --git a/rust/candid_parser/tests/assets/ok/comment.rs b/rust/candid_parser/tests/assets/ok/comment.rs index 10af3ebb..5427cfce 100644 --- a/rust/candid_parser/tests/assets/ok/comment.rs +++ b/rust/candid_parser/tests/assets/ok/comment.rs @@ -9,6 +9,8 @@ pub enum A { #[serde(rename="a")] A, #[serde(rename="b")] B(Box) } /// This is a type comment #[derive(CandidType, Deserialize)] pub struct B (pub candid::Int,pub candid::Nat,); +/// gjkf +/// jgfkg pub type Id = u8; diff --git a/rust/candid_parser/tests/assets/ok/example.d.ts b/rust/candid_parser/tests/assets/ok/example.d.ts index 284d1e0e..406f857d 100644 --- a/rust/candid_parser/tests/assets/ok/example.d.ts +++ b/rust/candid_parser/tests/assets/ok/example.d.ts @@ -4,8 +4,14 @@ import type { IDL } from '@dfinity/candid'; export type A = B; export type B = [] | [A]; +/** + * This is a type block comment + */ export type List = [] | [ { + /** + * This is a field block comment + */ 'head' : bigint, /** * This is a field comment @@ -22,7 +28,12 @@ export type b = [bigint, bigint]; /** * This is another type comment */ -export interface broker { 'find' : ActorMethod<[string], Principal> } +export interface broker { + /** + * This is a service method block comment + */ + 'find' : ActorMethod<[string], Principal>, +} export type f = ActorMethod<[List, [Principal, string]], [[] | [List], res]>; export type list = [] | [node]; /** @@ -57,8 +68,18 @@ export type nested_res = { { 'Err' : [bigint] } }; export interface node { 'head' : bigint, 'tail' : list } -export type res = { 'Ok' : [bigint, bigint] } | - { 'Err' : { 'error' : string } }; +export type res = { + /** + * This is a block comment for variant Ok + */ + 'Ok' : [bigint, bigint] + } | + { + /** + * This comment is a block comment for variant Err + */ + 'Err' : { 'error' : string } + }; export interface s { 'f' : t, 'g' : ActorMethod<[list], [B, tree, stream]> } export type stream = [] | [{ 'head' : bigint, 'next' : [Principal, string] }]; export type t = ActorMethod<[Principal], undefined>; @@ -85,6 +106,9 @@ export interface _SERVICE { [my_type, List, [] | [List], nested], [bigint, Principal, nested_res] >, + /** + * This is a block comment for a method + */ 'h' : ActorMethod< [ Array<[] | [string]>, @@ -94,6 +118,11 @@ export interface _SERVICE { ], { _42_ : {}, 'id' : bigint } >, + /** + * This is a block comment for a method + * that spans multiple lines, + * even with wrong indentation + */ 'i' : f, /** * This is another method comment diff --git a/rust/candid_parser/tests/assets/ok/example.did b/rust/candid_parser/tests/assets/ok/example.did index 7e45a77a..9571c800 100644 --- a/rust/candid_parser/tests/assets/ok/example.did +++ b/rust/candid_parser/tests/assets/ok/example.did @@ -1,6 +1,8 @@ type A = B; type B = opt A; +// This is a type block comment type List = opt record { + // This is a field block comment head : int; // This is a field comment tail : List; @@ -10,6 +12,7 @@ type a = variant { a; b : b }; type b = record { int; nat }; // This is another type comment type broker = service { + // This is a service method block comment find : (name : text) -> (service { current : () -> (nat32); up : () -> () }); }; type f = func (List, func (int32) -> (int64)) -> (opt List, res); @@ -33,7 +36,12 @@ type nested_res = variant { Err : variant { Ok : record { content : text }; Err : record { int } }; }; type node = record { head : nat; tail : list }; -type res = variant { Ok : record { int; nat }; Err : record { error : text } }; +type res = variant { + // This is a block comment for variant Ok + Ok : record { int; nat }; + // This comment is a block comment for variant Err + Err : record { error : text }; +}; type s = service { f : t; g : (list) -> (B, tree, stream) }; type stream = opt record { head : nat; next : func () -> (stream) query }; type t = func (server : s) -> (); @@ -51,9 +59,13 @@ service : { g : (list) -> (B, tree, stream); // This is a method comment g1 : (my_type, List, opt List, nested) -> (int, broker, nested_res) query; + // This is a block comment for a method h : (vec opt text, variant { A : nat; B : opt text }, opt List) -> ( record { 42 : record {}; id : nat }, ); + // This is a block comment for a method + // that spans multiple lines, + // even with wrong indentation i : f; // This is another method comment // that spans multiple lines for methods diff --git a/rust/candid_parser/tests/assets/ok/example.mo b/rust/candid_parser/tests/assets/ok/example.mo index 7f22878b..6c883cca 100644 --- a/rust/candid_parser/tests/assets/ok/example.mo +++ b/rust/candid_parser/tests/assets/ok/example.mo @@ -4,7 +4,9 @@ module { public type A = B; public type B = ?A; + /// This is a type block comment public type List = ?{ + /// This is a field block comment head : Int; /// This is a field comment tail : List; @@ -14,6 +16,7 @@ module { public type b = (Int, Nat); /// This is another type comment public type broker = actor { + /// This is a service method block comment find : shared (name : Text) -> async actor { current : shared () -> async Nat32; up : shared () -> async (); @@ -43,7 +46,12 @@ module { #Err : { #Ok : { content : Text }; #Err : { _0_ : Int } }; }; public type node = { head : Nat; tail : list }; - public type res = { #Ok : (Int, Nat); #Err : { error : Text } }; + public type res = { + /// This is a block comment for variant Ok + #Ok : (Int, Nat); + /// This comment is a block comment for variant Err + #Err : { error : Text }; + }; public type s = actor { f : t; g : shared list -> async (B, tree, stream) }; public type stream = ?{ head : Nat; next : shared query () -> async stream }; public type t = shared (server : s) -> async (); @@ -65,10 +73,14 @@ module { broker, nested_res, ); + /// This is a block comment for a method h : shared ([?Text], { #A : Nat; #B : ?Text }, ?List) -> async { _42_ : {}; id : Nat; }; + /// This is a block comment for a method + /// that spans multiple lines, + /// even with wrong indentation i : f; /// This is another method comment /// that spans multiple lines for methods diff --git a/rust/candid_parser/tests/assets/ok/example.rs b/rust/candid_parser/tests/assets/ok/example.rs index 58a29ceb..5d00d3c9 100644 --- a/rust/candid_parser/tests/assets/ok/example.rs +++ b/rust/candid_parser/tests/assets/ok/example.rs @@ -38,6 +38,7 @@ candid::define_function!(pub(crate) T : (S) -> ()); type CanisterId = Principal; #derive[CandidType, Deserialize, Clone] pub(crate) struct ListInner { + /// This is a field block comment #[serde(skip_deserializing)] #[serde(rename="head")] HEAD: candid::Int, @@ -45,6 +46,7 @@ pub(crate) struct ListInner { #[serde(skip_deserializing)] tail: Arc, } +/// This is a type block comment #[derive(CandidType, Deserialize, Debug)] pub(crate) struct MyList(pub(crate) Option); #[derive(CandidType, Deserialize, Debug)] @@ -127,9 +129,13 @@ impl Service { pub async fn G11(&self, id: &CanisterId, list: &MyList, is_okay: &Option, arg3: &Nested) -> Result<(i128,Broker,NestedRes,)> { ic_cdk::call(self.0, "g1", (id,list,is_okay,arg3,)).await } + /// This is a block comment for a method pub async fn h(&self, arg0: &Vec>, arg1: &HArg1, arg2: &Option) -> Result<(HRet,)> { ic_cdk::call(self.0, "h", (arg0,arg1,arg2,)).await } + /// This is a block comment for a method + /// that spans multiple lines, + /// even with wrong indentation pub async fn i(&self, arg0: &MyList, arg1: &FArg1) -> Result<(Option,Res,)> { ic_cdk::call(self.0, "i", (arg0,arg1,)).await } @@ -147,8 +153,10 @@ pub const service : Service = Service(CANISTER_ID); #[test] fn test_Arc_MyList_() { // Generated from ListInner.record.tail.use_type = "Arc" - let candid_src = r#"type List = opt ListInner; + let candid_src = r#"// This is a type block comment +type List = opt ListInner; type ListInner = record { + // This is a field block comment head : int; // This is a field comment tail : List;