fixed comments breaking the parser

src/parse/lex.rs

@@ -868,13 +868,18 @@ impl Iterator for LexStream {
         self.cursor += 1;
 
         while let Some(ch) = get_char(&self.source, self.cursor) {
-          self.cursor += 1;
+          self.cursor += ch.len_utf8();
           if ch == '\n' {
             break;
           }
         }
 
-        self.get_token(ch_idx..self.cursor, TkRule::Comment)
+        if self.flags.contains(LexFlags::LEX_UNFINISHED) {
+          self.get_token(ch_idx..self.cursor, TkRule::Comment)
+        } else {
+					// After consuming the comment, we call next() recursively. This effectively filters out comment tokens.
+          return self.next();
+        }
       }
       '|' => {
         let ch_idx = self.cursor;

src/parse/mod.rs

@@ -87,6 +87,7 @@ impl ParsedSrc {
         Err(error) => return Err(vec![error]),
       }
     }
+		log::debug!("Tokens: {:#?}", tokens);
 
     let mut errors = vec![];
     let mut nodes = vec![];
@@ -1595,7 +1596,6 @@ impl ParseStream {
             redirs.push(redir);
           }
         }
-        TkRule::Comment => { /* Skip comments in command position */ }
         _ => unimplemented!("Unexpected token rule `{:?}` in parse_cmd()", tk.class),
       }
     }