Implemented syntax highlighting

src/builtin/alias.rs

@@ -85,7 +85,7 @@ pub fn unalias(node: Node, io_stack: &mut IoStack, job: &mut JobBldr) -> ShResul
     write(stdout, alias_output.as_bytes())?; // Write it
   } else {
     for (arg, span) in argv {
-      flog!(DEBUG, arg);
+      log::debug!("{arg:?}");
       if read_logic(|l| l.get_alias(&arg)).is_none() {
         return Err(ShErr::full(
           ShErrKind::SyntaxErr,

src/builtin/flowctl.rs

@@ -32,7 +32,7 @@ pub fn flowctl(node: Node, kind: ShErrKind) -> ShResult<()> {
     code = status;
   }
 
-  flog!(DEBUG, code);
+  log::debug!("{code:?}");
 
   let kind = match kind {
     LoopContinue(_) => LoopContinue(code),

src/builtin/read.rs

@@ -16,10 +16,10 @@ pub const READ_OPTS: [OptSpec;7] = [
 bitflags! {
 	pub struct ReadFlags: u32 {
 		const NO_ESCAPES = 	0b000001;
-		const NO_ECHO = 		0b000010;
+		const NO_ECHO = 		0b000010; // TODO: unused
-		const ARRAY = 			0b000100;
+		const ARRAY = 			0b000100; // TODO: unused
-		const N_CHARS = 		0b001000;
+		const N_CHARS = 		0b001000; // TODO: unused
-		const TIMEOUT = 		0b010000;
+		const TIMEOUT = 		0b010000; // TODO: unused
 	}
 }
 

src/builtin/test.rs

@@ -247,9 +247,9 @@ pub fn double_bracket_test(node: Node) -> ShResult<bool> {
         let rhs = rhs.expand()?.get_words().join(" ");
         conjunct_op = conjunct;
         let test_op = operator.as_str().parse::<TestOp>()?;
-        flog!(DEBUG, lhs);
+        log::debug!("{lhs:?}");
-        flog!(DEBUG, rhs);
+        log::debug!("{rhs:?}");
-        flog!(DEBUG, test_op);
+        log::debug!("{test_op:?}");
         match test_op {
           TestOp::Unary(_) => {
             return Err(ShErr::Full {
@@ -298,7 +298,7 @@ pub fn double_bracket_test(node: Node) -> ShResult<bool> {
         }
       }
     };
-    flog!(DEBUG, last_result);
+    log::debug!("{last_result:?}");
 
     if let Some(op) = conjunct_op {
       match op {
@@ -316,6 +316,6 @@ pub fn double_bracket_test(node: Node) -> ShResult<bool> {
       last_result = result;
     }
   }
-  flog!(DEBUG, last_result);
+  log::debug!("{last_result:?}");
   Ok(last_result)
 }

src/expand.rs

@@ -462,13 +462,14 @@ pub fn expand_raw(chars: &mut Peekable<Chars<'_>>) -> ShResult<String> {
         result.push_str(&fd_path);
       }
       VAR_SUB => {
-        flog!(INFO, chars);
+        log::info!("{chars:?}");
         let expanded = expand_var(chars)?;
         result.push_str(&expanded);
       }
       _ => result.push(ch),
     }
   }
+	log::debug!("expand_raw result: {result:?}");
   Ok(result)
 }
 
@@ -511,14 +512,14 @@ pub fn expand_var(chars: &mut Peekable<Chars<'_>>) -> ShResult<String> {
 					return Ok(NULL_EXPAND.to_string());
 				}
 
-        flog!(DEBUG, val);
+        log::debug!("{val:?}");
         return Ok(val);
       }
       ch if is_hard_sep(ch) || !(ch.is_alphanumeric() || ch == '_' || ch == '-') => {
         let val = read_vars(|v| v.get_var(&var_name));
-        flog!(INFO, var_name);
+        log::info!("{var_name:?}");
-        flog!(INFO, val);
+        log::info!("{val:?}");
-        flog!(INFO, ch);
+        log::info!("{ch:?}");
         return Ok(val);
       }
       _ => {
@@ -529,7 +530,7 @@ pub fn expand_var(chars: &mut Peekable<Chars<'_>>) -> ShResult<String> {
   }
   if !var_name.is_empty() {
     let var_val = read_vars(|v| v.get_var(&var_name));
-    flog!(INFO, var_val);
+    log::info!("{var_val:?}");
     Ok(var_val)
   } else {
     Ok(String::new())
@@ -780,7 +781,7 @@ pub fn expand_proc_sub(raw: &str, is_input: bool) -> ShResult<String> {
     ForkResult::Parent { child } => {
       write_jobs(|j| j.register_fd(child, register_fd));
       let registered = read_jobs(|j| j.registered_fds().to_vec());
-      flog!(DEBUG, registered);
+      log::debug!("{registered:?}");
       // Do not wait; process may run in background
       Ok(path)
     }
@@ -789,8 +790,8 @@ pub fn expand_proc_sub(raw: &str, is_input: bool) -> ShResult<String> {
 
 /// Get the command output of a given command input as a String
 pub fn expand_cmd_sub(raw: &str) -> ShResult<String> {
-  flog!(DEBUG, "in expand_cmd_sub");
+  log::debug!("in expand_cmd_sub");
-  flog!(DEBUG, raw);
+  log::debug!("{raw:?}");
   if raw.starts_with('(') && raw.ends_with(')')
     && let Ok(output) = expand_arithmetic(raw) {
       return Ok(output); // It's actually an arithmetic sub
@@ -814,7 +815,7 @@ pub fn expand_cmd_sub(raw: &str) -> ShResult<String> {
       std::mem::drop(cmd_sub_io_frame); // Closes the write pipe
 
       // Read output first (before waiting) to avoid deadlock if child fills pipe buffer
-      flog!(DEBUG, "filling buffer");
+      log::debug!("filling buffer");
       loop {
         match io_buf.fill_buffer() {
           Ok(()) => break,
@@ -822,7 +823,7 @@ pub fn expand_cmd_sub(raw: &str) -> ShResult<String> {
           Err(e) => return Err(e.into()),
         }
       }
-      flog!(DEBUG, "done");
+      log::debug!("done");
 
       // Wait for child with EINTR retry
       let status = loop {
@@ -1104,7 +1105,7 @@ pub fn unescape_math(raw: &str) -> String {
   let mut result = String::new();
 
   while let Some(ch) = chars.next() {
-    flog!(DEBUG, result);
+    log::debug!("{result:?}");
     match ch {
       '\\' => {
         if let Some(next_ch) = chars.next() {
@@ -1147,7 +1148,7 @@ pub fn unescape_math(raw: &str) -> String {
       _ => result.push(ch),
     }
   }
-  flog!(INFO, result);
+  log::info!("{result:?}");
   result
 }
 
@@ -1301,9 +1302,9 @@ pub fn perform_param_expansion(raw: &str) -> ShResult<String> {
     }
   }
 
-  flog!(DEBUG, rest);
+  log::debug!("{rest:?}");
   if let Ok(expansion) = rest.parse::<ParamExp>() {
-    flog!(DEBUG, expansion);
+    log::debug!("{expansion:?}");
     match expansion {
       ParamExp::Len => unreachable!(),
       ParamExp::DefaultUnsetOrNull(default) => {
@@ -1522,7 +1523,7 @@ fn glob_to_regex(glob: &str, anchored: bool) -> Regex {
   if anchored {
     regex.push('$');
   }
-  flog!(DEBUG, regex);
+  log::debug!("{regex:?}");
   Regex::new(&regex).unwrap()
 }
 
@@ -1945,7 +1946,7 @@ pub fn expand_prompt(raw: &str) -> ShResult<String> {
 			PromptTk::FailureSymbol => todo!(),
 			PromptTk::JobCount => todo!(),
 			PromptTk::Function(f) => {
-				flog!(DEBUG, "Expanding prompt function: {}", f);
+				log::debug!("Expanding prompt function: {}", f);
 				let output = expand_cmd_sub(&f)?;
 				result.push_str(&output);
 			}

src/jobs.rs

@@ -84,7 +84,7 @@ impl ChildProc {
     if let Some(pgid) = pgid {
       child.set_pgid(pgid).ok();
     }
-    flog!(TRACE, "new child: {:?}", child);
+    log::trace!("new child: {:?}", child);
     Ok(child)
   }
   pub fn pid(&self) -> Pid {
@@ -520,11 +520,11 @@ impl Job {
   }
   pub fn wait_pgrp(&mut self) -> ShResult<Vec<WtStat>> {
     let mut stats = vec![];
-    flog!(TRACE, "waiting on children");
+    log::trace!("waiting on children");
-    flog!(TRACE, self.children);
+    log::trace!("{:?}", self.children);
     for child in self.children.iter_mut() {
-      flog!(TRACE, "shell pid {}", Pid::this());
+      log::trace!("shell pid {}", Pid::this());
-      flog!(TRACE, "child pid {}", child.pid);
+      log::trace!("child pid {}", child.pid);
       if child.pid == Pid::this() {
         // TODO: figure out some way to get the exit code of builtins
         let code = state::get_status();
@@ -667,7 +667,7 @@ pub fn wait_fg(job: Job) -> ShResult<()> {
   if job.children().is_empty() {
     return Ok(()); // Nothing to do
   }
-  flog!(TRACE, "Waiting on foreground job");
+  log::trace!("Waiting on foreground job");
   let mut code = 0;
   let mut was_stopped = false;
   attach_tty(job.pgid())?;
@@ -699,7 +699,7 @@ pub fn wait_fg(job: Job) -> ShResult<()> {
   }
   take_term()?;
   set_status(code);
-  flog!(TRACE, "exit code: {}", code);
+  log::trace!("exit code: {}", code);
   enable_reaping();
   Ok(())
 }
@@ -719,7 +719,7 @@ pub fn attach_tty(pgid: Pid) -> ShResult<()> {
   if !isatty(0).unwrap_or(false) || pgid == term_ctlr() || killpg(pgid, None).is_err() {
     return Ok(());
   }
-  flog!(TRACE, "Attaching tty to pgid: {}", pgid);
+  log::trace!("Attaching tty to pgid: {}", pgid);
 
   if pgid == getpgrp() && term_ctlr() != getpgrp() {
     kill(term_ctlr(), Signal::SIGTTOU).ok();
@@ -745,7 +745,7 @@ pub fn attach_tty(pgid: Pid) -> ShResult<()> {
   match result {
     Ok(_) => Ok(()),
     Err(e) => {
-      flog!(ERROR, "error while switching term control: {}", e);
+      log::error!("error while switching term control: {}", e);
       tcsetpgrp(borrow_fd(0), getpgrp())?;
       Ok(())
     }

src/libsh/utils.rs

@@ -84,7 +84,7 @@ impl TkVecUtils<Tk> for Vec<Tk> {
   }
   fn debug_tokens(&self) {
     for token in self {
-      flog!(DEBUG, "token: {}", token)
+      log::debug!("token: {}", token)
     }
   }
 }

src/parse/execute.rs

@@ -5,15 +5,12 @@ use crate::{
 		alias::{alias, unalias}, cd::cd, echo::echo, export::export, flowctl::flowctl, jobctl::{JobBehavior, continue_job, jobs}, pwd::pwd, read::read_builtin, shift::shift, shopt::shopt, source::source, test::double_bracket_test, zoltraak::zoltraak
 	},
 	expand::expand_aliases,
-	jobs::{ChildProc, JobBldr, JobStack, dispatch_job},
+	jobs::{ChildProc, JobStack, dispatch_job},
-	libsh::{
+	libsh::error::{ShErr, ShErrKind, ShResult, ShResultExt},
-		error::{ShErr, ShErrKind, ShResult, ShResultExt},
-		utils::RedirVecUtils,
-	},
 	prelude::*,
-	procio::{IoFrame, IoMode, IoStack},
+	procio::{IoMode, IoStack},
 	state::{
-		self, FERN, ShFunc, VarFlags, read_logic, write_logic, write_meta, write_vars
+		self, ShFunc, VarFlags, read_logic, write_logic, write_vars
 	},
 };
 
@@ -141,7 +138,7 @@ impl Dispatcher {
 		}
 	}
 	pub fn begin_dispatch(&mut self) -> ShResult<()> {
-		flog!(TRACE, "beginning dispatch");
+		log::trace!("beginning dispatch");
 		while let Some(node) = self.nodes.pop_front() {
 			let blame = node.get_span();
 			self.dispatch_node(node).try_blame(blame)?;
@@ -543,7 +540,7 @@ impl Dispatcher {
 			return self.dispatch_cmd(cmd);
 		}
 
-		flog!(TRACE, "doing builtin");
+		log::trace!("doing builtin");
 		let result = match cmd_raw.span.as_str() {
 			"echo" => echo(cmd, io_stack_mut, curr_job_mut),
 			"cd" => cd(cmd, curr_job_mut),
@@ -596,20 +593,47 @@ impl Dispatcher {
 		self.io_stack.append_to_frame(cmd.redirs);
 
 		let exec_args = ExecArgs::new(argv)?;
-		if self.interactive {
-			log::info!("expanded argv: {:?}", exec_args.argv.iter().map(|s| s.to_str().unwrap()).collect::<Vec<_>>());
-		}
 
 		let _guard = self.io_stack
 			.pop_frame()
 			.redirect()?;
 
-		run_fork(
+		let job = self.job_stack.curr_job_mut().unwrap();
-			Some(exec_args),
+
-			self.job_stack.curr_job_mut().unwrap(),
+		match unsafe { fork()? } {
-			def_child_action,
+			ForkResult::Child => {
-			def_parent_action,
+				let cmd = &exec_args.cmd.0;
-		)?;
+				let span = exec_args.cmd.1;
+
+				let Err(e) = execvpe(cmd, &exec_args.argv, &exec_args.envp);
+
+				// execvpe only returns on error
+				let cmd_str = cmd.to_str().unwrap().to_string();
+				match e {
+					Errno::ENOENT => {
+						let err = ShErr::full(ShErrKind::CmdNotFound(cmd_str), "", span);
+						eprintln!("{err}");
+					}
+					_ => {
+						let err = ShErr::full(ShErrKind::Errno(e), format!("{e}"), span);
+						eprintln!("{err}");
+					}
+				}
+				exit(e as i32)
+			}
+			ForkResult::Parent { child } => {
+				let cmd_name = exec_args.cmd.0.to_str().unwrap();
+
+				let child_pgid = if let Some(pgid) = job.pgid() {
+					pgid
+				} else {
+					job.set_pgid(child);
+					child
+				};
+				let child_proc = ChildProc::new(child, Some(cmd_name), Some(child_pgid))?;
+				job.push_child(child_proc);
+			}
+		}
 
 		for var in env_vars_to_unset {
 			unsafe { std::env::set_var(&var, "") };
@@ -671,67 +695,6 @@ pub fn prepare_argv(argv: Vec<Tk>) -> ShResult<Vec<(String, Span)>> {
 	Ok(args)
 }
 
-pub fn run_fork<C, P>(
-	exec_args: Option<ExecArgs>,
-	job: &mut JobBldr,
-	child_action: C,
-	parent_action: P,
-) -> ShResult<()>
-where
-	C: Fn(Option<ExecArgs>),
-	P: Fn(&mut JobBldr, Option<&str>, Pid) -> ShResult<()>,
-{
-	match unsafe { fork()? } {
-		ForkResult::Child => {
-			child_action(exec_args);
-			exit(0); // Just in case
-		}
-		ForkResult::Parent { child } => {
-			let cmd = if let Some(args) = exec_args {
-				Some(args.cmd.0.to_str().unwrap().to_string())
-			} else {
-				None
-			};
-			parent_action(job, cmd.as_deref(), child)
-		}
-	}
-}
-
-/// The default behavior for the child process after forking
-pub fn def_child_action(exec_args: Option<ExecArgs>) {
-	let exec_args = exec_args.unwrap();
-	let cmd = &exec_args.cmd.0;
-	let span = exec_args.cmd.1;
-
-	let Err(e) = execvpe(cmd, &exec_args.argv, &exec_args.envp);
-
-	let cmd = cmd.to_str().unwrap().to_string();
-	match e {
-		Errno::ENOENT => {
-			let err = ShErr::full(ShErrKind::CmdNotFound(cmd), "", span);
-			eprintln!("{err}");
-		}
-		_ => {
-			let err = ShErr::full(ShErrKind::Errno(e), format!("{e}"), span);
-			eprintln!("{err}");
-		}
-	}
-	exit(e as i32)
-}
-
-/// The default behavior for the parent process after forking
-pub fn def_parent_action(job: &mut JobBldr, cmd: Option<&str>, child_pid: Pid) -> ShResult<()> {
-	let child_pgid = if let Some(pgid) = job.pgid() {
-		pgid
-	} else {
-		job.set_pgid(child_pid);
-		child_pid
-	};
-	let child = ChildProc::new(child_pid, cmd, Some(child_pgid))?;
-	job.push_child(child);
-	Ok(())
-}
-
 /// Initialize the pipes for a pipeline
 /// The first command gets `(None, WPipe)`
 /// The last command gets `(RPipe, None)`

src/parse/lex.rs

@@ -176,7 +176,7 @@ bitflags! {
 
 impl LexStream {
   pub fn new(source: Arc<String>, flags: LexFlags) -> Self {
-    flog!(TRACE, "new lex stream");
+    log::trace!("new lex stream");
     let flags = flags | LexFlags::FRESH | LexFlags::NEXT_IS_CMD;
     Self {
       source,
@@ -405,10 +405,6 @@ impl LexStream {
               Span::new(paren_pos..paren_pos + 1, self.source.clone()),
             ));
           }
-          let mut proc_sub_tk = self.get_token(self.cursor..pos, TkRule::Str);
-          proc_sub_tk.flags |= TkFlags::IS_PROCSUB;
-          self.cursor = pos;
-          return Ok(proc_sub_tk);
         }
         '>' if chars.peek() == Some(&'(') => {
           pos += 2;
@@ -445,10 +441,6 @@ impl LexStream {
               Span::new(paren_pos..paren_pos + 1, self.source.clone()),
             ));
           }
-          let mut proc_sub_tk = self.get_token(self.cursor..pos, TkRule::Str);
-          proc_sub_tk.flags |= TkFlags::IS_PROCSUB;
-          self.cursor = pos;
-          return Ok(proc_sub_tk);
         }
         '$' if chars.peek() == Some(&'(') => {
           pos += 2;
@@ -485,10 +477,6 @@ impl LexStream {
               Span::new(paren_pos..paren_pos + 1, self.source.clone()),
             ));
           }
-          let mut cmdsub_tk = self.get_token(self.cursor..pos, TkRule::Str);
-          cmdsub_tk.flags |= TkFlags::IS_CMDSUB;
-          self.cursor = pos;
-          return Ok(cmdsub_tk);
         }
         '(' if self.next_is_cmd() && can_be_subshell => {
           pos += 1;
@@ -802,7 +790,7 @@ impl Iterator for LexStream {
           match self.read_string() {
             Ok(tk) => tk,
             Err(e) => {
-              flog!(ERROR, e);
+              log::error!("{e:?}");
               return Some(Err(e));
             }
           }

src/procio.rs

@@ -137,9 +137,9 @@ impl<R: Read> IoBuf<R> {
   pub fn fill_buffer(&mut self) -> io::Result<()> {
     let mut temp_buf = vec![0; 1024]; // Read in chunks
     loop {
-      flog!(DEBUG, "reading bytes");
+      log::debug!("reading bytes");
       let bytes_read = self.reader.read(&mut temp_buf)?;
-      flog!(DEBUG, bytes_read);
+      log::debug!("{bytes_read:?}");
       if bytes_read == 0 {
         break; // EOF reached
       }
@@ -220,11 +220,11 @@ impl<'e> IoFrame {
     self.save();
     for redir in &mut self.redirs {
       let io_mode = &mut redir.io_mode;
-      flog!(DEBUG, io_mode);
+      log::debug!("{io_mode:?}");
       if let IoMode::File { .. } = io_mode {
         *io_mode = io_mode.clone().open_file()?;
       };
-      flog!(DEBUG, io_mode);
+      log::debug!("{io_mode:?}");
       let tgt_fd = io_mode.tgt_fd();
       let src_fd = io_mode.src_fd();
       dup2(src_fd, tgt_fd)?;

src/prompt/highlight.rs

@@ -1 +0,0 @@
-

src/prompt/mod.rs

@@ -18,7 +18,7 @@ pub fn get_prompt() -> ShResult<String> {
     return expand_prompt(default);
   };
 	let sanitized = format!("\\e[0m{prompt}");
-	flog!(DEBUG, "Using prompt: {}", sanitized.replace("\n", "\\n"));
+	log::debug!("Using prompt: {}", sanitized.replace("\n", "\\n"));
 
   expand_prompt(&sanitized)
 }

src/prompt/readline/highlight.rs

@@ -0,0 +1,245 @@
+use std::{env, path::{Path, PathBuf}};
+
+use crate::{libsh::term::{Style, StyleSet, Styled}, prompt::readline::{annotate_input, markers}, state::read_logic};
+
+pub struct Highlighter {
+	input: String,
+	output: String,
+	style_stack: Vec<StyleSet>,
+	last_was_reset: bool,
+}
+
+impl Highlighter {
+	pub fn new() -> Self {
+		Self {
+			input: String::new(),
+			output: String::new(),
+			style_stack: Vec::new(),
+			last_was_reset: true, // start as true so we don't emit a leading reset
+		}
+	}
+
+	pub fn load_input(&mut self, input: &str) {
+		let input = annotate_input(input);
+		self.input = input;
+	}
+
+	pub fn highlight(&mut self) {
+		let input = self.input.clone();
+		let mut input_chars = input.chars().peekable();
+		while let Some(ch) = input_chars.next() {
+			match ch {
+				markers::STRING_DQ_END |
+				markers::STRING_SQ_END |
+				markers::VAR_SUB_END |
+				markers::CMD_SUB_END |
+				markers::PROC_SUB_END |
+				markers::SUBSH_END => self.pop_style(),
+
+				markers::CMD_SEP |
+				markers::RESET => self.clear_styles(),
+
+
+				markers::STRING_DQ |
+				markers::STRING_SQ |
+				markers::KEYWORD => self.push_style(Style::Yellow),
+				markers::BUILTIN => self.push_style(Style::Green),
+				markers::CASE_PAT => self.push_style(Style::Blue),
+				markers::ARG => self.push_style(Style::White),
+				markers::COMMENT => self.push_style(Style::BrightBlack),
+
+				markers::GLOB => self.push_style(Style::Blue),
+
+				markers::REDIRECT |
+				markers::OPERATOR => self.push_style(Style::Magenta | Style::Bold),
+
+				markers::ASSIGNMENT => {
+					let mut var_name = String::new();
+
+					while let Some(ch) = input_chars.peek() {
+						if ch == &'=' {
+							input_chars.next(); // consume the '='
+							break;
+						}
+						match *ch {
+							markers::RESET => break,
+							_ => {
+								var_name.push(*ch);
+								input_chars.next();
+							}
+						}
+					}
+
+					self.output.push_str(&var_name);
+					self.push_style(Style::Blue);
+					self.output.push('=');
+					self.pop_style();
+				}
+
+				markers::COMMAND => {
+					let mut cmd_name = String::new();
+					while let Some(ch) = input_chars.peek() {
+						if *ch == markers::RESET {
+							break;
+						}
+						cmd_name.push(*ch);
+						input_chars.next();
+					}
+					let style = if Self::is_valid(&cmd_name) {
+						Style::Green.into()
+					} else {
+						Style::Red | Style::Bold
+					};
+					self.push_style(style);
+					self.output.push_str(&cmd_name);
+					self.last_was_reset = false;
+				}
+				markers::CMD_SUB | markers::SUBSH | markers::PROC_SUB => {
+					let mut inner = String::new();
+					let mut incomplete = true;
+					let end_marker = match ch {
+						markers::CMD_SUB => markers::CMD_SUB_END,
+						markers::SUBSH => markers::SUBSH_END,
+						markers::PROC_SUB => markers::PROC_SUB_END,
+						_ => unreachable!(),
+					};
+					while let Some(ch) = input_chars.peek() {
+						if *ch == end_marker {
+							incomplete = false;
+							input_chars.next(); // consume the end marker
+							break;
+						}
+						inner.push(*ch);
+						input_chars.next();
+					}
+
+					// Determine prefix from content (handles both <( and >( for proc subs)
+					let prefix = match ch {
+						markers::CMD_SUB => "$(",
+						markers::SUBSH => "(",
+						markers::PROC_SUB => {
+							if inner.starts_with("<(") { "<(" }
+							else if inner.starts_with(">(") { ">(" }
+							else { "<(" } // fallback
+						}
+						_ => unreachable!(),
+					};
+
+					let inner_content = if incomplete {
+						inner
+							.strip_prefix(prefix)
+							.unwrap_or(&inner)
+					} else {
+						inner
+							.strip_prefix(prefix)
+							.and_then(|s| s.strip_suffix(")"))
+							.unwrap_or(&inner)
+					};
+
+					let mut recursive_highlighter = Self::new();
+					recursive_highlighter.load_input(inner_content);
+					recursive_highlighter.highlight();
+					self.push_style(Style::Blue);
+					self.output.push_str(prefix);
+					self.pop_style();
+					self.output.push_str(&recursive_highlighter.take());
+					if !incomplete {
+						self.push_style(Style::Blue);
+						self.output.push(')');
+						self.pop_style();
+					}
+					self.last_was_reset = false;
+				}
+				markers::VAR_SUB => {
+					let mut var_sub = String::new();
+					while let Some(ch) = input_chars.peek() {
+						if *ch == markers::VAR_SUB_END {
+							input_chars.next(); // consume the end marker
+							break;
+						}
+						var_sub.push(*ch);
+						input_chars.next();
+					}
+					let style = Style::Cyan;
+					self.push_style(style);
+					self.output.push_str(&var_sub);
+					self.pop_style();
+				}
+				_ => {
+					self.output.push(ch);
+					self.last_was_reset = false;
+				}
+			}
+		}
+	}
+
+	pub fn take(&mut self) -> String {
+		log::info!("Highlighting result: {:?}", self.output);
+		self.input.clear();
+		self.clear_styles();
+		std::mem::take(&mut self.output)
+	}
+
+	fn is_valid(command: &str) -> bool {
+		let path = env::var("PATH").unwrap_or_default();
+		let paths = path.split(':');
+		if PathBuf::from(&command).exists() {
+			return true;
+		} else {
+			for path in paths {
+				let path = PathBuf::from(path).join(command);
+				if path.exists() {
+					return true;
+				}
+			}
+
+			let found = read_logic(|l| l.get_func(command).is_some() || l.get_alias(command).is_some());
+			if found {
+				return true;
+			}
+		}
+
+		false
+	}
+
+	fn emit_reset(&mut self) {
+		if !self.last_was_reset {
+			self.output.push_str(&Style::Reset.to_string());
+			self.last_was_reset = true;
+		}
+	}
+
+	fn emit_style(&mut self, style: &StyleSet) {
+		self.output.push_str(&style.to_string());
+		self.last_was_reset = false;
+	}
+
+	pub fn push_style(&mut self, style: impl Into<StyleSet>) {
+		let set: StyleSet = style.into();
+		self.style_stack.push(set.clone());
+		self.emit_style(&set);
+	}
+
+	pub fn pop_style(&mut self) {
+		self.style_stack.pop();
+		if let Some(style) = self.style_stack.last().cloned() {
+			self.emit_style(&style);
+		} else {
+			self.emit_reset();
+		}
+	}
+
+	pub fn clear_styles(&mut self) {
+		self.style_stack.clear();
+		self.emit_reset();
+	}
+
+	pub fn trivial_replace(&mut self) {
+		self.input = self.input
+			.replace([markers::RESET, markers::ARG], "\x1b[0m")
+			.replace(markers::KEYWORD, "\x1b[33m")
+			.replace(markers::CASE_PAT, "\x1b[34m")
+			.replace(markers::COMMENT, "\x1b[90m")
+			.replace(markers::OPERATOR, "\x1b[35m");
+	}
+}

src/prompt/readline/history.rs

@@ -53,18 +53,10 @@ impl HistEntry {
   }
   fn with_escaped_newlines(&self) -> String {
     let mut escaped = String::new();
-    let mut chars = self.command.chars();
+    for ch in self.command.chars() {
-    while let Some(ch) = chars.next() {
       match ch {
-        '\\' => {
+        '\\' => escaped.push_str("\\\\"), // escape all backslashes
-          escaped.push(ch);
+        '\n' => escaped.push_str("\\\n"), // line continuation
-          if let Some(ch) = chars.next() {
-            escaped.push(ch)
-          }
-        }
-        '\n' => {
-          escaped.push_str("\\\n");
-        }
         _ => escaped.push(ch),
       }
     }
@@ -155,8 +147,10 @@ impl FromStr for HistEntries {
           match ch {
             '\\' => {
               if let Some(esc_ch) = chars.next() {
+                // Unescape: \\ -> \, \n stays as literal n after backslash was written as \\n
                 cur_line.push(esc_ch);
               } else {
+                // Trailing backslash = line continuation in history file format
                 cur_line.push('\n');
                 feeding_lines = true;
               }
@@ -228,20 +222,17 @@ impl History {
       format!("{home}/.fern_history")
     }));
     let mut entries = read_hist_file(&path)?;
-    {
+    // Create pending entry for current input
-      let id = entries.last().map(|ent| ent.id + 1).unwrap_or(0);
+    let id = entries.last().map(|ent| ent.id + 1).unwrap_or(0);
-      let timestamp = SystemTime::now();
+    entries.push(HistEntry {
-      let command = "".into();
+      id,
-      entries.push(HistEntry {
+      timestamp: SystemTime::now(),
-        id,
+      command: String::new(),
-        timestamp,
+      new: true,
-        command,
+    });
-        new: true,
-      })
-    }
     let search_mask = dedupe_entries(&entries);
-    let cursor = entries.len() - 1;
+    let cursor = search_mask.len().saturating_sub(1);
-    let mut new = Self {
+    Ok(Self {
       path,
       entries,
       search_mask,
@@ -249,11 +240,14 @@ impl History {
       search_direction: Direction::Backward,
       ignore_dups: true,
       max_size: None,
-    };
+    })
-    new.push_empty_entry(); // Current pending command
-    Ok(new)
   }
 
+	pub fn reset(&mut self) {
+		self.search_mask = dedupe_entries(&self.entries);
+		self.cursor = self.search_mask.len().saturating_sub(1);
+	}
+
   pub fn entries(&self) -> &[HistEntry] {
     &self.entries
   }
@@ -262,7 +256,16 @@ impl History {
     &self.search_mask
   }
 
-  pub fn push_empty_entry(&mut self) {}
+  pub fn push_empty_entry(&mut self) {
+		let timestamp = SystemTime::now();
+		let id = self.get_new_id();
+		self.entries.push(HistEntry {
+			id,
+			timestamp,
+			command: String::new(),
+			new: true,
+		});
+	}
 
   pub fn cursor_entry(&self) -> Option<&HistEntry> {
     self.search_mask.get(self.cursor)
@@ -300,7 +303,7 @@ impl History {
   }
 
   pub fn constrain_entries(&mut self, constraint: SearchConstraint) {
-    flog!(DEBUG, constraint);
+    log::debug!("{constraint:?}");
     let SearchConstraint { kind, term } = constraint;
     match kind {
       SearchKind::Prefix => {
@@ -315,6 +318,7 @@ impl History {
             .collect();
 
           self.search_mask = dedupe_entries(&filtered);
+					log::debug!("search mask len: {}", self.search_mask.len());
         }
         self.cursor = self.search_mask.len().saturating_sub(1);
       }
@@ -324,10 +328,12 @@ impl History {
 
   pub fn hint_entry(&self) -> Option<&HistEntry> {
     let second_to_last = self.search_mask.len().checked_sub(2)?;
+		log::info!("search mask: {:?}", self.search_mask.iter().map(|e| e.command()).collect::<Vec<_>>());
     self.search_mask.get(second_to_last)
   }
 
   pub fn get_hint(&self) -> Option<String> {
+		log::info!("checking cursor entry: {:?}", self.cursor_entry());
     if self
       .cursor_entry()
       .is_some_and(|ent| ent.is_new() && !ent.command().is_empty())
@@ -382,9 +388,7 @@ impl History {
 
     let last_file_entry = self
       .entries
-      .iter()
+      .iter().rfind(|ent| !ent.new)
-      .filter(|ent| !ent.new)
-      .next_back()
       .map(|ent| ent.command.clone())
       .unwrap_or_default();
 
@@ -405,6 +409,8 @@ impl History {
     }
 
     file.write_all(data.as_bytes())?;
+		self.push_empty_entry(); // Prepare for next command
+		self.reset(); // Reset search mask to include new pending entry
 
     Ok(())
   }

src/prompt/readline/linebuf.rs

@@ -368,7 +368,7 @@ impl LineBuf {
     } else {
       self.hint = None
     }
-    flog!(DEBUG, self.hint)
+    log::debug!("{:?}", self.hint)
   }
   pub fn accept_hint(&mut self) {
     let Some(hint) = self.hint.take() else { return };
@@ -406,7 +406,7 @@ impl LineBuf {
   #[track_caller]
   pub fn update_graphemes(&mut self) {
     let indices: Vec<_> = self.buffer.grapheme_indices(true).map(|(i, _)| i).collect();
-    flog!(DEBUG, std::panic::Location::caller());
+    log::debug!("{:?}", std::panic::Location::caller());
     self.cursor.set_max(indices.len());
     self.grapheme_indices = Some(indices)
   }
@@ -564,6 +564,8 @@ impl LineBuf {
     self.update_graphemes();
   }
   pub fn drain(&mut self, start: usize, end: usize) -> String {
+		let start = start.max(0);
+		let end = end.min(self.grapheme_indices().len());
     let drained = if end == self.grapheme_indices().len() {
       if start == self.grapheme_indices().len() {
         return String::new();
@@ -575,7 +577,7 @@ impl LineBuf {
       let end = self.grapheme_indices()[end];
       self.buffer.drain(start..end).collect()
     };
-    flog!(DEBUG, drained);
+    log::debug!("{drained:?}");
     self.update_graphemes();
     drained
   }
@@ -1071,7 +1073,7 @@ impl LineBuf {
           let Some(gr) = self.grapheme_at(idx) else {
             break;
           };
-          flog!(DEBUG, gr);
+          log::debug!("{gr:?}");
           if is_whitespace(gr) {
             end += 1;
           } else {
@@ -1201,7 +1203,7 @@ impl LineBuf {
           let Some(gr) = self.grapheme_at(idx) else {
             break;
           };
-          flog!(DEBUG, gr);
+          log::debug!("{gr:?}");
           if is_whitespace(gr) {
             end += 1;
           } else {
@@ -1899,10 +1901,10 @@ impl LineBuf {
         let Some(line) = self.slice(start..end).map(|s| s.to_string()) else {
           return MotionKind::Null;
         };
-        flog!(DEBUG, target_col);
+        log::debug!("{target_col:?}");
-        flog!(DEBUG, target_col);
+        log::debug!("{target_col:?}");
         let mut target_pos = self.grapheme_index_for_display_col(&line, target_col);
-        flog!(DEBUG, target_pos);
+        log::debug!("{target_pos:?}");
         if self.cursor.exclusive
           && line.ends_with("\n")
           && self.grapheme_at(target_pos) == Some("\n")
@@ -2085,7 +2087,7 @@ impl LineBuf {
           };
           match direction {
             Direction::Forward => pos.add(ch_pos + 1),
-            Direction::Backward => pos.sub(ch_pos.saturating_sub(1)),
+            Direction::Backward => pos.sub(ch_pos + 1),
           }
 
           if dest == Dest::Before {
@@ -2105,7 +2107,7 @@ impl LineBuf {
             Motion::BackwardChar => target.sub(1),
             Motion::ForwardChar => {
               if self.cursor.exclusive && self.grapheme_at(target.ret_add(1)) == Some("\n") {
-                flog!(DEBUG, "returning null");
+                log::debug!("returning null");
                 return MotionKind::Null;
               }
               target.add(1);
@@ -2114,7 +2116,7 @@ impl LineBuf {
             _ => unreachable!(),
           }
           if self.grapheme_at(target.get()) == Some("\n") {
-            flog!(DEBUG, "returning null outside of match");
+            log::debug!("returning null outside of match");
             return MotionKind::Null;
           }
         }
@@ -2130,7 +2132,7 @@ impl LineBuf {
         }) else {
           return MotionKind::Null;
         };
-        flog!(DEBUG, self.slice(start..end));
+        log::debug!("{:?}", self.slice(start..end));
 
         let target_col = if let Some(col) = self.saved_col {
           col
@@ -2143,10 +2145,10 @@ impl LineBuf {
         let Some(line) = self.slice(start..end).map(|s| s.to_string()) else {
           return MotionKind::Null;
         };
-        flog!(DEBUG, target_col);
+        log::debug!("{target_col:?}");
-        flog!(DEBUG, target_col);
+        log::debug!("{target_col:?}");
         let mut target_pos = self.grapheme_index_for_display_col(&line, target_col);
-        flog!(DEBUG, target_pos);
+        log::debug!("{target_pos:?}");
         if self.cursor.exclusive
           && line.ends_with("\n")
           && self.grapheme_at(target_pos) == Some("\n")
@@ -2171,8 +2173,8 @@ impl LineBuf {
         }) else {
           return MotionKind::Null;
         };
-        flog!(DEBUG, start, end);
+        log::debug!("{start:?}, {end:?}");
-        flog!(DEBUG, self.slice(start..end));
+        log::debug!("{:?}", self.slice(start..end));
 
         let target_col = if let Some(col) = self.saved_col {
           col
@@ -2237,9 +2239,9 @@ impl LineBuf {
 
       let has_consumed_hint = (self.cursor.exclusive && self.cursor.get() >= last_grapheme_pos)
         || (!self.cursor.exclusive && self.cursor.get() > last_grapheme_pos);
-      flog!(DEBUG, has_consumed_hint);
+      log::debug!("{has_consumed_hint:?}");
-      flog!(DEBUG, self.cursor.get());
+      log::debug!("{:?}", self.cursor.get());
-      flog!(DEBUG, last_grapheme_pos);
+      log::debug!("{last_grapheme_pos:?}");
 
       if has_consumed_hint {
         let buf_end = if self.cursor.exclusive {
@@ -2401,7 +2403,7 @@ impl LineBuf {
         } else {
           let drained = self.drain(start, end);
           self.update_graphemes();
-          flog!(DEBUG, self.cursor);
+          log::debug!("{:?}", self.cursor);
           drained
         };
         register.write_to_register(register_text);
@@ -2848,6 +2850,10 @@ impl LineBuf {
   pub fn as_str(&self) -> &str {
     &self.buffer // FIXME: this will have to be fixed up later
   }
+
+	pub fn get_hint_text(&self) -> String {
+		self.hint.clone().map(|h| h.styled(Style::BrightBlack)).unwrap_or_default()
+	}
 }
 
 impl Display for LineBuf {
@@ -2873,9 +2879,6 @@ impl Display for LineBuf {
         }
       }
     }
-    if let Some(hint) = self.hint.as_ref() {
-      full_buf.push_str(&hint.styled(Style::BrightBlack));
-    }
     write!(f, "{}", full_buf)
   }
 }

src/prompt/readline/mod.rs

@@ -6,10 +6,10 @@ use term::{get_win_size, KeyReader, Layout, LineWriter, PollReader, TermWriter};
 use vicmd::{CmdFlags, Motion, MotionCmd, RegisterName, To, Verb, VerbCmd, ViCmd};
 use vimode::{CmdReplay, ModeReport, ViInsert, ViMode, ViNormal, ViReplace, ViVisual};
 
-use crate::libsh::{
+use crate::{libsh::{
   error::{ShErrKind, ShResult},
   term::{Style, Styled},
-};
+}, parse::lex::{self, LexFlags, Tk, TkFlags, TkRule}, prompt::readline::highlight::Highlighter};
 use crate::prelude::*;
 
 pub mod history;
@@ -20,6 +20,39 @@ pub mod register;
 pub mod term;
 pub mod vicmd;
 pub mod vimode;
+pub mod highlight;
+
+pub mod markers {
+	// token-level (derived from token class)
+	pub const COMMAND: char = '\u{fdd0}';
+	pub const BUILTIN: char = '\u{fdd1}';
+	pub const ARG: char = '\u{fdd2}';
+	pub const KEYWORD: char = '\u{fdd3}';
+	pub const OPERATOR: char = '\u{fdd4}';
+	pub const REDIRECT: char = '\u{fdd5}';
+	pub const COMMENT: char = '\u{fdd6}';
+	pub const ASSIGNMENT: char = '\u{fdd7}';
+	pub const CMD_SEP: char = '\u{fde0}';
+	pub const CASE_PAT: char = '\u{fde1}';
+	pub const SUBSH: char = '\u{fde7}';
+	pub const SUBSH_END: char = '\u{fde8}';
+
+	// sub-token (needs scanning)
+	pub const VAR_SUB: char = '\u{fdda}';
+	pub const VAR_SUB_END: char = '\u{fde3}';
+	pub const CMD_SUB: char = '\u{fdd8}';
+	pub const CMD_SUB_END: char = '\u{fde4}';
+	pub const PROC_SUB: char = '\u{fdd9}';
+	pub const PROC_SUB_END: char = '\u{fde9}';
+	pub const STRING_DQ: char = '\u{fddb}';
+	pub const STRING_DQ_END: char = '\u{fde5}';
+	pub const STRING_SQ: char = '\u{fddc}';
+	pub const STRING_SQ_END: char = '\u{fde6}';
+	pub const ESCAPE: char = '\u{fddd}';
+	pub const GLOB: char = '\u{fdde}';
+
+	pub const RESET: char = '\u{fde2}';
+}
 
 /// Non-blocking readline result
 pub enum ReadlineEvent {
@@ -35,6 +68,7 @@ pub struct FernVi {
   pub reader: PollReader,
   pub writer: Box<dyn LineWriter>,
   pub prompt: String,
+	pub highlighter: Highlighter,
   pub mode: Box<dyn ViMode>,
   pub old_layout: Option<Layout>,
   pub repeat_action: Option<CmdReplay>,
@@ -50,6 +84,7 @@ impl FernVi {
 				reader: PollReader::new(),
 				writer: Box::new(TermWriter::new(STDOUT_FILENO)),
 				prompt: prompt.unwrap_or("$ ".styled(Style::Green)),
+				highlighter: Highlighter::new(),
 				mode: Box::new(ViInsert::new()),
 				old_layout: None,
 				repeat_action: None,
@@ -71,6 +106,9 @@ impl FernVi {
   /// Feed raw bytes from stdin into the reader's buffer
   pub fn feed_bytes(&mut self, bytes: &[u8]) {
 		log::info!("Feeding bytes: {:?}", bytes.iter().map(|b| *b as char).collect::<String>());
+		let test_input = "echo \"hello $USER\" | grep $(whoami)";
+		let annotated = annotate_input(test_input);
+		log::info!("Annotated test input: {:?}", annotated);
     self.reader.feed_bytes(bytes);
   }
 
@@ -84,8 +122,8 @@ impl FernVi {
     if let Some(p) = prompt {
       self.prompt = p;
     }
-    self.editor.buffer.clear();
+    self.editor = Default::default();
-    self.editor.cursor = Default::default();
+		self.mode = Box::new(ViInsert::new());
     self.old_layout = None;
     self.needs_redraw = true;
   }
@@ -101,7 +139,7 @@ impl FernVi {
 
     // Process all available keys
     while let Some(key) = self.reader.read_key()? {
-      flog!(DEBUG, key);
+      log::debug!("{key:?}");
 
       if self.should_accept_hint(&key) {
         self.editor.accept_hint();
@@ -113,7 +151,7 @@ impl FernVi {
       let Some(mut cmd) = self.mode.handle_key(key) else {
         continue;
       };
-      flog!(DEBUG, cmd);
+      log::debug!("{cmd:?}");
       cmd.alter_line_motion_if_no_verb();
 
       if self.should_grab_history(&cmd) {
@@ -165,15 +203,14 @@ impl FernVi {
     Ok(ReadlineEvent::Pending)
   }
 
-  pub fn get_layout(&mut self) -> Layout {
+  pub fn get_layout(&mut self, line: &str) -> Layout {
-    let line = self.editor.to_string();
+    log::debug!("{line:?}");
-    flog!(DEBUG, line);
     let to_cursor = self.editor.slice_to_cursor().unwrap_or_default();
     let (cols, _) = get_win_size(STDIN_FILENO);
     Layout::from_parts(/* tab_stop: */ 8, cols, &self.prompt, to_cursor, &line)
   }
   pub fn scroll_history(&mut self, cmd: ViCmd) {
-    flog!(DEBUG, "scrolling");
+    log::debug!("scrolling");
     /*
     if self.history.cursor_entry().is_some_and(|ent| ent.is_new()) {
       let constraint = SearchConstraint::new(SearchKind::Prefix, self.editor.to_string());
@@ -182,23 +219,23 @@ impl FernVi {
     */
     let count = &cmd.motion().unwrap().0;
     let motion = &cmd.motion().unwrap().1;
-    flog!(DEBUG, count, motion);
+    log::debug!("{count:?}, {motion:?}");
-    flog!(DEBUG, self.history.masked_entries());
+    log::debug!("{:?}", self.history.masked_entries());
     let entry = match motion {
       Motion::LineUpCharwise => {
         let Some(hist_entry) = self.history.scroll(-(*count as isize)) else {
           return;
         };
-        flog!(DEBUG, "found entry");
+        log::debug!("found entry");
-        flog!(DEBUG, hist_entry.command());
+        log::debug!("{:?}", hist_entry.command());
         hist_entry
       }
       Motion::LineDownCharwise => {
         let Some(hist_entry) = self.history.scroll(*count as isize) else {
           return;
         };
-        flog!(DEBUG, "found entry");
+        log::debug!("found entry");
-        flog!(DEBUG, hist_entry.command());
+        log::debug!("{:?}", hist_entry.command());
         hist_entry
       }
       _ => unreachable!(),
@@ -223,8 +260,8 @@ impl FernVi {
     self.editor = buf
   }
   pub fn should_accept_hint(&self, event: &KeyEvent) -> bool {
-    flog!(DEBUG, self.editor.cursor_at_max());
+    log::debug!("{:?}", self.editor.cursor_at_max());
-    flog!(DEBUG, self.editor.cursor);
+    log::debug!("{:?}", self.editor.cursor);
     if self.editor.cursor_at_max() && self.editor.has_hint() {
       match self.mode.report_mode() {
         ModeReport::Replace | ModeReport::Insert => {
@@ -255,15 +292,25 @@ impl FernVi {
         && !self.history.cursor_entry().is_some_and(|ent| ent.is_new()))
   }
 
+	pub fn line_text(&mut self) -> String {
+		let line = self.editor.to_string();
+		self.highlighter.load_input(&line);
+		self.highlighter.highlight();
+		let highlighted = self.highlighter.take();
+		let hint = self.editor.get_hint_text();
+		format!("{highlighted}{hint}")
+	}
+
   pub fn print_line(&mut self) -> ShResult<()> {
-    let new_layout = self.get_layout();
+		let line = self.line_text();
+    let new_layout = self.get_layout(&line);
     if let Some(layout) = self.old_layout.as_ref() {
       self.writer.clear_rows(layout)?;
     }
 
     self
       .writer
-      .redraw(&self.prompt, &self.editor, &new_layout)?;
+      .redraw(&self.prompt, &line, &new_layout)?;
 
     self.writer.flush_write(&self.mode.cursor_style())?;
 
@@ -426,3 +473,270 @@ impl FernVi {
     Ok(())
   }
 }
+
+/// Annotate a given input with helpful markers that give quick contextual syntax information
+/// Useful for syntax highlighting and completion
+pub fn annotate_input(input: &str) -> String {
+	let mut annotated = input.to_string();
+	let input = Arc::new(input.to_string());
+	let tokens: Vec<Tk> = lex::LexStream::new(input, LexFlags::LEX_UNFINISHED)
+		.flatten()
+		.collect();
+
+	for tk in tokens.into_iter().rev() {
+		annotate_token(&mut annotated, tk);
+	}
+
+	annotated
+}
+
+pub fn marker_for(class: &TkRule) -> Option<char> {
+	match class {
+		TkRule::Pipe |
+		TkRule::ErrPipe |
+		TkRule::And |
+		TkRule::Or |
+		TkRule::Bg => Some(markers::OPERATOR),
+		TkRule::Sep => Some(markers::CMD_SEP),
+		TkRule::Redir => Some(markers::REDIRECT),
+		TkRule::CasePattern => Some(markers::CASE_PAT),
+		TkRule::BraceGrpStart => todo!(),
+		TkRule::BraceGrpEnd => todo!(),
+		TkRule::Comment => todo!(),
+		TkRule::Expanded { exp: _ } |
+		TkRule::EOI |
+		TkRule::SOI |
+		TkRule::Null |
+		TkRule::Str => None,
+	}
+}
+
+pub fn annotate_token(input: &mut String, token: Tk) {
+	if token.class != TkRule::Str
+	&& let Some(marker) = marker_for(&token.class) {
+		input.insert(token.span.end, markers::RESET);
+		input.insert(token.span.start, marker);
+		return;
+	} else if token.flags.contains(TkFlags::IS_SUBSH) {
+		let token_raw = token.span.as_str();
+		if token_raw.ends_with(')') {
+			input.insert(token.span.end, markers::SUBSH_END);
+		}
+		input.insert(token.span.start, markers::SUBSH);
+		return;
+	}
+
+
+	let token_raw = token.span.as_str();
+	let mut token_chars = token_raw
+		.char_indices()
+		.peekable();
+
+	let span_start = token.span.start;
+
+	let mut in_dub_qt = false;
+	let mut in_sng_qt = false;
+	let mut cmd_sub_depth = 0;
+	let mut proc_sub_depth = 0;
+
+	let mut insertions: Vec<(usize, char)> = vec![];
+
+	if token.flags.contains(TkFlags::BUILTIN) {
+		insertions.insert(0, (span_start, markers::BUILTIN));
+	} else if token.flags.contains(TkFlags::IS_CMD) {
+		insertions.insert(0, (span_start, markers::COMMAND));
+	}
+
+	if token.flags.contains(TkFlags::KEYWORD) {
+		insertions.insert(0, (span_start, markers::KEYWORD));
+	}
+
+	if token.flags.contains(TkFlags::ASSIGN) {
+		insertions.insert(0, (span_start, markers::ASSIGNMENT));
+	}
+
+	insertions.insert(0, (token.span.end, markers::RESET)); // reset at the end of the token
+
+	while let Some((i,ch)) = token_chars.peek() {
+		let index = *i; // we have to dereference this here because rustc is a very pedantic program
+		match ch {
+			')' if cmd_sub_depth > 0 || proc_sub_depth > 0 => {
+				token_chars.next(); // consume the paren
+				if cmd_sub_depth > 0 {
+					cmd_sub_depth -= 1;
+					if cmd_sub_depth == 0 {
+						insertions.push((span_start + index + 1, markers::CMD_SUB_END));
+					}
+				} else if proc_sub_depth > 0 {
+					proc_sub_depth -= 1;
+					if proc_sub_depth == 0 {
+						insertions.push((span_start + index + 1, markers::PROC_SUB_END));
+					}
+				}
+			}
+			'$' if !in_sng_qt => {
+				let dollar_pos = index;
+				token_chars.next(); // consume the dollar
+				if let Some((_, dollar_ch)) = token_chars.peek() {
+					match dollar_ch {
+						'(' => {
+							cmd_sub_depth += 1;
+							if cmd_sub_depth == 1 {
+								// only mark top level command subs
+								insertions.push((span_start + dollar_pos, markers::CMD_SUB));
+							}
+							token_chars.next(); // consume the paren
+						}
+						'{' if cmd_sub_depth == 0 => {
+							insertions.push((span_start + dollar_pos, markers::VAR_SUB));
+							token_chars.next(); // consume the brace
+							let mut end_pos = dollar_pos + 2; // position after ${
+							while let Some((cur_i, br_ch)) = token_chars.peek() {
+								end_pos = *cur_i;
+								// TODO: implement better parameter expansion awareness here
+								// this is a little too permissive
+								if br_ch.is_ascii_alphanumeric()
+								|| *br_ch == '_'
+								|| *br_ch == '!'
+								|| *br_ch == '#'
+								|| *br_ch == '%'
+								|| *br_ch == ':'
+								|| *br_ch == '-'
+								|| *br_ch == '+'
+								|| *br_ch == '='
+								|| *br_ch == '/' // parameter expansion symbols
+								|| *br_ch == '?' {
+									token_chars.next();
+								} else if *br_ch == '}' {
+									token_chars.next(); // consume the closing brace
+									insertions.push((span_start + end_pos + 1, markers::VAR_SUB_END));
+									break;
+								} else {
+									// malformed, insert end at current position
+									insertions.push((span_start + end_pos, markers::VAR_SUB_END));
+									break;
+								}
+							}
+						}
+						_ if cmd_sub_depth == 0 && (dollar_ch.is_ascii_alphanumeric() || *dollar_ch == '_') => {
+							insertions.push((span_start + dollar_pos, markers::VAR_SUB));
+							let mut end_pos = dollar_pos + 1;
+							// consume the var name
+							while let Some((cur_i, var_ch)) = token_chars.peek() {
+								if var_ch.is_ascii_alphanumeric() || *var_ch == '_' {
+									end_pos = *cur_i + 1;
+									token_chars.next();
+								} else {
+									break;
+								}
+							}
+							insertions.push((span_start + end_pos, markers::VAR_SUB_END));
+						}
+						_ => { /* Just a plain dollar sign, no marker needed */ }
+					}
+				}
+			}
+			ch if cmd_sub_depth > 0 || proc_sub_depth > 0 => {
+				// We are inside of a command sub or process sub right now
+				// We don't mark any of this text. It will later be recursively annotated
+				// by the syntax highlighter
+				token_chars.next(); // consume the char with no special handling
+			}
+
+			'\\' if !in_sng_qt => {
+				token_chars.next(); // consume the backslash
+				if token_chars.peek().is_some() {
+					token_chars.next(); // consume the escaped char
+				}
+			}
+			'<' | '>' if !in_dub_qt && !in_sng_qt && cmd_sub_depth == 0 && proc_sub_depth == 0 => {
+				token_chars.next();
+				if let Some((_, proc_sub_ch)) = token_chars.peek()
+				&& *proc_sub_ch == '(' {
+					proc_sub_depth += 1;
+					token_chars.next(); // consume the paren
+					if proc_sub_depth == 1 {
+						insertions.push((span_start + index, markers::PROC_SUB));
+					}
+				}
+			}
+			'"' if !in_sng_qt => {
+				if in_dub_qt {
+					insertions.push((span_start + *i + 1, markers::STRING_DQ_END));
+				} else {
+					insertions.push((span_start + *i, markers::STRING_DQ));
+				}
+				in_dub_qt = !in_dub_qt;
+				token_chars.next(); // consume the quote
+			}
+			'\'' if !in_dub_qt => {
+				if in_sng_qt {
+					insertions.push((span_start + *i + 1, markers::STRING_SQ_END));
+				} else {
+					insertions.push((span_start + *i, markers::STRING_SQ));
+				}
+				in_sng_qt = !in_sng_qt;
+				token_chars.next(); // consume the quote
+			}
+			'[' if !in_dub_qt && !in_sng_qt => {
+				token_chars.next(); // consume the opening bracket
+				let start_pos = span_start + index;
+				let mut is_glob_pat = false;
+				const VALID_CHARS: &[char] = &['!', '^', '-'];
+
+				while let Some((cur_i, ch)) = token_chars.peek() {
+					if *ch == ']' {
+						is_glob_pat = true;
+						insertions.push((span_start + *cur_i + 1, markers::RESET));
+						insertions.push((span_start + *cur_i, markers::GLOB));
+						token_chars.next(); // consume the closing bracket
+						break;
+					} else if !ch.is_ascii_alphanumeric() && !VALID_CHARS.contains(ch) {
+						token_chars.next();
+						break;
+					} else {
+						token_chars.next();
+					}
+				}
+
+				if is_glob_pat {
+					insertions.push((start_pos + 1, markers::RESET));
+					insertions.push((start_pos, markers::GLOB));
+				}
+			}
+			'*' | '?' if (!in_dub_qt && !in_sng_qt) => {
+				insertions.push((span_start + *i, markers::GLOB));
+				token_chars.next(); // consume the glob char
+			}
+			_ => {
+				token_chars.next(); // consume the char with no special handling
+			}
+		}
+	}
+
+	// Sort by position descending, with priority ordering at same position:
+	// - RESET first (inserted first, ends up rightmost)
+	// - Regular markers middle
+	// - END markers last (inserted last, ends up leftmost)
+	// Result: [END][TOGGLE][RESET]
+	insertions.sort_by(|a, b| {
+		match b.0.cmp(&a.0) {
+			std::cmp::Ordering::Equal => {
+				let priority = |m: char| -> u8 {
+					match m {
+						markers::RESET => 0,
+						markers::VAR_SUB_END | markers::CMD_SUB_END => 2,
+						_ => 1,
+					}
+				};
+				priority(a.1).cmp(&priority(b.1))
+			}
+			other => other,
+		}
+	});
+
+	for (pos, marker) in insertions {
+		let pos = pos.max(0).min(input.len());
+		input.insert(pos, marker);
+	}
+}

src/prompt/readline/term.rs

@@ -179,7 +179,7 @@ pub trait KeyReader {
 
 pub trait LineWriter {
   fn clear_rows(&mut self, layout: &Layout) -> ShResult<()>;
-  fn redraw(&mut self, prompt: &str, line: &LineBuf, new_layout: &Layout) -> ShResult<()>;
+  fn redraw(&mut self, prompt: &str, line: &str, new_layout: &Layout) -> ShResult<()>;
   fn flush_write(&mut self, buf: &str) -> ShResult<()>;
 }
 
@@ -239,13 +239,13 @@ impl TermBuffer {
 impl Read for TermBuffer {
   fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
     assert!(isatty(self.tty).is_ok_and(|r| r));
-    flog!(DEBUG, "TermBuffer::read() ENTERING read syscall");
+    log::debug!("TermBuffer::read() ENTERING read syscall");
     let result = nix::unistd::read(self.tty, buf);
-    flog!(DEBUG, "TermBuffer::read() EXITED read syscall: {:?}", result);
+    log::debug!("TermBuffer::read() EXITED read syscall: {:?}", result);
     match result {
       Ok(n) => Ok(n),
       Err(Errno::EINTR) => {
-        flog!(DEBUG, "TermBuffer::read() returning EINTR");
+        log::debug!("TermBuffer::read() returning EINTR");
         Err(Errno::EINTR.into())
       }
       Err(e) => Err(std::io::Error::from_raw_os_error(e as i32)),
@@ -643,7 +643,7 @@ impl KeyReader for TermReader {
 
     loop {
       let byte = self.next_byte()?;
-      flog!(DEBUG, "read byte: {:?}", byte as char);
+      log::debug!("read byte: {:?}", byte as char);
       collected.push(byte);
 
       // If it's an escape seq, delegate to ESC sequence handler
@@ -706,7 +706,7 @@ impl Layout {
     to_cursor: &str,
     to_end: &str,
   ) -> Self {
-    flog!(DEBUG, to_cursor);
+    log::debug!("{to_cursor:?}");
     let prompt_end = Self::calc_pos(tab_stop, term_width, prompt, Pos { col: 0, row: 0 });
     let cursor = Self::calc_pos(tab_stop, term_width, to_cursor, prompt_end);
     let end = Self::calc_pos(tab_stop, term_width, to_end, prompt_end);
@@ -903,7 +903,7 @@ impl LineWriter for TermWriter {
     Ok(())
   }
 
-  fn redraw(&mut self, prompt: &str, line: &LineBuf, new_layout: &Layout) -> ShResult<()> {
+  fn redraw(&mut self, prompt: &str, line: &str, new_layout: &Layout) -> ShResult<()> {
     let err = |_| {
       ShErr::simple(
         ShErrKind::InternalErr,

src/prompt/readline/vimode.rs

@@ -348,18 +348,14 @@ impl ViNormal {
   /// End the parse and clear the pending sequence
   #[track_caller]
   pub fn quit_parse(&mut self) -> Option<ViCmd> {
-    flog!(DEBUG, std::panic::Location::caller());
+    log::debug!("{:?}", std::panic::Location::caller());
-    flog!(
+    log::warn!("exiting parse early with sequence: {}", self.pending_seq);
-      WARN,
-      "exiting parse early with sequence: {}",
-      self.pending_seq
-    );
     self.clear_cmd();
     None
   }
   pub fn try_parse(&mut self, ch: char) -> Option<ViCmd> {
     self.pending_seq.push(ch);
-    flog!(DEBUG, "parsing {}", ch);
+    log::debug!("parsing {}", ch);
     let mut chars = self.pending_seq.chars().peekable();
 
     /*
@@ -1002,8 +998,8 @@ impl ViNormal {
     };
 
     if chars.peek().is_some() {
-      flog!(WARN, "Unused characters in Vi command parse!");
+      log::warn!("Unused characters in Vi command parse!");
-      flog!(WARN, "{:?}", chars)
+      log::warn!("{:?}", chars)
     }
 
     let verb_ref = verb.as_ref().map(|v| &v.1);
@@ -1149,12 +1145,8 @@ impl ViVisual {
   /// End the parse and clear the pending sequence
   #[track_caller]
   pub fn quit_parse(&mut self) -> Option<ViCmd> {
-    flog!(DEBUG, std::panic::Location::caller());
+    log::debug!("{:?}", std::panic::Location::caller());
-    flog!(
+    log::warn!("exiting parse early with sequence: {}", self.pending_seq);
-      WARN,
-      "exiting parse early with sequence: {}",
-      self.pending_seq
-    );
     self.clear_cmd();
     None
   }
@@ -1638,7 +1630,7 @@ impl ViVisual {
           ));
         }
         ch if ch == 'i' || ch == 'a' => {
-          flog!(DEBUG, "in text_obj parse");
+          log::debug!("in text_obj parse");
           let bound = match ch {
             'i' => Bound::Inside,
             'a' => Bound::Around,
@@ -1662,7 +1654,7 @@ impl ViVisual {
             _ => return self.quit_parse(),
           };
           chars = chars_clone;
-          flog!(DEBUG, obj, bound);
+          log::debug!("{obj:?}, {bound:?}");
           break 'motion_parse Some(MotionCmd(count, Motion::TextObj(obj)));
         }
         _ => return self.quit_parse(),
@@ -1670,13 +1662,13 @@ impl ViVisual {
     };
 
     if chars.peek().is_some() {
-      flog!(WARN, "Unused characters in Vi command parse!");
+      log::warn!("Unused characters in Vi command parse!");
-      flog!(WARN, "{:?}", chars)
+      log::warn!("{:?}", chars)
     }
 
     let verb_ref = verb.as_ref().map(|v| &v.1);
     let motion_ref = motion.as_ref().map(|m| &m.1);
-    flog!(DEBUG, verb_ref, motion_ref);
+    log::debug!("{verb_ref:?}, {motion_ref:?}");
 
     match self.validate_combination(verb_ref, motion_ref) {
       CmdState::Complete => Some(ViCmd {

src/signal.rs

@@ -29,38 +29,38 @@ pub fn signals_pending() -> bool {
 
 pub fn check_signals() -> ShResult<()> {
 	if GOT_SIGINT.swap(false, Ordering::SeqCst) {
-		flog!(DEBUG, "check_signals: processing SIGINT");
+		log::debug!("check_signals: processing SIGINT");
 		interrupt()?;
 		return Err(ShErr::simple(ShErrKind::ClearReadline, ""));
 	}
 	if GOT_SIGHUP.swap(false, Ordering::SeqCst) {
-		flog!(DEBUG, "check_signals: processing SIGHUP");
+		log::debug!("check_signals: processing SIGHUP");
 		hang_up(0);
 	}
 	if GOT_SIGTSTP.swap(false, Ordering::SeqCst) {
-		flog!(DEBUG, "check_signals: processing SIGTSTP");
+		log::debug!("check_signals: processing SIGTSTP");
 		terminal_stop()?;
 	}
 	if REAPING_ENABLED.load(Ordering::SeqCst) && GOT_SIGCHLD.swap(false, Ordering::SeqCst) {
-		flog!(DEBUG, "check_signals: processing SIGCHLD (reaping enabled)");
+		log::debug!("check_signals: processing SIGCHLD (reaping enabled)");
 		wait_child()?;
 	} else if GOT_SIGCHLD.load(Ordering::SeqCst) {
-		flog!(DEBUG, "check_signals: SIGCHLD pending but reaping disabled");
+		log::debug!("check_signals: SIGCHLD pending but reaping disabled");
 	}
 	if SHOULD_QUIT.load(Ordering::SeqCst) {
 		let code = QUIT_CODE.load(Ordering::SeqCst);
-		flog!(DEBUG, "check_signals: SHOULD_QUIT set, exiting with code {}", code);
+		log::debug!("check_signals: SHOULD_QUIT set, exiting with code {}", code);
 		return Err(ShErr::simple(ShErrKind::CleanExit(code), "exit"));
 	}
 	Ok(())
 }
 
 pub fn disable_reaping() {
-	flog!(DEBUG, "disable_reaping: turning off SIGCHLD processing");
+	log::debug!("disable_reaping: turning off SIGCHLD processing");
 	REAPING_ENABLED.store(false, Ordering::SeqCst);
 }
 pub fn enable_reaping() {
-	flog!(DEBUG, "enable_reaping: turning on SIGCHLD processing");
+	log::debug!("enable_reaping: turning on SIGCHLD processing");
 	REAPING_ENABLED.store(true, Ordering::SeqCst);
 }
 
@@ -166,13 +166,13 @@ extern "C" fn handle_sigint(_: libc::c_int) {
 }
 
 pub fn interrupt() -> ShResult<()> {
-  flog!(DEBUG, "interrupt: checking for fg job to send SIGINT");
+  log::debug!("interrupt: checking for fg job to send SIGINT");
   write_jobs(|j| {
     if let Some(job) = j.get_fg_mut() {
-      flog!(DEBUG, "interrupt: sending SIGINT to fg job pgid {}", job.pgid());
+      log::debug!("interrupt: sending SIGINT to fg job pgid {}", job.pgid());
       job.killpg(Signal::SIGINT)
     } else {
-      flog!(DEBUG, "interrupt: no fg job, clearing readline");
+      log::debug!("interrupt: no fg job, clearing readline");
       Ok(())
     }
   })
@@ -188,28 +188,28 @@ extern "C" fn handle_sigchld(_: libc::c_int) {
 }
 
 pub fn wait_child() -> ShResult<()> {
-  flog!(DEBUG, "wait_child: starting reap loop");
+  log::debug!("wait_child: starting reap loop");
   let flags = WtFlag::WNOHANG | WtFlag::WSTOPPED;
   while let Ok(status) = waitpid(None, Some(flags)) {
     match status {
       WtStat::Exited(pid, code) => {
-        flog!(DEBUG, "wait_child: pid {} exited with code {}", pid, code);
+        log::debug!("wait_child: pid {} exited with code {}", pid, code);
         child_exited(pid, status)?;
       }
       WtStat::Signaled(pid, signal, _) => {
-        flog!(DEBUG, "wait_child: pid {} signaled with {:?}", pid, signal);
+        log::debug!("wait_child: pid {} signaled with {:?}", pid, signal);
         child_signaled(pid, signal)?;
       }
       WtStat::Stopped(pid, signal) => {
-        flog!(DEBUG, "wait_child: pid {} stopped with {:?}", pid, signal);
+        log::debug!("wait_child: pid {} stopped with {:?}", pid, signal);
         child_stopped(pid, signal)?;
       }
       WtStat::Continued(pid) => {
-        flog!(DEBUG, "wait_child: pid {} continued", pid);
+        log::debug!("wait_child: pid {} continued", pid);
         child_continued(pid)?;
       }
       WtStat::StillAlive => {
-        flog!(DEBUG, "wait_child: no more children to reap");
+        log::debug!("wait_child: no more children to reap");
         break;
       }
       _ => unimplemented!(),

src/state.rs

@@ -315,10 +315,10 @@ impl LogTab {
     self.aliases.get(name).cloned()
   }
   pub fn remove_alias(&mut self, name: &str) {
-    flog!(DEBUG, self.aliases);
+    log::debug!("{:?}", self.aliases);
-    flog!(DEBUG, name);
+    log::debug!("{name:?}");
     self.aliases.remove(name);
-    flog!(DEBUG, self.aliases);
+    log::debug!("{:?}", self.aliases);
   }
   pub fn clear_aliases(&mut self) {
     self.aliases.clear()
@@ -655,7 +655,7 @@ impl VarTab {
     }
   }
   pub fn var_exists(&self, var_name: &str) -> bool {
-		flog!(DEBUG, "checking existence of {}",var_name);
+		log::debug!("checking existence of {}", var_name);
     if let Ok(param) = var_name.parse::<ShellParam>() {
       return self.params.contains_key(&param);
     }

src/tests/highlight.rs

@@ -1 +0,0 @@
-