<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/lexer.rs`."><title>lexer.rs - source</title><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/FiraSans-Regular-018c141bf0843ffd.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/FiraSans-Medium-8f9a781e4970d388.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/SourceCodePro-Regular-562dcc5011b6de7d.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/SourceSerif4-Bold-a2c9cd1067f8b328.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../static.files/SourceCodePro-Semibold-d899c5a5c4aeb14a.ttf.woff2"><link rel="stylesheet" href="../../static.files/normalize-76eba96aa4d2e634.css"><link rel="stylesheet" href="../../static.files/rustdoc-cb6f1f67f1bcd037.css" id="mainThemeStyle"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="llvm_kaleidoscope_rs" data-themes="" data-resource-suffix="" data-rustdoc-version="1.73.0 (cc66ad468 2023-10-03)" data-channel="1.73.0" data-search-js="search-6dfdfced5eff6596.js" data-settings-js="settings-de11bff964e9d4e5.js" data-settings-css="settings-8c76f75bfb6bd192.css" data-theme-light-css="light-1596385f77d47ef2.css" data-theme-dark-css="dark-0a43001d3fc2282c.css" data-theme-ayu-css="ayu-fd19013d6ce078bf.css" ><script src="../../static.files/storage-db41da1a38ea3cb8.js"></script><script defer src="../../static.files/src-script-3280b574d94e47b4.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-0795b7d26be81095.js"></script><noscript><link rel="stylesheet" media="(prefers-color-scheme:light)" href="../../static.files/light-1596385f77d47ef2.css"><link rel="stylesheet" media="(prefers-color-scheme:dark)" href="../../static.files/dark-0a43001d3fc2282c.css"><link rel="stylesheet" href="../../static.files/noscript-cffde32267a19fd6.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-16x16-8b506e7a72182f1c.png"><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-422f7d1d52889060.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-2c020d218678b618.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"></nav><main><nav class="sub"><a class="sub-logo-container" href="../../llvm_kaleidoscope_rs/index.html"><img class="rust-logo" src="../../static.files/rust-logo-151179464ae7ed46.svg" alt="logo"></a><form class="search-form"><span></span><input class="search-input" name="search" aria-label="Run search in the documentation" autocomplete="off" spellcheck="false" placeholder="Click or press ‘S’ to search, ‘?’ for more options…" type="search"><div id="help-button" title="help" tabindex="-1"><a href="../../help.html">?</a></div><div id="settings-menu" tabindex="-1"><a href="../../settings.html" title="settings"><img width="22" height="22" alt="Change settings" src="../../static.files/wheel-7b819b6101059cd0.svg"></a></div></form></nav><section id="main-content" class="content"><div class="example-wrap"><div data-nosnippet><pre class="src-line-numbers"><a href="#1" id="1">1</a>
<a href="#2" id="2">2</a>
<a href="#3" id="3">3</a>
<a href="#4" id="4">4</a>
<a href="#5" id="5">5</a>
<a href="#6" id="6">6</a>
<a href="#7" id="7">7</a>
<a href="#8" id="8">8</a>
<a href="#9" id="9">9</a>
<a href="#10" id="10">10</a>
<a href="#11" id="11">11</a>
<a href="#12" id="12">12</a>
<a href="#13" id="13">13</a>
<a href="#14" id="14">14</a>
<a href="#15" id="15">15</a>
<a href="#16" id="16">16</a>
<a href="#17" id="17">17</a>
<a href="#18" id="18">18</a>
<a href="#19" id="19">19</a>
<a href="#20" id="20">20</a>
<a href="#21" id="21">21</a>
<a href="#22" id="22">22</a>
<a href="#23" id="23">23</a>
<a href="#24" id="24">24</a>
<a href="#25" id="25">25</a>
<a href="#26" id="26">26</a>
<a href="#27" id="27">27</a>
<a href="#28" id="28">28</a>
<a href="#29" id="29">29</a>
<a href="#30" id="30">30</a>
<a href="#31" id="31">31</a>
<a href="#32" id="32">32</a>
<a href="#33" id="33">33</a>
<a href="#34" id="34">34</a>
<a href="#35" id="35">35</a>
<a href="#36" id="36">36</a>
<a href="#37" id="37">37</a>
<a href="#38" id="38">38</a>
<a href="#39" id="39">39</a>
<a href="#40" id="40">40</a>
<a href="#41" id="41">41</a>
<a href="#42" id="42">42</a>
<a href="#43" id="43">43</a>
<a href="#44" id="44">44</a>
<a href="#45" id="45">45</a>
<a href="#46" id="46">46</a>
<a href="#47" id="47">47</a>
<a href="#48" id="48">48</a>
<a href="#49" id="49">49</a>
<a href="#50" id="50">50</a>
<a href="#51" id="51">51</a>
<a href="#52" id="52">52</a>
<a href="#53" id="53">53</a>
<a href="#54" id="54">54</a>
<a href="#55" id="55">55</a>
<a href="#56" id="56">56</a>
<a href="#57" id="57">57</a>
<a href="#58" id="58">58</a>
<a href="#59" id="59">59</a>
<a href="#60" id="60">60</a>
<a href="#61" id="61">61</a>
<a href="#62" id="62">62</a>
<a href="#63" id="63">63</a>
<a href="#64" id="64">64</a>
<a href="#65" id="65">65</a>
<a href="#66" id="66">66</a>
<a href="#67" id="67">67</a>
<a href="#68" id="68">68</a>
<a href="#69" id="69">69</a>
<a href="#70" id="70">70</a>
<a href="#71" id="71">71</a>
<a href="#72" id="72">72</a>
<a href="#73" id="73">73</a>
<a href="#74" id="74">74</a>
<a href="#75" id="75">75</a>
<a href="#76" id="76">76</a>
<a href="#77" id="77">77</a>
<a href="#78" id="78">78</a>
<a href="#79" id="79">79</a>
<a href="#80" id="80">80</a>
<a href="#81" id="81">81</a>
<a href="#82" id="82">82</a>
<a href="#83" id="83">83</a>
<a href="#84" id="84">84</a>
<a href="#85" id="85">85</a>
<a href="#86" id="86">86</a>
<a href="#87" id="87">87</a>
<a href="#88" id="88">88</a>
<a href="#89" id="89">89</a>
<a href="#90" id="90">90</a>
<a href="#91" id="91">91</a>
<a href="#92" id="92">92</a>
<a href="#93" id="93">93</a>
<a href="#94" id="94">94</a>
<a href="#95" id="95">95</a>
<a href="#96" id="96">96</a>
<a href="#97" id="97">97</a>
<a href="#98" id="98">98</a>
<a href="#99" id="99">99</a>
<a href="#100" id="100">100</a>
<a href="#101" id="101">101</a>
<a href="#102" id="102">102</a>
<a href="#103" id="103">103</a>
<a href="#104" id="104">104</a>
<a href="#105" id="105">105</a>
<a href="#106" id="106">106</a>
<a href="#107" id="107">107</a>
<a href="#108" id="108">108</a>
<a href="#109" id="109">109</a>
<a href="#110" id="110">110</a>
<a href="#111" id="111">111</a>
<a href="#112" id="112">112</a>
<a href="#113" id="113">113</a>
<a href="#114" id="114">114</a>
<a href="#115" id="115">115</a>
<a href="#116" id="116">116</a>
<a href="#117" id="117">117</a>
<a href="#118" id="118">118</a>
<a href="#119" id="119">119</a>
<a href="#120" id="120">120</a>
<a href="#121" id="121">121</a>
<a href="#122" id="122">122</a>
<a href="#123" id="123">123</a>
<a href="#124" id="124">124</a>
<a href="#125" id="125">125</a>
<a href="#126" id="126">126</a>
<a href="#127" id="127">127</a>
<a href="#128" id="128">128</a>
<a href="#129" id="129">129</a>
<a href="#130" id="130">130</a>
<a href="#131" id="131">131</a>
<a href="#132" id="132">132</a>
<a href="#133" id="133">133</a>
<a href="#134" id="134">134</a>
<a href="#135" id="135">135</a>
<a href="#136" id="136">136</a>
<a href="#137" id="137">137</a>
<a href="#138" id="138">138</a>
<a href="#139" id="139">139</a>
<a href="#140" id="140">140</a>
<a href="#141" id="141">141</a>
<a href="#142" id="142">142</a>
<a href="#143" id="143">143</a>
<a href="#144" id="144">144</a>
<a href="#145" id="145">145</a>
<a href="#146" id="146">146</a>
<a href="#147" id="147">147</a>
<a href="#148" id="148">148</a>
<a href="#149" id="149">149</a>
<a href="#150" id="150">150</a>
<a href="#151" id="151">151</a>
<a href="#152" id="152">152</a>
<a href="#153" id="153">153</a>
<a href="#154" id="154">154</a>
<a href="#155" id="155">155</a>
<a href="#156" id="156">156</a>
<a href="#157" id="157">157</a>
<a href="#158" id="158">158</a>
<a href="#159" id="159">159</a>
<a href="#160" id="160">160</a>
<a href="#161" id="161">161</a>
<a href="#162" id="162">162</a>
<a href="#163" id="163">163</a>
<a href="#164" id="164">164</a>
<a href="#165" id="165">165</a>
<a href="#166" id="166">166</a>
<a href="#167" id="167">167</a>
<a href="#168" id="168">168</a>
<a href="#169" id="169">169</a>
<a href="#170" id="170">170</a>
<a href="#171" id="171">171</a>
<a href="#172" id="172">172</a>
<a href="#173" id="173">173</a>
<a href="#174" id="174">174</a>
<a href="#175" id="175">175</a>
<a href="#176" id="176">176</a>
<a href="#177" id="177">177</a>
<a href="#178" id="178">178</a>
<a href="#179" id="179">179</a>
<a href="#180" id="180">180</a>
<a href="#181" id="181">181</a>
<a href="#182" id="182">182</a>
<a href="#183" id="183">183</a>
<a href="#184" id="184">184</a>
<a href="#185" id="185">185</a>
<a href="#186" id="186">186</a>
<a href="#187" id="187">187</a>
<a href="#188" id="188">188</a>
<a href="#189" id="189">189</a>
<a href="#190" id="190">190</a>
<a href="#191" id="191">191</a>
<a href="#192" id="192">192</a>
<a href="#193" id="193">193</a>
<a href="#194" id="194">194</a>
<a href="#195" id="195">195</a>
<a href="#196" id="196">196</a>
<a href="#197" id="197">197</a>
<a href="#198" id="198">198</a>
<a href="#199" id="199">199</a>
<a href="#200" id="200">200</a>
<a href="#201" id="201">201</a>
<a href="#202" id="202">202</a>
<a href="#203" id="203">203</a>
<a href="#204" id="204">204</a>
<a href="#205" id="205">205</a>
<a href="#206" id="206">206</a>
</pre></div><pre class="rust"><code><span class="attr">#[derive(Debug, PartialEq)]
</span><span class="kw">pub enum </span>Token {
Eof,
Def,
Extern,
Identifier(String),
Number(f64),
Char(char),
If,
Then,
Else,
For,
In,
}
<span class="kw">pub struct </span>Lexer<I>
<span class="kw">where
</span>I: Iterator<Item = char>,
{
input: I,
last_char: <span class="prelude-ty">Option</span><char>,
}
<span class="kw">impl</span><I> Lexer<I>
<span class="kw">where
</span>I: Iterator<Item = char>,
{
<span class="kw">pub fn </span>new(<span class="kw-2">mut </span>input: I) -> Lexer<I> {
<span class="kw">let </span>last_char = input.next();
Lexer { input, last_char }
}
<span class="kw">fn </span>step(<span class="kw-2">&mut </span><span class="self">self</span>) -> <span class="prelude-ty">Option</span><char> {
<span class="self">self</span>.last_char = <span class="self">self</span>.input.next();
<span class="self">self</span>.last_char
}
<span class="doccomment">/// Lex and return the next token.
///
/// Implement `int gettok();` from the tutorial.
</span><span class="kw">pub fn </span>gettok(<span class="kw-2">&mut </span><span class="self">self</span>) -> Token {
<span class="comment">// Eat up whitespaces.
</span><span class="kw">while </span><span class="macro">matches!</span>(<span class="self">self</span>.last_char, <span class="prelude-val">Some</span>(c) <span class="kw">if </span>c.is_ascii_whitespace()) {
<span class="self">self</span>.step();
}
<span class="comment">// Unpack last char or return EOF.
</span><span class="kw">let </span>last_char = <span class="kw">if let </span><span class="prelude-val">Some</span>(c) = <span class="self">self</span>.last_char {
c
} <span class="kw">else </span>{
<span class="kw">return </span>Token::Eof;
};
<span class="comment">// Identifier: [a-zA-Z][a-zA-Z0-9]*
</span><span class="kw">if </span>last_char.is_ascii_alphabetic() {
<span class="kw">let </span><span class="kw-2">mut </span>ident = String::new();
ident.push(last_char);
<span class="kw">while let </span><span class="prelude-val">Some</span>(c) = <span class="self">self</span>.step() {
<span class="kw">if </span>c.is_ascii_alphanumeric() {
ident.push(c)
} <span class="kw">else </span>{
<span class="kw">break</span>;
}
}
<span class="kw">match </span>ident.as_ref() {
<span class="string">"def" </span>=> <span class="kw">return </span>Token::Def,
<span class="string">"extern" </span>=> <span class="kw">return </span>Token::Extern,
<span class="string">"if" </span>=> <span class="kw">return </span>Token::If,
<span class="string">"then" </span>=> <span class="kw">return </span>Token::Then,
<span class="string">"else" </span>=> <span class="kw">return </span>Token::Else,
<span class="string">"for" </span>=> <span class="kw">return </span>Token::For,
<span class="string">"in" </span>=> <span class="kw">return </span>Token::In,
<span class="kw">_ </span>=> {}
}
<span class="kw">return </span>Token::Identifier(ident);
}
<span class="comment">// Number: [0-9.]+
</span><span class="kw">if </span>last_char.is_ascii_digit() || last_char == <span class="string">'.' </span>{
<span class="kw">let </span><span class="kw-2">mut </span>num = String::new();
num.push(last_char);
<span class="kw">while let </span><span class="prelude-val">Some</span>(c) = <span class="self">self</span>.step() {
<span class="kw">if </span>c.is_ascii_digit() || c == <span class="string">'.' </span>{
num.push(c)
} <span class="kw">else </span>{
<span class="kw">break</span>;
}
}
<span class="kw">let </span>num: f64 = num.parse().unwrap_or_default();
<span class="kw">return </span>Token::Number(num);
}
<span class="comment">// Eat up comment.
</span><span class="kw">if </span>last_char == <span class="string">'#' </span>{
<span class="kw">loop </span>{
<span class="kw">match </span><span class="self">self</span>.step() {
<span class="prelude-val">Some</span>(c) <span class="kw">if </span>c == <span class="string">'\r' </span>|| c == <span class="string">'\n' </span>=> <span class="kw">return </span><span class="self">self</span>.gettok(),
<span class="prelude-val">None </span>=> <span class="kw">return </span>Token::Eof,
<span class="kw">_ </span>=> { <span class="comment">/* consume comment */ </span>}
}
}
}
<span class="comment">// Advance last char and return currently last char.
</span><span class="self">self</span>.step();
Token::Char(last_char)
}
}
<span class="attr">#[cfg(test)]
</span><span class="kw">mod </span>test {
<span class="kw">use super</span>::{Lexer, Token};
<span class="attr">#[test]
</span><span class="kw">fn </span>test_identifier() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"a b c"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"a"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"b"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"c"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_keyword() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"def extern"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Def, lex.gettok());
<span class="macro">assert_eq!</span>(Token::Extern, lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_number() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"12.34"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Number(<span class="number">12.34f64</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">" 1.0 2.0 3.0"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Number(<span class="number">1.0f64</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Number(<span class="number">2.0f64</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Number(<span class="number">3.0f64</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"12.34.56"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Number(<span class="number">0f64</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_comment() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"# some comment"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"abc # some comment \n xyz"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"abc"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"xyz"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_chars() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"a+b-c"</span>.chars());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"a"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Char(<span class="string">'+'</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"b"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Char(<span class="string">'-'</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"c"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_whitespaces() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">" +a b c! "</span>.chars());
<span class="macro">assert_eq!</span>(Token::Char(<span class="string">'+'</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"a"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"b"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"c"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Char(<span class="string">'!'</span>), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"\n a \n\r b \r \n c \r\r \n "</span>.chars());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"a"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"b"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Identifier(<span class="string">"c"</span>.into()), lex.gettok());
<span class="macro">assert_eq!</span>(Token::Eof, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_ite() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"if then else"</span>.chars());
<span class="macro">assert_eq!</span>(Token::If, lex.gettok());
<span class="macro">assert_eq!</span>(Token::Then, lex.gettok());
<span class="macro">assert_eq!</span>(Token::Else, lex.gettok());
}
<span class="attr">#[test]
</span><span class="kw">fn </span>test_for() {
<span class="kw">let </span><span class="kw-2">mut </span>lex = Lexer::new(<span class="string">"for in"</span>.chars());
<span class="macro">assert_eq!</span>(Token::For, lex.gettok());
<span class="macro">assert_eq!</span>(Token::In, lex.gettok());
}
}
</code></pre></div></section></main></body></html>
