Merge pull request #248 from gcanat/wc_chars

[wc] faster character count

Author: jgarzik

text/wc.rs

@@ -11,7 +11,7 @@ use clap::Parser;
 use gettextrs::{bind_textdomain_codeset, setlocale, textdomain, LocaleCategory};
 use plib::PROJECT_NAME;
 use std::ffi::OsStr;
-use std::io::{self, BufRead, Read};
+use std::io::{self, Read};
 use std::path::PathBuf;
 
 /// wc - word, line, and byte or character count
@@ -42,7 +42,6 @@ struct CountInfo {
     words: usize,
     chars: usize,
     nl: usize,
-    was_space: bool,
 }
 
 impl CountInfo {
@@ -51,7 +50,6 @@ impl CountInfo {
             words: 0,
             chars: 0,
             nl: 0,
-            was_space: true,
         }
     }
 
@@ -124,67 +122,36 @@ fn build_display_str(args: &Args, count: &CountInfo, filename: &OsStr) -> String
     output
 }
 
-fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, table: &[bool; 256]) -> io::Result<()> {
+fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, chars_mode: bool) -> io::Result<()> {
     let mut file = plib::io::input_stream(pathname, false)?;
 
     let mut buffer = [0; plib::BUFSZ];
-    let mut was_space = count.was_space;
+    let mut was_space = true;
 
     loop {
         let n_read = file.read(&mut buffer[..])?;
         if n_read == 0 {
             break;
         }
 
-        count.chars = count.chars + n_read;
-
         let bufslice = &buffer[0..n_read];
 
+        if !chars_mode {
+            // number of bytes read
+            count.chars = count.chars + n_read;
+        } else {
+            // number of UTF-8 unicode codepoints in this slice of bytes
+            count.chars += bufslice.iter().filter(|&ch| (ch >> 6) != 0b10).count();
+        }
+
+
         for ch_u8 in bufslice {
-            let is_space = table[*ch_u8 as usize];
+            let is_space = BYTE_TABLE[*ch_u8 as usize];
             count.nl += (ch_u8 == &10) as usize;
             count.words += (!is_space && was_space) as usize;
             was_space = is_space;
         }
     }
-    count.was_space = was_space;
-
-    Ok(())
-}
-
-fn wc_file_chars(args: &Args, count: &mut CountInfo, pathname: &PathBuf) -> io::Result<()> {
-    let mut reader = plib::io::input_reader(pathname, false)?;
-
-    loop {
-        let mut buffer = String::new();
-        let n_read = reader.read_line(&mut buffer)?;
-        if n_read == 0 {
-            break;
-        }
-
-        count.nl = count.nl + 1;
-        count.chars = count.chars + n_read;
-
-        if args.words {
-            let mut in_word = false;
-
-            for ch in buffer.chars() {
-                if ch.is_whitespace() {
-                    if in_word {
-                        in_word = false;
-                        count.words = count.words + 1;
-                    }
-                } else {
-                    if !in_word {
-                        in_word = true;
-                    }
-                }
-            }
-            if in_word {
-                count.words = count.words + 1;
-            }
-        }
-    }
 
     Ok(())
 }
@@ -194,13 +161,8 @@ fn wc_file(
     chars_mode: bool,
     pathname: &PathBuf,
     count: &mut CountInfo,
-    table: &[bool; 256],
 ) -> io::Result<()> {
-    if chars_mode {
-        wc_file_chars(args, count, pathname)?;
-    } else {
-        wc_file_bytes(count, pathname, table)?;
-    }
+    wc_file_bytes(count, pathname, chars_mode)?;
 
     let output = build_display_str(args, count, pathname.as_os_str());
 
@@ -236,7 +198,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     if args.files.is_empty() {
         let mut count = CountInfo::new();
 
-        if let Err(e) = wc_file(&args, chars_mode, &PathBuf::new(), &mut count, &BYTE_TABLE) {
+        if let Err(e) = wc_file(&args, chars_mode, &PathBuf::new(), &mut count) {
             exit_code = 1;
             eprintln!("stdin: {}", e);
         }
@@ -246,7 +208,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         for filename in &args.files {
             let mut count = CountInfo::new();
 
-            if let Err(e) = wc_file(&args, chars_mode, filename, &mut count, &BYTE_TABLE) {
+            if let Err(e) = wc_file(&args, chars_mode, filename, &mut count) {
                 exit_code = 1;
                 eprintln!("{}: {}", filename.display(), e);
             }