[wc] count chars from bytes

Author: gcanat

text/wc.rs

@@ -11,7 +11,7 @@ use clap::Parser;
 use gettextrs::{bind_textdomain_codeset, setlocale, textdomain, LocaleCategory};
 use plib::PROJECT_NAME;
 use std::ffi::OsStr;
-use std::io::{self, BufRead, Read};
+use std::io::{self, Read};
 use std::path::PathBuf;
 
 /// wc - word, line, and byte or character count
@@ -122,7 +122,7 @@ fn build_display_str(args: &Args, count: &CountInfo, filename: &OsStr) -> String
     output
 }
 
-fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, table: &[bool; 256]) -> io::Result<()> {
+fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, table: &[bool; 256], chars_mode: bool) -> io::Result<()> {
     let mut file = plib::io::input_stream(pathname, false)?;
 
     let mut buffer = [0; plib::BUFSZ];
@@ -134,10 +134,17 @@ fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, table: &[bool; 256])
             break;
         }
 
-        count.chars = count.chars + n_read;
-
         let bufslice = &buffer[0..n_read];
 
+        if !chars_mode {
+            // number of bytes read
+            count.chars = count.chars + n_read;
+        } else {
+            // number of UTF-8 unicode codepoints in this slice of bytes
+            count.chars += bufslice.iter().filter(|&ch| (ch >> 6) != 0b10).count();
+        }
+
+
         for ch_u8 in bufslice {
             let is_space = table[*ch_u8 as usize];
             count.nl += (ch_u8 == &10) as usize;
@@ -149,55 +156,14 @@ fn wc_file_bytes(count: &mut CountInfo, pathname: &PathBuf, table: &[bool; 256])
     Ok(())
 }
 
-fn wc_file_chars(args: &Args, count: &mut CountInfo, pathname: &PathBuf) -> io::Result<()> {
-    let mut reader = plib::io::input_reader(pathname, false)?;
-
-    loop {
-        let mut buffer = String::new();
-        let n_read = reader.read_line(&mut buffer)?;
-        if n_read == 0 {
-            break;
-        }
-
-        count.nl = count.nl + 1;
-        count.chars = count.chars + n_read;
-
-        if args.words {
-            let mut in_word = false;
-
-            for ch in buffer.chars() {
-                if ch.is_whitespace() {
-                    if in_word {
-                        in_word = false;
-                        count.words = count.words + 1;
-                    }
-                } else {
-                    if !in_word {
-                        in_word = true;
-                    }
-                }
-            }
-            if in_word {
-                count.words = count.words + 1;
-            }
-        }
-    }
-
-    Ok(())
-}
-
 fn wc_file(
     args: &Args,
     chars_mode: bool,
     pathname: &PathBuf,
     count: &mut CountInfo,
     table: &[bool; 256],
 ) -> io::Result<()> {
-    if chars_mode {
-        wc_file_chars(args, count, pathname)?;
-    } else {
-        wc_file_bytes(count, pathname, table)?;
-    }
+    wc_file_bytes(count, pathname, table, chars_mode)?;
 
     let output = build_display_str(args, count, pathname.as_os_str());