abstio/
io_native.rs

1//! Normal file IO using the filesystem
2
3use std::io::{stdout, BufReader, BufWriter, Read, Write};
4use std::path::Path;
5
6use anyhow::{Context, Result};
7use fs_err::File;
8use instant::Instant;
9use serde::de::DeserializeOwned;
10use serde::Serialize;
11
12use abstutil::time::{clear_current_line, prettyprint_time};
13use abstutil::{elapsed_seconds, prettyprint_usize, to_json, Timer, PROGRESS_FREQUENCY_SECONDS};
14
15pub use crate::io::*;
16
17pub fn file_exists<I: AsRef<str>>(path: I) -> bool {
18    Path::new(path.as_ref()).exists()
19}
20
21/// Returns full paths
22pub fn list_dir(path: String) -> Vec<String> {
23    let mut files: Vec<String> = Vec::new();
24    match fs_err::read_dir(&path) {
25        Ok(iter) => {
26            for entry in iter {
27                files.push(entry.unwrap().path().to_str().unwrap().to_string());
28            }
29        }
30        Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {}
31        Err(e) => panic!("Couldn't read_dir {:?}: {}", path, e),
32    };
33    files.sort();
34    files
35}
36
37pub fn slurp_file<I: AsRef<str>>(path: I) -> Result<Vec<u8>> {
38    inner_slurp_file(path.as_ref())
39}
40fn inner_slurp_file(path: &str) -> Result<Vec<u8>> {
41    || -> Result<Vec<u8>> {
42        let mut file = File::open(path)?;
43        let mut buffer = Vec::new();
44        file.read_to_end(&mut buffer)?;
45        Ok(buffer)
46    }()
47    .with_context(|| path.to_string())
48}
49
50pub fn maybe_read_binary<T: DeserializeOwned>(path: String, timer: &mut Timer) -> Result<T> {
51    if !path.ends_with(".bin") {
52        panic!("read_binary needs {} to end with .bin", path);
53    }
54
55    timer.read_file(&path)?;
56    bincode::deserialize_from(timer).map_err(|err| err.into())
57}
58
59// TODO Idea: Have a wrapper type DotJSON(...) and DotBin(...) to distinguish raw path strings
60fn maybe_write_json<T: Serialize>(path: &str, obj: &T) -> Result<()> {
61    if !path.ends_with(".json") {
62        panic!("write_json needs {} to end with .json", path);
63    }
64    fs_err::create_dir_all(std::path::Path::new(path).parent().unwrap())
65        .expect("Creating parent dir failed");
66
67    let mut file = File::create(path)?;
68    file.write_all(to_json(obj).as_bytes())?;
69    Ok(())
70}
71
72pub fn write_json<T: Serialize>(path: String, obj: &T) {
73    if let Err(err) = maybe_write_json(&path, obj) {
74        panic!("Can't write_json({}): {}", path, err);
75    }
76    info!("Wrote {}", path);
77}
78
79fn maybe_write_binary<T: Serialize>(path: &str, obj: &T) -> Result<()> {
80    if !path.ends_with(".bin") {
81        panic!("write_binary needs {} to end with .bin", path);
82    }
83
84    fs_err::create_dir_all(std::path::Path::new(path).parent().unwrap())
85        .expect("Creating parent dir failed");
86
87    let file = BufWriter::new(File::create(path)?);
88    bincode::serialize_into(file, obj).map_err(|err| err.into())
89}
90
91pub fn write_binary<T: Serialize>(path: String, obj: &T) {
92    if let Err(err) = maybe_write_binary(&path, obj) {
93        panic!("Can't write_binary({}): {}", path, err);
94    }
95    info!("Wrote {}", path);
96}
97
98pub fn write_raw(path: String, bytes: &[u8]) -> Result<()> {
99    fs_err::create_dir_all(std::path::Path::new(&path).parent().unwrap())?;
100
101    let mut file = BufWriter::new(File::create(path)?);
102    file.write_all(bytes)?;
103    Ok(())
104}
105
106/// Idempotent
107pub fn delete_file<I: AsRef<str>>(path: I) {
108    let path = path.as_ref();
109    if fs_err::remove_file(path).is_ok() {
110        info!("Deleted {}", path);
111    } else {
112        info!("{} doesn't exist, so not deleting it", path);
113    }
114}
115
116// TODO I'd like to get rid of this and just use Timer.read_file, but external libraries consume
117// the reader. :\
118pub struct FileWithProgress {
119    inner: BufReader<File>,
120
121    path: String,
122    processed_bytes: usize,
123    total_bytes: usize,
124    started_at: Instant,
125    last_printed_at: Instant,
126}
127
128impl FileWithProgress {
129    /// Also hands back a callback that'll add the final result to the timer. The caller must run
130    /// it.
131    // TODO It's really a FnOnce, but I don't understand the compiler error.
132    pub fn new(path: &str) -> Result<(FileWithProgress, Box<dyn Fn(&mut Timer)>)> {
133        let file = File::open(path)?;
134        let path_copy = path.to_string();
135        let total_bytes = file.metadata()?.len() as usize;
136        let start = Instant::now();
137        Ok((
138            FileWithProgress {
139                inner: BufReader::new(file),
140                path: path.to_string(),
141                processed_bytes: 0,
142                total_bytes,
143                started_at: start,
144                last_printed_at: start,
145            },
146            Box::new(move |ref mut timer| {
147                let elapsed = elapsed_seconds(start);
148                timer.add_result(
149                    elapsed,
150                    format!(
151                        "Reading {} ({} MB)... {}",
152                        path_copy,
153                        prettyprint_usize(total_bytes / 1024 / 1024),
154                        prettyprint_time(elapsed)
155                    ),
156                );
157            }),
158        ))
159    }
160}
161
162impl Read for FileWithProgress {
163    fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
164        let bytes = self.inner.read(buf)?;
165        self.processed_bytes += bytes;
166        if self.processed_bytes > self.total_bytes {
167            panic!(
168                "{} is too many bytes read from {}",
169                prettyprint_usize(self.processed_bytes),
170                self.path
171            );
172        }
173
174        let done = self.processed_bytes == self.total_bytes && bytes == 0;
175        if elapsed_seconds(self.last_printed_at) >= PROGRESS_FREQUENCY_SECONDS || done {
176            self.last_printed_at = Instant::now();
177            clear_current_line();
178            if done {
179                // TODO Not seeing this case happen!
180                println!(
181                    "Read {} ({})... {}",
182                    self.path,
183                    prettyprint_usize(self.total_bytes / 1024 / 1024),
184                    prettyprint_time(elapsed_seconds(self.started_at))
185                );
186            } else {
187                print!(
188                    "Reading {}: {}/{} MB... {}",
189                    self.path,
190                    prettyprint_usize(self.processed_bytes / 1024 / 1024),
191                    prettyprint_usize(self.total_bytes / 1024 / 1024),
192                    prettyprint_time(elapsed_seconds(self.started_at))
193                );
194                stdout().flush().unwrap();
195            }
196        }
197
198        Ok(bytes)
199    }
200}
201
202/// Returns path on success
203pub fn write_file(path: String, contents: String) -> Result<String> {
204    let mut file = File::create(&path)?;
205    write!(file, "{}", contents)?;
206    Ok(path)
207}