1use std::collections::{BTreeMap, BTreeSet};
23use serde::{Deserialize, Serialize};
45use crate::CityName;
67/// A list of all canonical data files for A/B Street that're uploaded somewhere. The file formats
8/// are tied to the latest version of the git repo. Players use the updater crate to sync these
9/// files with local copies.
10#[derive(Serialize, Deserialize)]
11pub struct Manifest {
12/// Keyed by path, starting with "data/"
13pub entries: BTreeMap<String, Entry>,
14}
1516/// A single file
17#[derive(Serialize, Deserialize)]
18pub struct Entry {
19/// md5sum of the file
20pub checksum: String,
21/// Uncompressed size in bytes. Because we have some massive files more than 2^32 bytes
22 /// described by this, explicitly use u64 instead of usize, so wasm doesn't break.
23pub uncompressed_size_bytes: u64,
24/// Compressed size in bytes
25pub compressed_size_bytes: u64,
26}
2728impl Manifest {
29#[cfg(not(target_arch = "wasm32"))]
30pub fn load() -> Manifest {
31crate::maybe_read_json(
32crate::path("MANIFEST.json"),
33&mut abstutil::Timer::throwaway(),
34 )
35 .unwrap()
36 }
3738#[cfg(target_arch = "wasm32")]
39pub fn load() -> Manifest {
40 abstutil::from_json(&include_bytes!("../../data/MANIFEST.json").to_vec()).unwrap()
41 }
4243/// Removes entries from the Manifest to match the DataPacks that should exist locally.
44pub fn filter(mut self, data_packs: DataPacks) -> Manifest {
45let mut remove = Vec::new();
46for path in self.entries.keys() {
47if path.starts_with("data/system/extra_fonts") {
48// Always grab all of these
49continue;
50 }
51// If the user has opted into any input data at all, we want to grab some of the shared
52 // input files.
53if path.starts_with("data/input/shared") {
54// But some of the files are large, so of course we hardcode some more overrides
55 // here. Maybe some of this data should be scoped to a country, not a city.
56if path.ends_with("Road Safety Data - Accidents 2019.csv")
57 || path.ends_with("wu03ew_v2.csv")
58 || path.ends_with("zones_core.geojson")
59 {
60if data_packs.input.iter().any(|x| x.starts_with("gb/")) {
61continue;
62 }
63 } else if path.ends_with("kc_2016_lidar.tif") {
64if data_packs.input.contains("us/seattle") {
65continue;
66 }
67 } else if !data_packs.input.is_empty() {
68continue;
69 }
70 }
7172let parts = path.split('/').collect::<Vec<_>>();
73let mut data_pack = format!("{}/{}", parts[2], parts[3]);
74if Manifest::is_file_part_of_huge_seattle(path) {
75 data_pack = "us/huge_seattle".to_string();
76 }
77if parts[1] == "input" {
78if data_packs.input.contains(&data_pack) {
79continue;
80 }
81 } else if parts[1] == "system" {
82if data_packs.runtime.contains(&data_pack) {
83continue;
84 }
85 } else {
86panic!("Wait what's {}", path);
87 }
88 remove.push(path.clone());
89 }
90for path in remove {
91self.entries.remove(&path).unwrap();
92 }
93self
94}
9596/// Because there are so many Seattle maps and they get downloaded in one data pack on native,
97 /// managing the total file size is important. The "us/seattle" data pack only contains small
98 /// maps; the "us/huge_seattle" pack has the rest. This returns true for files belonging to
99 /// "us/huge_seattle".
100pub fn is_file_part_of_huge_seattle(path: &str) -> bool {
101let path = path
102 .strip_prefix(&crate::path(""))
103 .or_else(|| path.strip_prefix("data/"))
104 .unwrap_or(path);
105let name = if let Some(x) = path.strip_prefix("system/us/seattle/maps/") {
106 x.strip_suffix(".bin").unwrap()
107 } else if let Some(x) = path.strip_prefix("system/us/seattle/scenarios/") {
108 x.split('/').next().unwrap()
109 } else if let Some(x) = path.strip_prefix("system/us/seattle/prebaked_results/") {
110 x.split('/').next().unwrap()
111 } else {
112return false;
113 };
114 name == "huge_seattle"
115|| name == "north_seattle"
116|| name == "south_seattle"
117|| name == "west_seattle"
118}
119120/// If an entry's path is system data, return the city.
121pub fn path_to_city(path: &str) -> Option<CityName> {
122let parts = path.split('/').collect::<Vec<_>>();
123if parts[1] == "system" {
124if parts[2] == "assets"
125|| parts[2] == "extra_fonts"
126|| parts[2] == "ltn_proposals"
127|| parts[2] == "proposals"
128|| parts[2] == "study_areas"
129{
130return None;
131 }
132return Some(CityName::new(parts[2], parts[3]));
133 }
134None
135}
136137/// Look up an entry.
138pub fn get_entry(&self, path: &str) -> Option<&Entry> {
139let path = path.strip_prefix(&crate::path("")).unwrap_or(path);
140self.entries.get(&format!("data/{}", path))
141 }
142}
143144/// Player-chosen groups of files to opt into downloading
145#[derive(Serialize, Deserialize)]
146pub struct DataPacks {
147/// A list of cities to download for using in A/B Street. Expressed the same as
148 /// `CityName::to_path`, like "gb/london".
149pub runtime: BTreeSet<String>,
150/// A list of cities to download for running the map importer.
151pub input: BTreeSet<String>,
152}
153154impl DataPacks {
155/// Load the player's config for what files to download, or create the config.
156#[cfg(not(target_arch = "wasm32"))]
157pub fn load_or_create() -> DataPacks {
158let path = crate::path_player("data.json");
159match crate::maybe_read_json::<DataPacks>(path.clone(), &mut abstutil::Timer::throwaway()) {
160Ok(cfg) => cfg,
161Err(err) => {
162warn!("player/data.json invalid, assuming defaults: {}", err);
163let mut cfg = DataPacks {
164 runtime: BTreeSet::new(),
165 input: BTreeSet::new(),
166 };
167 cfg.runtime.insert("us/seattle".to_string());
168crate::write_json(path, &cfg);
169 cfg
170 }
171 }
172 }
173174/// Saves the player's config for what files to download.
175#[cfg(not(target_arch = "wasm32"))]
176pub fn save(&self) {
177crate::write_json(crate::path_player("data.json"), self);
178 }
179180/// Fill out all data packs based on the local manifest.
181pub fn all_data_packs() -> DataPacks {
182let mut data_packs = DataPacks {
183 runtime: BTreeSet::new(),
184 input: BTreeSet::new(),
185 };
186for path in Manifest::load().entries.keys() {
187if path.starts_with("data/system/extra_fonts") || path.starts_with("data/input/shared")
188 {
189continue;
190 }
191let parts = path.split('/').collect::<Vec<_>>();
192let mut city = format!("{}/{}", parts[2], parts[3]);
193if Manifest::is_file_part_of_huge_seattle(path) {
194 city = "us/huge_seattle".to_string();
195 }
196if parts[1] == "input" {
197 data_packs.input.insert(city);
198 } else if parts[1] == "system" {
199 data_packs.runtime.insert(city);
200 }
201 }
202 data_packs
203 }
204}