importer/
uk.rs

1use std::collections::HashMap;
2
3use anyhow::Result;
4use fs_err::File;
5use rand::SeedableRng;
6use rand_xorshift::XorShiftRng;
7use serde::Deserialize;
8
9use abstio::path_shared_input;
10use abstutil::{prettyprint_usize, Timer};
11use geom::{GPSBounds, Polygon};
12use map_model::Map;
13use popdat::od::DesireLine;
14use raw_map::RawMap;
15use synthpop::{Scenario, TrafficCounts, TripEndpoint, TripMode};
16
17use crate::configuration::ImporterConfiguration;
18use crate::utils::download;
19
20pub async fn import_collision_data(
21    map: &RawMap,
22    config: &ImporterConfiguration,
23    timer: &mut Timer<'_>,
24) {
25    download(
26        config,
27        path_shared_input("Road Safety Data - Accidents 2019.csv"),
28        "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip").await;
29
30    // Always do this, it's idempotent and fast
31    let shapes = kml::ExtraShapes::load_csv(
32        path_shared_input("Road Safety Data - Accidents 2019.csv"),
33        &map.streets.gps_bounds,
34        timer,
35    )
36    .unwrap();
37    let collisions = collisions::import_stats19(
38        shapes,
39        "http://data.dft.gov.uk.s3.amazonaws.com/road-accidents-safety-data/DfTRoadSafety_Accidents_2019.zip");
40    abstio::write_binary(
41        map.get_city_name().input_path("collisions.bin"),
42        &collisions,
43    );
44}
45
46pub async fn generate_scenario(
47    map: &Map,
48    config: &ImporterConfiguration,
49    timer: &mut Timer<'_>,
50) -> Result<()> {
51    timer.start("prepare input");
52    download(
53        config,
54        path_shared_input("wu03ew_v2.csv"),
55        "https://s3-eu-west-1.amazonaws.com/statistics.digitalresources.jisc.ac.uk/dkan/files/FLOW/wu03ew_v2/wu03ew_v2.csv").await;
56    // https://mapit.mysociety.org/area/45350.html (for geocode) E02004277 is an example place to
57    // debug where these zones are.
58    download(
59        config,
60        path_shared_input("zones_core.geojson"),
61        "https://github.com/cyipt/actdev/releases/download/0.1.13/zones_core.geojson",
62    )
63    .await;
64
65    let desire_lines = parse_desire_lines(path_shared_input("wu03ew_v2.csv"))?;
66    let zones = parse_zones(
67        map.get_gps_bounds(),
68        path_shared_input("zones_core.geojson"),
69    )?;
70    timer.stop("prepare input");
71
72    timer.start("disaggregate");
73    // Could plumb this in as a flag to the importer, but it's not critical.
74    let mut rng = XorShiftRng::seed_from_u64(42);
75    let mut scenario = Scenario::empty(map, "background");
76    // Include all buses/trains
77    scenario.only_seed_buses = None;
78    scenario.people = popdat::od::disaggregate(
79        map,
80        zones,
81        desire_lines,
82        popdat::od::Options::default(),
83        &mut rng,
84        timer,
85    );
86    // Some zones have very few buildings, and people wind up with a home and workplace that're the
87    // same!
88    scenario = scenario.remove_weird_schedules(false);
89    // TODO For temporary development of the UK OD pipeline...
90    if false {
91        check_sensor_data(map, &scenario, "/home/dabreegster/sensors.json", timer);
92    }
93    info!(
94        "Generated background traffic scenario with {} people",
95        prettyprint_usize(scenario.people.len())
96    );
97    timer.stop("disaggregate");
98
99    // Does this map belong to the actdev project?
100    match load_study_area(map) {
101        Ok(study_area) => {
102            // Remove people from the scenario we just generated that live in the study area. The
103            // data imported using importer/actdev_scenarios.sh already covers them.
104            let before = scenario.people.len();
105            scenario.people.retain(|p| match p.trips[0].origin {
106                TripEndpoint::Building(b) => !study_area.contains_pt(map.get_b(b).polygon.center()),
107                _ => true,
108            });
109            info!(
110                "Removed {} people from the background scenario that live in the study area",
111                prettyprint_usize(before - scenario.people.len())
112            );
113
114            // Create two scenarios, merging the background traffic with the base/active scenarios.
115            let mut base: Scenario = abstio::maybe_read_binary::<Scenario>(
116                abstio::path_scenario(map.get_name(), "base"),
117                timer,
118            )?;
119            base.people.extend(scenario.people.clone());
120            base.scenario_name = "base_with_bg".to_string();
121            base.save();
122
123            let mut go_active: Scenario = abstio::maybe_read_binary(
124                abstio::path_scenario(map.get_name(), "go_active"),
125                timer,
126            )?;
127            go_active.people.extend(scenario.people);
128            go_active.scenario_name = "go_active_with_bg".to_string();
129            go_active.save();
130
131            // Don't save background.bin for actdev sites -- use base_with_bg instead.
132        }
133        Err(err) => {
134            // We're a "normal" city -- just save the background traffic.
135            info!("{} has no study area: {}", map.get_name().describe(), err);
136            scenario.save();
137        }
138    }
139
140    Ok(())
141}
142
143fn parse_desire_lines(path: String) -> Result<Vec<DesireLine>> {
144    let mut output = Vec::new();
145    for rec in csv::Reader::from_reader(File::open(path)?).deserialize() {
146        let rec: Record = rec?;
147        for (mode, number_commuters) in [
148            (TripMode::Drive, rec.num_drivers),
149            (TripMode::Bike, rec.num_bikers),
150            (TripMode::Walk, rec.num_pedestrians),
151            (
152                TripMode::Transit,
153                rec.num_transit1 + rec.num_transit2 + rec.num_transit3,
154            ),
155        ] {
156            if number_commuters > 0 {
157                output.push(DesireLine {
158                    home_zone: rec.home_zone.clone(),
159                    work_zone: rec.work_zone.clone(),
160                    mode,
161                    number_commuters,
162                });
163            }
164        }
165    }
166    Ok(output)
167}
168
169// An entry in wu03ew_v2.csv. For now, ignores people who work from home, take a taxi, motorcycle,
170// are a passenger in a car, or use "another method of travel".
171#[derive(Debug, Deserialize)]
172struct Record {
173    #[serde(rename = "Area of residence")]
174    home_zone: String,
175    #[serde(rename = "Area of workplace")]
176    work_zone: String,
177    #[serde(rename = "Underground, metro, light rail, tram")]
178    num_transit1: usize,
179    #[serde(rename = "Train")]
180    num_transit2: usize,
181    #[serde(rename = "Bus, minibus or coach")]
182    num_transit3: usize,
183    #[serde(rename = "Driving a car or van")]
184    num_drivers: usize,
185    #[serde(rename = "Bicycle")]
186    num_bikers: usize,
187    #[serde(rename = "On foot")]
188    num_pedestrians: usize,
189}
190
191// Transforms all zones into the map's coordinate space, no matter how far out-of-bounds they are.
192fn parse_zones(gps_bounds: &GPSBounds, path: String) -> Result<HashMap<String, Polygon>> {
193    let mut zones = HashMap::new();
194    let require_in_bounds = false;
195    for (polygon, tags) in
196        Polygon::from_geojson_bytes(&abstio::slurp_file(path)?, gps_bounds, require_in_bounds)?
197    {
198        if let Some(code) = tags.get("geo_code") {
199            zones.insert(code.to_string(), polygon);
200        } else {
201            bail!("Input is missing geo_code: {:?}", tags);
202        }
203    }
204    Ok(zones)
205}
206
207fn load_study_area(map: &Map) -> Result<Polygon> {
208    let require_in_bounds = true;
209    let mut list = Polygon::from_geojson_bytes(
210        &abstio::slurp_file(abstio::path(format!(
211            "system/study_areas/{}.geojson",
212            map.get_name().city.city.replace("_", "-")
213        )))?,
214        map.get_gps_bounds(),
215        require_in_bounds,
216    )?;
217    if list.len() != 1 {
218        bail!("study area geojson has {} polygons", list.len());
219    }
220    Ok(list.pop().unwrap().0)
221}
222
223fn check_sensor_data(map: &Map, scenario: &Scenario, sensor_path: &str, timer: &mut Timer) {
224    use map_model::PathRequest;
225
226    let requests = scenario
227        .all_trips()
228        .filter_map(|trip| {
229            if trip.mode == TripMode::Drive {
230                TripEndpoint::path_req(trip.origin, trip.destination, trip.mode, map)
231            } else {
232                None
233            }
234        })
235        .collect();
236    let deduped = PathRequest::deduplicate(map, requests);
237    let model = TrafficCounts::from_path_requests(
238        map,
239        "the generated scenario".to_string(),
240        &deduped,
241        map.get_pathfinder(),
242        timer,
243    );
244
245    let sensors = abstio::maybe_read_json::<TrafficCounts>(sensor_path.to_string(), timer).unwrap();
246    sensors.quickly_compare(&model);
247}
importer/uk.rs

importer/
uk.rs
