kml/
lib.rs

Help
// TODO Time to rename this crate

#[macro_use]
extern crate anyhow;
#[macro_use]
extern crate log;

use std::collections::BTreeMap;

use anyhow::Result;
use serde::{Deserialize, Serialize};

use abstutil::{prettyprint_usize, Timer};
use geom::{GPSBounds, LonLat, PolyLine, Polygon};

/// Some dataset imported from KML, CSV, or something else. If the dataset is large, converting to
/// this format and serializing is faster than parsing the original again.
#[derive(Serialize, Deserialize)]
pub struct ExtraShapes {
    pub shapes: Vec<ExtraShape>,
}

/// A single object in the dataset.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ExtraShape {
    /// The object has a different inferred shape depending on the points:
    /// - a single point just represents a position
    /// - a ring of points (with the first and last matching) is interpreted as a polygon
    /// - multiple points are interpreted as a PolyLine
    pub points: Vec<LonLat>,
    /// Arbitrary key/value pairs associated with this object; no known schema.
    pub attributes: BTreeMap<String, String>,
}

/// Parses a .kml file and returns ExtraShapes. Objects will be clipped to the given gps_bounds. If
/// require_all_pts_in_bounds is true, objects that're partly out-of-bounds will be skipped.
pub fn load(
    path: String,
    gps_bounds: &GPSBounds,
    require_all_pts_in_bounds: bool,
    timer: &mut Timer,
) -> Result<ExtraShapes> {
    timer.start(format!("read {}", path));
    let bytes = abstio::slurp_file(&path)?;
    let raw_string = std::str::from_utf8(&bytes)?;
    let tree = roxmltree::Document::parse(raw_string)?;
    timer.stop(format!("read {}", path));

    let mut shapes = Vec::new();
    let mut skipped_count = 0;
    let mut kv = BTreeMap::new();

    timer.start("scrape objects");
    recurse(
        tree.root(),
        &mut shapes,
        &mut skipped_count,
        &mut kv,
        gps_bounds,
        require_all_pts_in_bounds,
    )?;
    timer.stop("scrape objects");

    info!(
        "Got {} shapes from {} and skipped {} shapes",
        prettyprint_usize(shapes.len()),
        path,
        prettyprint_usize(skipped_count)
    );

    Ok(ExtraShapes { shapes })
}

fn recurse(
    node: roxmltree::Node,
    shapes: &mut Vec<ExtraShape>,
    skipped_count: &mut usize,
    kv: &mut BTreeMap<String, String>,
    gps_bounds: &GPSBounds,
    require_all_pts_in_bounds: bool,
) -> Result<()> {
    for child in node.children() {
        recurse(
            child,
            shapes,
            skipped_count,
            kv,
            gps_bounds,
            require_all_pts_in_bounds,
        )?;
    }
    if node.tag_name().name() == "SimpleData" {
        let key = node.attribute("name").unwrap().to_string();
        let value = node
            .text()
            .map(|x| x.to_string())
            .unwrap_or_else(String::new);
        kv.insert(key, value);
    } else if node.tag_name().name() == "coordinates" {
        let mut any_oob = false;
        let mut any_ok = false;
        let mut pts: Vec<LonLat> = Vec::new();
        if let Some(txt) = node.text() {
            for pair in txt.trim().split(' ') {
                if let Some(pt) = parse_pt(pair) {
                    pts.push(pt);
                    if gps_bounds.contains(pt) {
                        any_ok = true;
                    } else {
                        any_oob = true;
                    }
                } else {
                    bail!("Malformed coordinates: {}", pair);
                }
            }
        }
        if any_ok && (!any_oob || !require_all_pts_in_bounds) {
            let attributes = std::mem::take(kv);
            shapes.push(ExtraShape {
                points: pts,
                attributes,
            });
        } else {
            *skipped_count += 1;
        }
    }
    Ok(())
}

fn parse_pt(input: &str) -> Option<LonLat> {
    let coords: Vec<&str> = input.split(',').collect();
    // Normally each coordinate is just (X, Y), but for census tract files, there's a third Z
    // component that's always 0. Just ignore it.
    if coords.len() < 2 {
        return None;
    }
    match (coords[0].parse::<f64>(), coords[1].parse::<f64>()) {
        (Ok(lon), Ok(lat)) => Some(LonLat::new(lon, lat)),
        _ => None,
    }
}

impl ExtraShapes {
    /// Parses a .csv file and returns ExtraShapes. Each record must EITHER have a column called
    /// 'Longitude' and 'Latitude', representing a single point, OR a column called 'geometry' with
    /// a WKT-style linestring. All other columns will just be attributes. Objects that're partly
    /// out-of-bounds will be excluded.
    pub fn load_csv(
        path: String,
        gps_bounds: &GPSBounds,
        timer: &mut Timer,
    ) -> Result<ExtraShapes> {
        timer.start(format!("read {}", path));
        let mut shapes = Vec::new();
        for rec in csv::Reader::from_path(&path)?.deserialize() {
            let mut rec: BTreeMap<String, String> = rec?;
            match (
                rec.remove("Longitude"),
                rec.remove("Latitude"),
                rec.remove("geometry"),
            ) {
                (Some(lon), Some(lat), _) => {
                    if let (Ok(lon), Ok(lat)) = (lon.parse::<f64>(), lat.parse::<f64>()) {
                        let pt = LonLat::new(lon, lat);
                        if gps_bounds.contains(pt) {
                            shapes.push(ExtraShape {
                                points: vec![pt],
                                attributes: rec,
                            });
                        }
                    }
                }
                (None, None, Some(raw)) => {
                    if let Some(points) = LonLat::parse_wkt_linestring(&raw) {
                        if gps_bounds.try_convert(&points).is_some() {
                            shapes.push(ExtraShape {
                                points,
                                attributes: rec,
                            });
                        }
                    }
                }
                _ => {
                    timer.stop(format!("read {}", path));
                    bail!(
                        "{} doesn't have a column called Longitude, Latitude, or geometry",
                        path
                    )
                }
            }
        }
        timer.stop(format!("read {}", path));
        Ok(ExtraShapes { shapes })
    }
}

impl ExtraShapes {
    /// Parses a .geojson file and returns ExtraShapes
    pub fn load_geojson_no_clipping(
        path: String,
        gps_bounds: &GPSBounds,
        require_in_bounds: bool,
    ) -> Result<ExtraShapes> {
        let bytes = abstio::slurp_file(path)?;
        let mut shapes = Vec::new();

        for (polygon, attributes) in
            Polygon::from_geojson_bytes(&bytes, gps_bounds, require_in_bounds)?
        {
            shapes.push(ExtraShape {
                // Awkward, but we have to convert back
                points: gps_bounds.convert_back(polygon.get_outer_ring().points()),
                attributes,
            });
        }
        for (pl, attributes) in PolyLine::from_geojson_bytes(&bytes, gps_bounds, require_in_bounds)?
        {
            shapes.push(ExtraShape {
                // Awkward, but we have to convert back
                points: gps_bounds.convert_back(pl.points()),
                attributes,
            });
        }

        Ok(ExtraShapes { shapes })
    }
}
kml/lib.rs

kml/
lib.rs
