Perspective: a homography product that overflows is refused instead of stored or drawn

Homography::then_after returned products whose entries could overflow to
infinity or become NaN. A move, transform or grid reproject of type or a
symbol instance whose stored projection has large finite entries stored such
a product, and the type was then drawn flat, since a projection that is not
finite reads as none.

- Homography::checked_then_after returns None when an entry of the product
  is not finite.
- project_node composes each type and symbol instance projection with it and
  returns false, leaving the object as it is, when one would not be finite;
  moves and transforms in perspective and Define Perspective Grid with
  reproject then return the horizon error and leave the document unchanged.
- transform_map, attach_homography and size_homography return None for such
  a product.
- Edit Text returns the horizon error, and does not enter isolation mode,
  when the type cannot be shown flat where it is drawn.

crates/engine/src/cmd/distortcmds/perspective.rs

@@ -52,8 +52,8 @@ pub(super) fn grid_set(s: &mut Session, p: &Value) -> Result<Value> {
             old.adopt_stored_attachments(d);
             for (id, plane, depth) in persp::attached_roots(d) {
                 let h = old.homography_at(plane, depth).and_then(|h| h.inverse()).zip(g.homography_at(plane, depth));
-                let (hi, h) = h.ok_or_else(|| EngineError::Other("an object in perspective would leave the edited grid".into()))?;
-                let m = h.then_after(&hi);
+                let m = h.and_then(|(hi, h)| h.checked_then_after(&hi));
+                let m = m.ok_or_else(|| EngineError::Other("an object in perspective would leave the edited grid".into()))?;
                 warp_checked(d, id, &m)?;
             }
         }
@@ -98,17 +98,15 @@ pub(super) fn roots(s: &Session, p: &Value) -> Result<Vec<NodeId>> {
     }
 }
 
-/// Map `n` with `f`, failing if any point would leave the plane's visible side.
 /// Map `n` with `h` (type and symbols in perspective keep their art and are drawn through it),
-/// failing if any point would leave the plane's visible side.
+/// failing if any point would leave the plane's visible side or a projection would not be finite.
 fn warp_checked(d: &mut Document, id: NodeId, h: &Homography) -> Result<()> {
     let n = d.node_mut(id).ok_or(EngineError::NoNode(id))?;
     let mut pts = vec![];
     collect_points(n, &mut pts);
-    if pts.iter().any(|q| h.apply(*q).is_none()) {
+    if pts.iter().any(|q| h.apply(*q).is_none()) || !project_node(n, h) {
         return Err(EngineError::Other("the object would cross the horizon of the perspective plane".into()));
     }
-    project_node(n, h);
     Ok(())
 }
 
@@ -599,11 +597,14 @@ pub(super) fn edit_text(s: &mut Session, p: &Value) -> Result<Value> {
         let NodeKind::Text(t) = &mut n.kind else { return Err(EngineError::Other("select type in perspective".into())) };
         // Shown flat where it is drawn: the type moves there and the projection moves the other
         // way, so the picture stays.
-        let (Some(flat), Some(drawn)) = (t.bounds(), t.bounds().and_then(|b| h.map_rect_bbox(b))) else { return Ok(()) };
+        let Some(flat) = t.bounds() else { return Ok(()) };
+        let cross = || EngineError::Other("the object would cross the horizon of the perspective plane".into());
+        let drawn = h.map_rect_bbox(flat).ok_or_else(cross)?;
         let shift = vectorcraft_geom::Affine::translate(drawn.center() - flat.center());
+        let moved = h.checked_then_after(&Homography::from_affine(shift.inverse())).ok_or_else(cross)?;
         t.transform(shift);
         if let Some(rec) = n.extras_if_any().and_then(|e| e.perspective.as_deref_mut()) {
-            rec.projection = Some(h.then_after(&Homography::from_affine(shift.inverse())).to_array());
+            rec.projection = Some(moved.to_array());
             rec.editing = true;
         }
         Ok(())

crates/engine/src/tests/persp_text.rs

@@ -113,6 +113,39 @@ fn moving_type_in_perspective_moves_its_projection() {
     let _ = rect;
 }
 
+#[test]
+fn moving_type_whose_projection_would_overflow_is_refused() {
+    let (mut s, _) = persp_session();
+    let id = persp_text(&mut s);
+    let NodeKind::Text(t) = &node(&s, id).kind else { panic!("type") };
+    let flat = t.bounds().unwrap();
+    // A stored projection that draws the type flat where it is, with every entry scaled by `k`: a
+    // document can hold it (its entries are finite) and the type draws, but its product with a
+    // scale by 2 in perspective overflows.
+    let step = grid(&s).transform_map(Plane::Right, 0.0, vectorcraft_geom::Affine::scale(2.0), 0.0).unwrap();
+    let scaled = |e: i32| {
+        let k = 2f64.powf(f64::from(e) / 16.0);
+        vectorcraft_geom::Homography([[k, 0.0, 0.0], [0.0, k, 0.0], [0.0, 0.0, k]])
+    };
+    let finite = |m: &vectorcraft_geom::Homography| m.to_array().iter().all(|v| v.is_finite());
+    let draws = |m: &vectorcraft_geom::Homography| {
+        let drawn = m.map_rect_bbox(flat);
+        drawn.is_some_and(|b| step.map_rect_bbox(b).is_some()) && vectorcraft_geom::Homography::from_array(m.to_array()) == Some(*m)
+    };
+    let e = (0..16 * 1024).find(|e| draws(&scaled(*e)) && !finite(&step.then_after(&scaled(*e)))).unwrap_or_else(|| panic!("{step:?} {flat:?}"));
+    let big = scaled(e);
+    let mut doc = s.doc().unwrap().doc.clone();
+    let mut extras = std::sync::Arc::make_mut(&mut doc).node_mut(id).unwrap().extras_mut();
+    extras.perspective.as_deref_mut().unwrap().projection = Some(big.to_array());
+    drop(extras);
+    s.doc_mut().unwrap().doc = doc;
+    let before = (node(&s, id), undo_len(&s));
+    assert!(s.execute("perspective.transform", &json!({"ids": [id.0], "matrix": [2, 0, 0, 2, 0, 0]})).is_err());
+    assert_eq!((node(&s, id), undo_len(&s)), before, "the document is unchanged");
+    let stored = node(&s, id).extras().perspective.as_deref().and_then(|p| p.projection).unwrap();
+    assert!(stored.iter().all(|v| v.is_finite()));
+}
+
 #[test]
 fn edit_text_shows_it_flat_in_isolation_then_projects_it_again() {
     let (mut s, _) = persp_session();
@@ -141,6 +174,37 @@ fn edit_text_shows_it_flat_in_isolation_then_projects_it_again() {
     assert!(s2.execute("perspective.editText", &json!({"id": rect.0})).is_err());
 }
 
+#[test]
+fn edit_text_whose_flat_view_would_overflow_is_refused() {
+    let (mut s, _) = persp_session();
+    let id = persp_text(&mut s);
+    let NodeKind::Text(t) = &node(&s, id).kind else { panic!("type") };
+    let flat = t.bounds().unwrap();
+    // A stored projection that draws the type 4 times its size, with every entry scaled by `k`: a
+    // document can hold it and the type draws, but the projection that shows it flat where it is
+    // drawn overflows.
+    let scaled = |e: i32| {
+        let k = 2f64.powf(f64::from(e) / 16.0);
+        vectorcraft_geom::Homography([[4.0 * k, 0.0, 0.0], [0.0, 4.0 * k, 0.0], [0.0, 0.0, k]])
+    };
+    let overflows = |m: &vectorcraft_geom::Homography| {
+        let Some(drawn) = m.map_rect_bbox(flat) else { return false };
+        let shift = vectorcraft_geom::Affine::translate(drawn.center() - flat.center());
+        let moved = m.then_after(&vectorcraft_geom::Homography::from_affine(shift.inverse()));
+        vectorcraft_geom::Homography::from_array(m.to_array()) == Some(*m) && moved.to_array().iter().any(|v| !v.is_finite())
+    };
+    let e = (0..16 * 1024).find(|e| overflows(&scaled(*e))).unwrap_or_else(|| panic!("{flat:?}"));
+    let mut doc = s.doc().unwrap().doc.clone();
+    let mut extras = std::sync::Arc::make_mut(&mut doc).node_mut(id).unwrap().extras_mut();
+    extras.perspective.as_deref_mut().unwrap().projection = Some(scaled(e).to_array());
+    drop(extras);
+    s.doc_mut().unwrap().doc = doc;
+    let before = (node(&s, id), undo_len(&s));
+    assert!(s.execute("perspective.editText", &json!({"id": id.0})).is_err());
+    assert_eq!((node(&s, id), undo_len(&s)), before, "the document is unchanged");
+    assert_eq!(s.doc().unwrap().isolation, None, "not in isolation mode");
+}
+
 /// Edit Text, which shows the type in isolation mode, is off while an opacity mask is edited.
 #[test]
 fn edit_text_is_off_while_an_opacity_mask_is_edited() {