Merge pull request #2683 from storytold/land/1976

DXF: read up to 100 million group codes, or a set limit, without running out of memory silently (#1976)

crates/dxf/src/lib.rs

@@ -23,8 +23,11 @@ pub enum DxfError {
     Eof,
     /// More group codes than [`MAX_TAGS`]: `tags` is the count found (or the count read when the
     /// limit was reached), `limit` the maximum.
-    #[error("drawing too large: {tags} DXF group codes, more than the {limit} CADCraft reads")]
+    #[error("drawing too large: {tags} DXF group codes, more than the {limit} CADCraft reads (set CADCRAFT_MAX_DXF_TAGS to read more)")]
     TooLarge { tags: usize, limit: usize },
+    /// The memory for `tags` group codes couldn't be had: an error, not the process ending.
+    #[error("not enough memory to read this drawing's {tags} DXF group codes")]
+    OutOfMemory { tags: usize },
 }
 
 pub type Result<T> = std::result::Result<T, DxfError>;
@@ -163,16 +166,32 @@ fn typed(code: i32, text: &str) -> Value {
 }
 
 const BINARY_SENTINEL: &[u8] = b"AutoCAD Binary DXF\r\n\x1a\0";
-/// Most group codes read from one file. A `Tag` takes 40 bytes plus its text, so this bounds the
-/// tag list to a few GB; real drawings stay far below (a 10 MB DWG is typically a few million).
-pub const MAX_TAGS: usize = 50_000_000;
+/// Most group codes read from one file by default. A `Tag` takes 40 bytes plus its text, so this
+/// bounds the tag list to about 4 GB plus text. 50 million rejected large DWGs that converted fine
+/// (a 172 MB drawing has 56 million, issue #1948). The file-size limits bound memory too: 512 MB
+/// for a DXF opened directly, 1 GB for a DWG's DXF rendition. Set `CADCRAFT_MAX_DXF_TAGS` to
+/// raise or lower it.
+pub const MAX_TAGS: usize = 100_000_000;
+
+/// Environment variable overriding [`MAX_TAGS`] (a positive integer).
+pub const MAX_TAGS_ENV: &str = "CADCRAFT_MAX_DXF_TAGS";
+
+/// The group-code limit in effect: [`MAX_TAGS_ENV`] when set to a positive integer, else
+/// [`MAX_TAGS`].
+pub fn max_tags() -> usize {
+    max_tags_from(std::env::var(MAX_TAGS_ENV).ok().as_deref())
+}
+
+fn max_tags_from(v: Option<&str>) -> usize {
+    v.and_then(|s| s.trim().replace('_', "").parse::<usize>().ok()).filter(|&n| n > 0).unwrap_or(MAX_TAGS)
+}
 
 mod codepage;
 pub use codepage::decode_code_page;
 
 /// Parse a DXF file (ASCII or binary).
 pub fn parse(bytes: &[u8]) -> Result<Vec<Tag>> {
-    parse_with_limit(bytes, MAX_TAGS)
+    parse_with_limit(bytes, max_tags())
 }
 
 /// [`parse`] with an explicit group-code limit. For ASCII DXF the limit is checked up front from
@@ -199,7 +218,7 @@ pub fn parse_with_limit(bytes: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
     if tags > max_tags {
         return Err(DxfError::TooLarge { tags, limit: max_tags });
     }
-    parse_ascii_with_limit(&codepage::decode_ascii_file(bytes), max_tags)
+    parse_ascii_counted(&codepage::decode_ascii_file(bytes), max_tags, tags)
 }
 
 /// The four hex digits of a `\U+XXXX` escape starting at `chars[i]`, if there is one.
@@ -271,12 +290,17 @@ fn unescape_unicode(s: &str) -> String {
 }
 
 pub fn parse_ascii(text: &str) -> Result<Vec<Tag>> {
-    parse_ascii_with_limit(text, MAX_TAGS)
+    parse_ascii_counted(text, max_tags(), 0)
 }
 
+/// [`parse_ascii`] of a file known to hold `expected` group codes. Their list is reserved first,
+/// with an allocation that can fail: where the system refuses that much memory, opening the file
+/// is an error instead of the end of the app. Growing the list instead could also briefly take
+/// half as much again.
 #[allow(clippy::while_let_loop)]
-fn parse_ascii_with_limit(text: &str, max_tags: usize) -> Result<Vec<Tag>> {
+fn parse_ascii_counted(text: &str, max_tags: usize, expected: usize) -> Result<Vec<Tag>> {
     let mut tags = Vec::new();
+    tags.try_reserve_exact(expected.min(max_tags)).map_err(|_| DxfError::OutOfMemory { tags: expected })?;
     let mut lines = text.strip_prefix('\u{feff}').unwrap_or(text).lines().enumerate();
     loop {
         let Some((ln, code_line)) = lines.next() else { break };
@@ -357,7 +381,7 @@ impl<'a> Cur<'a> {
 
 /// Binary DXF (R13+: 2-byte group codes).
 pub fn parse_binary(b: &[u8]) -> Result<Vec<Tag>> {
-    parse_binary_with_limit(b, MAX_TAGS)
+    parse_binary_with_limit(b, max_tags())
 }
 
 fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
@@ -777,3 +801,27 @@ mod tests {
         assert_eq!(parse(&b).unwrap()[0].str(), "图号");
     }
 }
+
+#[cfg(test)]
+mod limit_tests {
+    use super::*;
+
+    #[test]
+    fn tag_limit_defaults_and_env_override() {
+        assert_eq!(max_tags_from(None), MAX_TAGS);
+        assert_eq!(max_tags_from(Some("")), MAX_TAGS);
+        assert_eq!(max_tags_from(Some("0")), MAX_TAGS);
+        assert_eq!(max_tags_from(Some("abc")), MAX_TAGS);
+        assert_eq!(max_tags_from(Some(" 400_000_000 ")), 400_000_000);
+        // Issue #1948: the default must admit a DXF rendition well above 50 million group codes.
+        const { assert!(MAX_TAGS >= 100_000_000) };
+    }
+
+    #[test]
+    fn the_tag_list_is_reserved_once_and_a_refusal_is_an_error() {
+        let tags = parse_ascii_counted("0\nSECTION\n0\nENDSEC\n0\nEOF\n", MAX_TAGS, 3).unwrap();
+        assert_eq!((tags.len(), tags.capacity()), (3, 3));
+        let e = parse_ascii_counted("0\nEOF\n", usize::MAX, usize::MAX).unwrap_err();
+        assert!(matches!(e, DxfError::OutOfMemory { .. }), "{e}");
+    }
+}

crates/io/src/tests.rs

@@ -1039,7 +1039,7 @@ fn oversized_dwg_and_dxf_are_rejected_early() {
     // Lines that aren't even group codes: the size is reported before anything is parsed.
     let text = "not a group code\nx\n".repeat(60);
     let e = cadcraft_dxf::parse_with_limit(text.as_bytes(), 50).unwrap_err();
-    assert_eq!(e.to_string(), "drawing too large: 60 DXF group codes, more than the 50 CADCraft reads");
+    assert_eq!(e.to_string(), "drawing too large: 60 DXF group codes, more than the 50 CADCraft reads (set CADCRAFT_MAX_DXF_TAGS to read more)");
 }
 
 /// Issue #800: one unreadable object record doesn't fail the whole DWG. The reader skips it, the