Merge remote-tracking branch 'origin/master' into pr-3857

CHANGELOG.md

@@ -3,6 +3,7 @@
 
 ## Other
 
+- Update Cargo dependencies to resolve current RustSec advisories, see #3861 (@TyceHerrman)
 - Add instructions for removing fish help abbreviations to README, see #3655 (@claw-explorer). Closes #3536
 - Add .NET slnx extension, see #3682 (@ltrzesniewski)
 
@@ -24,6 +25,9 @@
 - Syntax highlighting for Python files using uv as script runner in shebang #3689 (@janlarres)
 
 ## Bugfixes
+- Avoid repeated scans of long lines in Log syntax highlighting, see #3876 (@Matei02355)
+- Fix `--list-languages` respecting `--paging=never`, see #3828 (@cyphercodes)
+- Fix `--sanitize` passing through the bidi control characters U+200E, U+200F and U+061C, see #3862 (@lenamonj)
 - `--strip-ansi`: also strip 8-bit C1 introducers (U+0090, U+0098, U+009B, U+009D, U+009E, U+009F) and DCS/SOS/PM/APC sequence bodies, which previously passed through. See #3729 (@curious-rabbit)
 - Fix `--ignored-suffix` not falling back to first-line/shebang detection when the ignored suffix is also a registered extension (e.g. `--ignored-suffix .txt` on a shebang script), see #2745 and #3816 (@adnrivera)
 - Fix `capacity overflow` panic when printing a snip separator at `--terminal-width=1` with multiple line ranges. Closes #3803, see #3804 (@leeewee)

Cargo.toml

@@ -70,13 +70,13 @@ regex = { version = "1.12.3", optional = true }
 walkdir = { version = "2.5", optional = true }
 bytesize = { version = "2.3.1" }
 encoding_rs = "0.8.35"
-execute = { version = "0.2.15", optional = true }
+execute = { version = "0.3.0", optional = true }
 terminal-colorsaurus = "1.0"
 unicode-segmentation = "1.13.2"
 itertools = "0.14.0"
 
 [dependencies.gix]
-version = "0.85"
+version = "0.86"
 optional = true
 default-features = false
 features = ["sha1", "blob-diff"]

README.md

@@ -902,7 +902,7 @@ cargo install --path . --locked --force
 ```
 
 If you want to build an application that uses `bat`'s pretty-printing
-features as a library, check out the [the API documentation](https://docs.rs/bat/).
+features as a library, check out the [API documentation](https://docs.rs/bat/).
 Note that you have to use either `regex-onig` or `regex-fancy` as a feature
 when you depend on `bat` as a library.
 

assets/syntaxes/02_Extra/log.sublime-syntax

@@ -38,22 +38,41 @@ contexts:
       scope: markup.underline.link.scheme.log
       push: url-host
   log_level_lines:
-    - match: (?=.*{{error}})
+    # Prevent retrying each lookahead at every position.
+    - match: ^(?=.*{{error}})
       push:
         - error_line_meta
         - main_pop_at_eol
-    - match: (?=.*{{warning}})
+    - match: ^(?=.*{{warning}})
       push:
         - warning_line_meta
         - main_pop_at_eol
-    - match: (?=.*{{info}})
+    - match: ^(?=.*{{info}})
       push:
         - info_line_meta
         - main_pop_at_eol
-    - match: (?=.*{{debug}})
+    - match: ^(?=.*{{debug}})
       push:
         - debug_line_meta
         - main_pop_at_eol
+  # For syntaxes that embed Log after consuming a prefix. Call this context once.
+  log_level_lines_at_current_position:
+    - match: (?=.*{{error}})
+      set:
+        - error_line_meta
+        - main_pop_at_eol
+    - match: (?=.*{{warning}})
+      set:
+        - warning_line_meta
+        - main_pop_at_eol
+    - match: (?=.*{{info}})
+      set:
+        - info_line_meta
+        - main_pop_at_eol
+    - match: (?=.*{{debug}})
+      set:
+        - debug_line_meta
+        - main_pop_at_eol
   log_levels:
     - match: '{{error}}'
       scope: markup.error.log

assets/syntaxes/02_Extra/syslog.sublime-syntax

@@ -50,6 +50,17 @@ contexts:
     - match: (?=\])
       pop: true
   text:
+    - include: text_special
+    - match: ''
+      set: classify_log_text
+  classify_log_text:
+    - include: scope:text.log#log_level_lines_at_current_position
+    - match: ''
+      set: text_body
+  text_body:
+    - include: text_special
+    - include: scope:text.log
+  text_special:
     - match: $
       pop: true
     - match: '<\w+>'
@@ -62,4 +73,3 @@ contexts:
       escape: \)$
       escape_captures:
         0: punctuation.section.block.end.syslog
-    - include: scope:text.log

src/assets.rs

@@ -363,11 +363,11 @@ impl HighlightingAssets {
         reader: &mut InputReader,
     ) -> Result<Option<SyntaxReferenceInSet<'_>>> {
         let syntax_set = self.get_syntax_set()?;
-        Ok(String::from_utf8(reader.first_line.clone())
+        Ok(std::str::from_utf8(&reader.first_line)
             .ok()
             .and_then(|l| {
                 // Strip UTF-8 BOM if present
-                let line = l.strip_prefix('\u{feff}').unwrap_or(&l);
+                let line = l.strip_prefix('\u{feff}').unwrap_or(l);
                 syntax_set.find_syntax_by_first_line(line)
             })
             .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))

src/bin/bat/main.rs

@@ -427,11 +427,9 @@ fn run() -> Result<bool> {
             if app.matches.get_flag("list-languages") {
                 let languages: String = get_languages(&config, cache_dir)?;
                 let inputs: Vec<Input> = vec![Input::from_reader(Box::new(languages.as_bytes()))];
-                let plain_config = Config {
+                let mut plain_config = config.clone();
-                    style_components: StyleComponents::new(StyleComponent::Plain.components(false)),
+                plain_config.style_components =
-                    paging_mode: PagingMode::QuitIfOneScreen,
+                    StyleComponents::new(StyleComponent::Plain.components(false));
-                    ..Default::default()
-                };
                 run_controller(inputs, &plain_config, cache_dir)
             } else if app.matches.get_flag("list-themes") {
                 list_themes(&config, config_dir, cache_dir, app.theme_options())?;

src/preprocessor.rs

@@ -168,7 +168,7 @@ pub fn sanitize(line: &str) -> String {
 #[inline]
 fn is_sanitize_trigger(b: u8) -> bool {
     // C0 controls minus \t \n \f; DEL; UTF-8 leads with dangerous codepoints.
-    matches!(b, 0x00..=0x08 | 0x0B | 0x0D..=0x1F | 0x7F | 0xC2 | 0xE2 | 0xEF)
+    matches!(b, 0x00..=0x08 | 0x0B | 0x0D..=0x1F | 0x7F | 0xC2 | 0xD8 | 0xE2 | 0xEF)
 }
 
 /// Substitutes the byte/sequence at `bytes[i]` (or passes it through on
@@ -200,8 +200,14 @@ fn sanitize_at(
             buffer.push('\u{FFFD}');
             3
         }
+        // 0xD8 0x9C = U+061C (Arabic letter mark, a bidi control). The rest of
+        // the 0xD8 block is ordinary Arabic text.
+        0xD8 if bytes.get(i + 1) == Some(&0x9C) => {
+            buffer.push('\u{FFFD}');
+            2
+        }
         // False-alarm trigger: pass the full UTF-8 sequence through.
-        lead @ (0xC2 | 0xE2 | 0xEF) => {
+        lead @ (0xC2 | 0xD8 | 0xE2 | 0xEF) => {
             let n = utf8_len_from_lead(lead);
             buffer.push_str(&full[i..i + n]);
             n
@@ -217,10 +223,11 @@ fn sanitize_at(
 
 #[inline]
 fn is_dangerous_e2(bytes: &[u8], i: usize) -> bool {
-    // U+200B..D (zero-width), U+202A..E (bidi controls), U+2066..9 (isolates).
+    // U+200B..D (zero-width), U+200E..F (LRM/RLM), U+202A..E (bidi embedding
+    // and override), U+2066..9 (bidi isolates).
     matches!(
         (bytes.get(i + 1), bytes.get(i + 2)),
-        (Some(0x80), Some(0x8B..=0x8D | 0xAA..=0xAE)) | (Some(0x81), Some(0xA6..=0xA9))
+        (Some(0x80), Some(0x8B..=0x8F | 0xAA..=0xAE)) | (Some(0x81), Some(0xA6..=0xA9))
     )
 }
 
@@ -411,6 +418,34 @@ fn test_sanitize_substitutes_bidi_and_zero_width() {
     assert_eq!(sanitize("a\u{FEFF}b"), format!("a{r}b"));
 }
 
+#[test]
+fn test_sanitize_substitutes_every_bidi_control() {
+    // Unicode Bidi_Control is exactly these 12 codepoints. Covering only some
+    // of them leaves a reordering attack available through the rest.
+    let r = '\u{FFFD}';
+    for c in [
+        '\u{061C}', '\u{200E}', '\u{200F}', '\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}',
+        '\u{202E}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}',
+    ] {
+        assert_eq!(
+            sanitize(&format!("a{c}b")),
+            format!("a{r}b"),
+            "U+{:04X} was not substituted",
+            c as u32
+        );
+    }
+}
+
+#[test]
+fn test_sanitize_preserves_arabic_sharing_the_alm_lead_byte() {
+    // U+061C is reached via lead byte 0xD8, which also leads ordinary Arabic.
+    assert_eq!(
+        sanitize("\u{0600}\u{061F}\u{06FF}"),
+        "\u{0600}\u{061F}\u{06FF}"
+    );
+    assert_eq!(sanitize("مرحبا بالعالم"), "مرحبا بالعالم");
+}
+
 #[test]
 fn test_sanitize_preserves_legitimate_bytes() {
     assert_eq!(sanitize("crlf\r\nline\r\n"), "crlf\r\nline\r\n");

tests/integration_tests.rs

@@ -689,12 +689,18 @@ fn list_themes_to_piped_output() {
 }
 
 #[test]
+#[serial]
 fn list_languages() {
-    bat()
+    mocked_pagers::with_mocked_versions_of_more_and_most_in_path(|| {
-        .arg("--list-languages")
+        bat()
-        .assert()
+            .env("PAGER", mocked_pagers::from("echo pager-output"))
-        .success()
+            .arg("--list-languages")
-        .stdout(predicate::str::contains("Rust").normalize());
+            .arg("--paging=never")
+            .assert()
+            .success()
+            .stdout(predicate::str::contains("Rust").normalize())
+            .stdout(predicate::str::contains("pager-output").not());
+    });
 }
 
 #[test]
@@ -2616,6 +2622,145 @@ fn no_first_line_fallback_when_mapping_to_invalid_syntax() {
         .stderr(predicate::str::contains("unknown syntax: 'InvalidSyntax'"));
 }
 
+#[test]
+fn stdin_detects_bash_from_first_line() {
+    let content = "#!/bin/bash\necho hi\n";
+
+    let detected_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    let explicit_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .arg("--language=bash")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    assert_eq!(
+        from_utf8(&detected_output).expect("output is valid utf-8"),
+        from_utf8(&explicit_output).expect("output is valid utf-8")
+    );
+}
+
+#[test]
+fn stdin_detects_diff_from_first_line() {
+    let content = "diff --git a/x b/y\n--- a/x\n+++ b/y\n@@ -1 +1 @@\n-old\n+new\n";
+
+    let detected_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    let explicit_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .arg("--language=diff")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    assert_eq!(
+        from_utf8(&detected_output).expect("output is valid utf-8"),
+        from_utf8(&explicit_output).expect("output is valid utf-8")
+    );
+}
+
+#[test]
+fn empty_stdin_does_not_detect_syntax() {
+    bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .write_stdin("")
+        .assert()
+        .success()
+        .stdout("")
+        .stderr("");
+}
+
+#[test]
+fn binary_stdin_does_not_detect_syntax_from_invalid_utf8_first_line() {
+    let content = b"#!/bin/bash\xff\necho hi\n";
+
+    let detected_output = bat()
+        .arg("--binary=as-text")
+        .arg("--color=always")
+        .arg("--style=plain")
+        .write_stdin(content.as_slice())
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    let plain_text_output = bat()
+        .arg("--binary=as-text")
+        .arg("--color=always")
+        .arg("--style=plain")
+        .arg("--language=txt")
+        .write_stdin(content.as_slice())
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    assert_eq!(
+        from_utf8(&detected_output).expect("output is valid utf-8"),
+        from_utf8(&plain_text_output).expect("output is valid utf-8")
+    );
+}
+
+#[test]
+fn explicit_language_overrides_stdin_first_line_detection() {
+    let content = "diff --git a/x b/y\n--- a/x\n+++ b/y\n@@ -1 +1 @@\n-old\n+new\n";
+
+    let detected_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    let explicit_output = bat()
+        .arg("--color=always")
+        .arg("--style=plain")
+        .arg("--language=json")
+        .arg("-")
+        .write_stdin(content)
+        .assert()
+        .success()
+        .get_output()
+        .stdout
+        .clone();
+
+    assert_ne!(
+        from_utf8(&detected_output).expect("output is valid utf-8"),
+        from_utf8(&explicit_output).expect("output is valid utf-8")
+    );
+}
+
 #[test]
 fn fallback_syntax_is_used_when_no_syntax_is_detected() {
     let content = "# comment\nfoo=bar\n";