Merge branch 'main' into fix/charset-none-fallback

Author: afourney

.github/dependabot.yml

@@ -4,3 +4,5 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 7

.github/workflows/pre-commit.yml

@@ -5,9 +5,9 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.x"
 

.github/workflows/tests.yml

@@ -5,8 +5,8 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: |
             3.10

README.md

@@ -188,6 +188,13 @@ Content Understanding is ideal when you need capabilities beyond what built-in o
 markitdown path-to-file.pdf --use-cu --cu-endpoint "<content_understanding_endpoint>"
 ```
 
+The endpoint can also be set once in the environment, so callers only need `--use-cu`:
+
+```bash
+export MARKITDOWN_CU_ENDPOINT="<content_understanding_endpoint>"
+markitdown path-to-file.pdf --use-cu
+```
+
 **Python API:**
 
 ```python
@@ -244,6 +251,13 @@ To use Microsoft Document Intelligence for conversion:
 markitdown path-to-file.pdf -o document.md -d -e "<document_intelligence_endpoint>"
 ```
 
+The endpoint can also be set once in the environment, so callers only need `-d`:
+
+```bash
+export MARKITDOWN_DOCINTEL_ENDPOINT="<document_intelligence_endpoint>"
+markitdown path-to-file.pdf -o document.md -d
+```
+
 More information about how to set up an Azure Document Intelligence Resource can be found [here](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/how-to-guides/create-document-intelligence-resource?view=doc-intel-4.0.0)
 
 ### Python API

packages/markitdown/src/markitdown/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Adam Fourney <adamfo@microsoft.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.6"
+__version__ = "0.1.7"

packages/markitdown/src/markitdown/__main__.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 import argparse
+import os
 import sys
 import codecs
 from typing import Any, Dict
@@ -98,13 +99,15 @@ def main():
         "-e",
         "--endpoint",
         type=str,
-        help="Document Intelligence Endpoint. Required if using Document Intelligence.",
+        default=os.environ.get("MARKITDOWN_DOCINTEL_ENDPOINT") or None,
+        help="Document Intelligence Endpoint. Required if using Document Intelligence. Defaults to the MARKITDOWN_DOCINTEL_ENDPOINT environment variable.",
     )
 
     parser.add_argument(
         "--cu-endpoint",
         type=str,
-        help="Content Understanding Endpoint. Required if using --use-cu.",
+        default=os.environ.get("MARKITDOWN_CU_ENDPOINT") or None,
+        help="Content Understanding Endpoint. Required if using --use-cu. Defaults to the MARKITDOWN_CU_ENDPOINT environment variable.",
     )
 
     parser.add_argument(
@@ -203,7 +206,8 @@ def main():
     if args.use_docintel:
         if args.endpoint is None:
             _exit_with_error(
-                "Document Intelligence Endpoint is required when using Document Intelligence."
+                "Document Intelligence Endpoint is required when using Document Intelligence. "
+                "Pass -e/--endpoint or set MARKITDOWN_DOCINTEL_ENDPOINT."
             )
         elif args.filename is None:
             _exit_with_error("Filename is required when using Document Intelligence.")
@@ -215,9 +219,8 @@ def main():
         if args.cu_endpoint is None:
             _exit_with_error(
                 "Content Understanding Endpoint (--cu-endpoint) is required when using --use-cu."
+                "Pass --cu-endpoint or set MARKITDOWN_CU_ENDPOINT."
             )
-        elif args.filename is None:
-            _exit_with_error("Filename is required when using Content Understanding.")
 
         cu_kwargs: Dict[str, Any] = {
             "cu_endpoint": args.cu_endpoint,

packages/markitdown/src/markitdown/_exceptions.py

@@ -73,4 +73,10 @@ def __init__(
                     else:
                         message += f" - {type(attempt.converter).__name__} threw {attempt.exc_info[0].__name__} with message: {attempt.exc_info[1]}\n"
 
+                        if isinstance(attempt.exc_info[1], UnicodeDecodeError):
+                            message += (
+                                "The charset may have been automatically guessed incorrectly. "
+                                "If you didn't specify the charset, please retry with an explicitly defined one.\n"
+                            )
+
         super().__init__(message)

packages/markitdown/src/markitdown/_markitdown.py

@@ -6,6 +6,8 @@
 import traceback
 import io
 from dataclasses import dataclass
+from email.message import Message
+from email.utils import collapse_rfc2231_value
 from importlib.metadata import entry_points
 from typing import Any, List, Dict, Optional, Union, BinaryIO
 from pathlib import Path
@@ -51,6 +53,23 @@
 )
 
 
+def _get_content_disposition_filename(content_disposition: str) -> Optional[str]:
+    message = Message()
+    message["content-disposition"] = content_disposition
+
+    fallback_filename: Optional[str] = None
+    extended_filename: Optional[str] = None
+    for key, value in message.get_params(header="content-disposition", unquote=True):
+        if key != "filename":
+            continue
+        if isinstance(value, tuple):
+            extended_filename = collapse_rfc2231_value(value)
+        elif fallback_filename is None:
+            fallback_filename = value
+
+    return extended_filename or fallback_filename
+
+
 # Lower priority values are tried first.
 PRIORITY_SPECIFIC_FILE_FORMAT = (
     0.0  # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia
@@ -512,9 +531,10 @@ def convert_response(
         filename: Optional[str] = None
         extension: Optional[str] = None
         if "content-disposition" in response.headers:
-            m = re.search(r"filename=([^;]+)", response.headers["content-disposition"])
-            if m:
-                filename = m.group(1).strip("\"'")
+            filename = _get_content_disposition_filename(
+                response.headers["content-disposition"]
+            )
+            if filename is not None:
                 _, _extension = os.path.splitext(filename)
                 if len(_extension) > 0:
                     extension = _extension
@@ -726,9 +746,9 @@ def _get_stream_info_guesses(
                 # If it's text, also guess the charset
                 charset = None
                 if result.prediction.output.is_text:
-                    # Read the first 4k to guess the charset
+                    # Read the first 64k to guess the charset
                     file_stream.seek(cur_pos)
-                    stream_page = file_stream.read(4096)
+                    stream_page = file_stream.read(65536)
                     charset_result = charset_normalizer.from_bytes(stream_page).best()
 
                     if charset_result is not None:

packages/markitdown/src/markitdown/_uri_utils.py

@@ -12,7 +12,10 @@ def file_uri_to_path(file_uri: str) -> Tuple[str | None, str]:
         raise ValueError(f"Not a file URL: {file_uri}")
 
     netloc = parsed.netloc if parsed.netloc else None
-    path = os.path.abspath(url2pathname(parsed.path))
+    path = url2pathname(parsed.path)
+    if os.name == "nt" and path[:1] in "/\\" and path[2:3] == ":":
+        path = path[1:]
+    path = os.path.abspath(path)
     return netloc, path
 
 

packages/markitdown/src/markitdown/converter_utils/docx/math/latex_dict.py

@@ -50,7 +50,7 @@
     "\u20e8": "\\threeunderdot{{{0}}}",
     "\u20ec": "\\underrightharpoondown{{{0}}}",
     "\u20ed": "\\underleftharpoondown{{{0}}}",
-    "\u20ee": "\\underledtarrow{{{0}}}",
+    "\u20ee": "\\underleftarrow{{{0}}}",
     "\u20ef": "\\underrightarrow{{{0}}}",
     # Over | group
     "\u23b4": "\\overbracket{{{0}}}",
@@ -180,6 +180,8 @@
     "\U0001d452": "e",
     "\U0001d453": "f",
     "\U0001d454": "g",
+    # U+1D455 is reserved: the math italic small h is unified with U+210E
+    "\u210e": "h",
     "\U0001d456": "i",
     "\U0001d457": "j",
     "\U0001d458": "k",
@@ -222,6 +224,7 @@
 
 CHR_DEFAULT = {
     "ACC_VAL": "\\hat{{{0}}}",
+    "GROUP_CHR_VAL": "\\underbrace{{{0}}}",
 }
 
 POS = {