diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index 7717f62d8..79d7656e5 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -1,5 +1,8 @@ +import codecs +import struct import sys -from typing import Any, Union, BinaryIO +from typing import Any, Dict, Union, BinaryIO +from charset_normalizer import from_bytes from .._stream_info import StreamInfo from .._base_converter import DocumentConverter, DocumentConverterResult from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE @@ -20,6 +23,56 @@ ACCEPTED_FILE_EXTENSIONS = [".msg"] +# Fixed-width MAPI properties (PT_LONG among them) are not stored in a stream of +# their own -- they are packed into the message's property stream, after a header +# that is 32 bytes long for a top-level message, as 16-byte entries of tag, flags +# and value. +PROPERTIES_STREAM = "__properties_version1.0" +PROPERTIES_HEADER_SIZE = 32 +PROPERTY_ENTRY_SIZE = 16 +PT_LONG = 0x0003 + +PR_MESSAGE_CODEPAGE = 0x3FFD # PidTagMessageCodepage +PR_INTERNET_CPID = 0x3FDE # PidTagInternetCodepage + +# Windows code page identifiers whose Python codec is not simply "cp". +# Anything absent from this map is looked up under that name, so the common +# single-byte and East Asian pages (1250-1258, 874, 932, 936, 949, 950, ...) +# need no entry here. +CODEPAGE_CODECS = { + 708: "iso8859-6", + 20127: "ascii", + 20866: "koi8-r", + 21866: "koi8-u", + 28591: "iso8859-1", + 28592: "iso8859-2", + 28593: "iso8859-3", + 28594: "iso8859-4", + 28595: "iso8859-5", + 28596: "iso8859-6", + 28597: "iso8859-7", + 28598: "iso8859-8", + 28599: "iso8859-9", + 28603: "iso8859-13", + 28605: "iso8859-15", + 10000: "mac_roman", + 10006: "mac_greek", + 10007: "mac_cyrillic", + 10029: "mac_latin2", + 10079: "mac_iceland", + 10081: "mac_turkish", + 50220: "iso2022_jp", + 50221: "iso2022_jp", + 50222: "iso2022_jp", + 50225: "iso2022_kr", + 51932: "euc_jp", + 51936: "gb2312", + 51949: "euc_kr", + 54936: "gb18030", + 65000: "utf-7", + 65001: "utf-8", +} + class OutlookMsgConverter(DocumentConverter): """Converts Outlook .msg files to markdown by extracting email metadata and content. @@ -95,14 +148,32 @@ def convert( ) # If we made it this far, olefile should be available msg = olefile.OleFileIO(file_stream) + # The code page that PT_STRING8 properties are written in is declared by + # the message itself, so read it once here rather than guessing at each + # property. PidTagMessageCodepage covers the message's string properties; + # PidTagInternetCodepage, when present, describes the body it arrived in. + long_properties = self._get_long_properties(msg) + message_encoding = self._get_codec_name( + long_properties.get(PR_MESSAGE_CODEPAGE) + ) + internet_encoding = self._get_codec_name(long_properties.get(PR_INTERNET_CPID)) + header_encoding = message_encoding or internet_encoding + body_encoding = internet_encoding or message_encoding + # Extract email metadata md_content = "# Email Message\n\n" # Get headers headers = { - "From": self._get_stream_data(msg, "__substg1.0_0C1F001F"), - "To": self._get_stream_data(msg, "__substg1.0_0E04001F"), - "Subject": self._get_stream_data(msg, "__substg1.0_0037001F"), + "From": self._get_property_data( + msg, "0C1F", header_encoding + ), # PR_SENDER_EMAIL_ADDRESS + "To": self._get_property_data( + msg, "0E04", header_encoding + ), # PR_DISPLAY_TO + "Subject": self._get_property_data( + msg, "0037", header_encoding + ), # PR_SUBJECT } # Add headers to markdown @@ -113,7 +184,7 @@ def convert( md_content += "\n## Content\n\n" # Get email body - body = self._get_stream_data(msg, "__substg1.0_1000001F") + body = self._get_property_data(msg, "1000", body_encoding) # PR_BODY if body: md_content += body @@ -124,6 +195,102 @@ def convert( title=headers.get("Subject"), ) + def _get_long_properties(self, msg: Any) -> Dict[int, int]: + """Helper to read the message's PT_LONG properties, keyed by property id. + + Fixed-width properties carry no stream of their own: they sit in the + message's property stream, whose entries are 16 bytes of tag, flags and + value, following a 32-byte header for a top-level message. + """ + assert olefile is not None + assert isinstance(msg, olefile.OleFileIO) + + properties: Dict[int, int] = {} + try: + if not msg.exists(PROPERTIES_STREAM): + return properties + data = msg.openstream(PROPERTIES_STREAM).read() + except Exception: + return properties + + offset = PROPERTIES_HEADER_SIZE + while offset + PROPERTY_ENTRY_SIZE <= len(data): + entry = data[offset : offset + PROPERTY_ENTRY_SIZE] + offset += PROPERTY_ENTRY_SIZE + tag, _flags, value = struct.unpack("> 16] = value + return properties + + def _get_codec_name(self, codepage: Union[int, None]) -> Union[str, None]: + """Helper to map a Windows code page identifier to a Python codec. + + Returns None for a code page the message does not declare, or for one + Python cannot decode -- either way the encoding has to be detected. + """ + if not codepage: + return None + + name = CODEPAGE_CODECS.get(codepage, "cp%d" % codepage) + try: + return codecs.lookup(name).name + except LookupError: + return None + + def _get_property_data( + self, msg: Any, property_tag: str, encoding: Union[str, None] = None + ) -> Union[str, None]: + """Helper to read a MAPI string property, whichever string type is used. + + Outlook stores each string property either as PT_UNICODE (stream type + 001F, UTF-16LE) or as PT_STRING8 (stream type 001E, the message's 8-bit + code page), depending on whether the message was saved in Unicode or in + the legacy non-Unicode format. A message carries one or the other, so + reading only 001F returns nothing at all for a non-Unicode .msg. + """ + value = self._get_stream_data(msg, "__substg1.0_%s001F" % property_tag) + if value: + return value + return self._get_ansi_stream_data( + msg, "__substg1.0_%s001E" % property_tag, encoding + ) + + def _get_ansi_stream_data( + self, msg: Any, stream_path: str, encoding: Union[str, None] = None + ) -> Union[str, None]: + """Helper to extract and decode a PT_STRING8 stream from the MSG file. + + These streams record no encoding of their own -- they are written in the + code page the message declares, which is what ``encoding`` carries. Only + when that declaration is missing, unsupported, or contradicted by the + bytes is the charset detected instead: a header is a short and highly + ambiguous sample, and detection routinely misreads one (a CP1252 + "Résumé" reads as UTF-16BE, a CP1251 "Привет мир" as CP1125). + """ + assert olefile is not None + assert isinstance(msg, olefile.OleFileIO) + + try: + if not msg.exists(stream_path): + return None + data = msg.openstream(stream_path).read() + except Exception: + return None + + if not data: + return None + + if encoding is not None: + try: + return data.decode(encoding).strip() + except (UnicodeDecodeError, LookupError): + pass # The declared code page does not fit; fall back to detection + + detected = from_bytes(data).best() + if detected is not None: + return str(detected).strip() + return data.decode("utf-8", errors="ignore").strip() + def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: """Helper to safely extract and decode stream data from the MSG file.""" assert olefile is not None diff --git a/packages/markitdown/tests/test_outlook_msg_ansi.py b/packages/markitdown/tests/test_outlook_msg_ansi.py new file mode 100644 index 000000000..57b4db15d --- /dev/null +++ b/packages/markitdown/tests/test_outlook_msg_ansi.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 -m pytest +"""Tests for .msg files saved in the legacy non-Unicode format.""" + +import io +import os +import struct +from unittest.mock import patch + +import olefile + +from markitdown import MarkItDown +from markitdown._stream_info import StreamInfo +from markitdown.converters._outlook_msg_converter import OutlookMsgConverter + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files") + +SENDER = "ana.lopez@example.com" +RECIPIENT = "carlos.ruiz@example.com" +SUBJECT = "Confirmación de la reunión del martes" +BODY = ( + "Hola Carlos,\r\n\r\n" + "Te confirmo la reunión del martes a las diez en la oficina de Bilbao. " + "He adjuntado el informe de facturación del último trimestre para que " + "puedas revisarlo antes, junto con la propuesta de calendario que " + "comentamos por teléfono la semana pasada.\r\n\r\n" + "Un saludo,\r\nAna" +) + +# Strings short enough, and in code pages close enough, that charset detection +# picks the wrong codec: charset_normalizer reads the first as UTF-16BE +# ("勩獵淩") and the second as CP1125 ("╧ЁштхҐ ьшЁ"). +AMBIGUOUS_LATIN = "Résumé" +AMBIGUOUS_CYRILLIC = "Привет мир" + +PR_MESSAGE_CODEPAGE = 0x3FFD +PR_INTERNET_CPID = 0x3FDE + +# Property ids of the string properties the converter reads. +SENDER_TAG = "0C1F" +RECIPIENT_TAG = "0E04" +SUBJECT_TAG = "0037" +BODY_TAG = "1000" + + +def _properties_stream(codepages: dict) -> bytes: + """Build a top-level message property stream declaring the given code pages. + + A 32-byte header, then one 16-byte entry per property: tag, flags, value. + """ + data = b"\x00" * 32 + for property_id, codepage in codepages.items(): + tag = (property_id << 16) | 0x0003 # PT_LONG + data += struct.pack(" dict: + """The streams Outlook writes when saving in the Unicode format.""" + return { + "__substg1.0_0C1F001F": SENDER.encode("utf-16-le"), + "__substg1.0_0E04001F": RECIPIENT.encode("utf-16-le"), + "__substg1.0_0037001F": SUBJECT.encode("utf-16-le"), + "__substg1.0_1000001F": BODY.encode("utf-16-le"), + } + + +def _ansi_streams(encoding: str = "cp1252", codepage: int = 1252) -> dict: + """The same message saved in the non-Unicode format, in a declared code page. + + Passing ``codepage=0`` leaves the declaration out, as a malformed message + would, so that only detection is left to fall back on. + """ + streams = { + "__substg1.0_0C1F001E": SENDER.encode(encoding), + "__substg1.0_0E04001E": RECIPIENT.encode(encoding), + "__substg1.0_0037001E": SUBJECT.encode(encoding), + "__substg1.0_1000001E": BODY.encode(encoding), + } + if codepage: + streams["__properties_version1.0"] = _properties_stream( + {PR_MESSAGE_CODEPAGE: codepage} + ) + return streams + + +def _fake_olefile(streams: dict): + """Build a stand-in for olefile.OleFileIO serving a fixed set of streams.""" + + class _FakeOleFileIO(olefile.OleFileIO): + def __init__(self, file_stream): + # No container to open. The flag keeps OleFileIO.__del__ from + # tripping over the state a real open() would have set up. + self._we_opened_fp = False + + def exists(self, path): + return path in streams + + def openstream(self, path): + return io.BytesIO(streams[path]) + + def close(self): + pass + + return _FakeOleFileIO + + +def _convert(streams: dict) -> str: + with patch.object(olefile, "OleFileIO", _fake_olefile(streams)): + return ( + OutlookMsgConverter() + .convert(io.BytesIO(b""), StreamInfo(extension=".msg")) + .markdown + ) + + +def test_ansi_message_keeps_headers_and_body() -> None: + """A non-Unicode .msg must convert like its Unicode counterpart.""" + markdown = _convert(_ansi_streams()) + + assert f"**From:** {SENDER}" in markdown + assert f"**To:** {RECIPIENT}" in markdown + assert f"**Subject:** {SUBJECT}" in markdown + assert "Te confirmo la reunión del martes" in markdown + assert "informe de facturación" in markdown + + +def test_ansi_message_is_not_silently_empty() -> None: + """The failure mode was scaffolding with every field dropped.""" + markdown = _convert(_ansi_streams()) + + assert markdown != "# Email Message\n\n## Content" + assert "**Subject:**" in markdown + + +def test_declared_codepage_decodes_cyrillic_headers() -> None: + """A CP1251 header is decoded from the declared code page, not detected. + + Detection reads this subject as CP1125 and yields "╧ЁштхҐ ьшЁ". + """ + streams = { + "__substg1.0_0037001E": AMBIGUOUS_CYRILLIC.encode("cp1251"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 1251}), + } + + assert f"**Subject:** {AMBIGUOUS_CYRILLIC}" in _convert(streams) + + +def test_declared_codepage_decodes_short_latin_headers() -> None: + """A short CP1252 header is decoded from the declared code page. + + Detection reads these six bytes as UTF-16BE and yields "勩獵淩". + """ + streams = { + "__substg1.0_0037001E": AMBIGUOUS_LATIN.encode("cp1252"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 1252}), + } + + assert f"**Subject:** {AMBIGUOUS_LATIN}" in _convert(streams) + + +def test_internet_codepage_decodes_the_body() -> None: + """PidTagInternetCodepage describes the body, PidTagMessageCodepage the rest.""" + streams = { + "__substg1.0_0037001E": AMBIGUOUS_LATIN.encode("cp1252"), + "__substg1.0_1000001E": AMBIGUOUS_CYRILLIC.encode("cp1251"), + "__properties_version1.0": _properties_stream( + {PR_MESSAGE_CODEPAGE: 1252, PR_INTERNET_CPID: 1251} + ), + } + markdown = _convert(streams) + + assert f"**Subject:** {AMBIGUOUS_LATIN}" in markdown + assert AMBIGUOUS_CYRILLIC in markdown + + +def test_greek_codepage_is_honored() -> None: + """CP1253 bytes are Greek, however little of the alphabet a header shows.""" + subject = "Ελληνικά" + streams = { + "__substg1.0_0037001E": subject.encode("cp1253"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 1253}), + } + + assert f"**Subject:** {subject}" in _convert(streams) + + +def test_codepage_needing_a_codec_alias_is_honored() -> None: + """Code pages whose codec is not named "cp" still have to map. + + Python has no "cp28595" -- ISO 8859-5 is reached under its own name. + """ + subject = AMBIGUOUS_CYRILLIC + streams = { + "__substg1.0_0037001E": subject.encode("iso8859-5"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 28595}), + } + + assert f"**Subject:** {subject}" in _convert(streams) + + +def test_utf8_codepage_is_honored() -> None: + """PT_STRING8 may be UTF-8, which the message declares as code page 65001.""" + streams = { + "__substg1.0_0037001E": SUBJECT.encode("utf-8"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 65001}), + } + + assert f"**Subject:** {SUBJECT}" in _convert(streams) + + +def test_missing_codepage_falls_back_to_detection() -> None: + """Without a declaration there is nothing to go on but the bytes.""" + markdown = _convert(_ansi_streams(codepage=0)) + + assert f"**From:** {SENDER}" in markdown + assert "Te confirmo la reunión del martes" in markdown + + +def test_unsupported_codepage_falls_back_to_detection() -> None: + """A code page Python cannot decode must not lose the message either.""" + streams = _ansi_streams() + streams["__properties_version1.0"] = _properties_stream( + {PR_MESSAGE_CODEPAGE: 99999} + ) + markdown = _convert(streams) + + assert f"**From:** {SENDER}" in markdown + assert "Te confirmo la reunión del martes" in markdown + + +def test_undecodable_bytes_fall_back_to_detection() -> None: + """A declaration the bytes contradict must not raise, nor drop the field.""" + streams = { + # US-ASCII (code page 20127) cannot represent the accents in the body. + "__substg1.0_1000001E": BODY.encode("cp1252"), + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: 20127}), + } + + assert "Te confirmo la reunión del martes" in _convert(streams) + + +def test_unicode_message_is_unaffected() -> None: + """The Unicode format must keep taking the same path as before.""" + markdown = _convert(_unicode_streams()) + + assert f"**From:** {SENDER}" in markdown + assert f"**To:** {RECIPIENT}" in markdown + assert f"**Subject:** {SUBJECT}" in markdown + assert "Te confirmo la reunión del martes" in markdown + + +def test_real_unicode_fixture_still_converts() -> None: + """Regression guard over the checked-in .msg, read through real olefile.""" + result = MarkItDown().convert(os.path.join(TEST_FILES_DIR, "test_outlook_msg.msg")) + + assert "**From:** test.sender@example.com" in result.markdown + assert "**Subject:** Test Email Message" in result.markdown + assert "This is the body of the test email message" in result.markdown + + +if __name__ == "__main__": + import pytest + + pytest.main([__file__, "-v"])