FIXED: Strip UTF-8 BOM from main files, not just transcluded files

diff --git a/Sources/libMultiMarkdown/transclude.c b/Sources/libMultiMarkdown/transclude.c
index 8f4942fb599681d5673c7221a852ef8fb88c6a6d..c828e470916f15234886fa94c491013061530106 100644 (file)
--- a/Sources/libMultiMarkdown/transclude.c
+++ b/Sources/libMultiMarkdown/transclude.c
@@ -285,6 +285,13 @@ DString * scan_file(const char * fname) {
 
        while ((bytes = fread(chunk, 1, kBUFFERSIZE, file)) > 0) {
                d_string_append_c_array(buffer, chunk, bytes);
+
+               if (buffer->currentStringLength < kBUFFERSIZE) {
+                       // Strip BOM
+                       if (strncmp(buffer->str, "\xef\xbb\xbf", 3) == 0) {
+                               d_string_erase(buffer, 0, 3);
+                       }
+               }
        }
 
        fclose(file);
@@ -467,11 +474,6 @@ void mmd_transclude_source(DString * source, const char * search_path, const cha
 
                                if (mmd_engine_has_metadata(e, &offset)) {
                                        d_string_erase(buffer, 0, offset);
-                               } else {
-                                       // Do we need to strip BOM?
-                                       if (strncmp(buffer->str, "\xef\xbb\xbf", 3) == 0) {
-                                               d_string_erase(buffer, 0, 3);
-                                       }
                                }
 
                                mmd_engine_free(e, false);

diff --git a/tests/MMD6Tests/BOM.fodt b/tests/MMD6Tests/BOM.fodt
new file mode 100644 (file)
index 0000000..bb6f76a
--- /dev/null
+++ b/tests/MMD6Tests/BOM.fodt
@@ -0,0 +1,284 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0"
+xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0"
+xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"
+xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"
+xmlns:xlink="http://www.w3.org/1999/xlink"
+xmlns:dc="http://purl.org/dc/elements/1.1/"
+xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0"
+xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0"
+xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"
+xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0"
+xmlns:dr3d="urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0"
+xmlns:math="http://www.w3.org/1998/Math/MathML"
+xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0"
+xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0"
+xmlns:ooo="http://openoffice.org/2004/office"
+xmlns:ooow="http://openoffice.org/2004/writer"
+xmlns:oooc="http://openoffice.org/2004/calc"
+xmlns:dom="http://www.w3.org/2001/xml-events"
+xmlns:xforms="http://www.w3.org/2002/xforms"
+xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xmlns:rpt="http://openoffice.org/2005/report"
+xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2"
+xmlns:xhtml="http://www.w3.org/1999/xhtml"
+xmlns:grddl="http://www.w3.org/2003/g/data-view#"
+xmlns:officeooo="http://openoffice.org/2009/office"
+xmlns:tableooo="http://openoffice.org/2009/table"
+xmlns:drawooo="http://openoffice.org/2010/draw"
+xmlns:calcext="urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0"
+xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0"
+xmlns:field="urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0"
+xmlns:formx="urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0"
+xmlns:css3t="http://www.w3.org/TR/css3-text/"
+office:version="1.2"
+office:mimetype="application/vnd.oasis.opendocument.text">
+<office:font-face-decls>
+   <style:font-face style:name="Courier New" svg:font-family="'Courier New'"
+                    style:font-adornments="Regular"
+                    style:font-family-generic="modern"
+                    style:font-pitch="fixed"/>
+</office:font-face-decls>
+<office:styles>
+<style:style style:name="Standard" style:family="paragraph" style:class="text">
+      <style:paragraph-properties fo:margin-top="0in" fo:margin-bottom="0.15in"     fo:text-align="justify" style:justify-single-word="false"/>
+   </style:style>
+<style:style style:name="Preformatted_20_Text" style:display-name="Preformatted Text"
+             style:family="paragraph"
+             style:parent-style-name="Standard"
+             style:class="html">
+   <style:paragraph-properties fo:margin-top="0in" fo:margin-bottom="0in" fo:text-align="start"
+                               style:justify-single-word="false"/>
+   <style:text-properties style:font-name="Courier New" fo:font-size="11pt"
+                          style:font-name-asian="Courier New"
+                          style:font-size-asian="11pt"
+                          style:font-name-complex="Courier New"
+                          style:font-size-complex="11pt"/>
+</style:style>
+<style:style style:name="Source_20_Text" style:display-name="Source Text"
+             style:family="text">
+   <style:text-properties style:font-name="Courier New" style:font-name-asian="Courier New"
+                          style:font-name-complex="Courier New"
+                          fo:font-size="11pt"/>
+</style:style>
+<style:style style:name="List" style:family="paragraph"
+             style:parent-style-name="Standard"
+             style:class="list">
+   <style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
+   <style:text-properties style:font-size-asian="12pt"/>
+</style:style>
+<style:style style:name="Quotations" style:family="paragraph"
+             style:parent-style-name="Standard"
+             style:class="html">
+   <style:paragraph-properties fo:margin-left="0.3937in" fo:margin-right="0.3937in" fo:margin-top="0in"
+                               fo:margin-bottom="0.1965in"
+                               fo:text-align="justify"                               style:justify-single-word="false"                               fo:text-indent="0in"
+                               style:auto-text-indent="false"/>
+</style:style>
+<style:style style:name="Table_20_Heading" style:display-name="Table Heading"
+             style:family="paragraph"
+             style:parent-style-name="Table_20_Contents"
+             style:class="extra">
+   <style:paragraph-properties fo:text-align="center" style:justify-single-word="false"
+                               text:number-lines="false"
+                               text:line-number="0"/>
+   <style:text-properties fo:font-weight="bold" style:font-weight-asian="bold"
+                          style:font-weight-complex="bold"/>
+</style:style>
+<style:style style:name="Horizontal_20_Line" style:display-name="Horizontal Line"
+             style:family="paragraph"
+             style:parent-style-name="Standard"
+             style:class="html">
+   <style:paragraph-properties fo:margin-top="0in" fo:margin-bottom="0.1965in"
+                               style:border-line-width-bottom="0.0008in 0.0138in 0.0008in"
+                               fo:padding="0in"
+                               fo:border-left="none"
+                               fo:border-right="none"
+                               fo:border-top="none"
+                               fo:border-bottom="0.0154in double #808080"
+                               text:number-lines="false"
+                               text:line-number="0"
+                               style:join-border="false"/>
+   <style:text-properties fo:font-size="6pt" style:font-size-asian="6pt" style:font-size-complex="6pt"/>
+</style:style>
+<style:style style:name="Footnote_20_anchor" style:display-name="Footnote anchor"              style:family="text">    <style:text-properties style:text-position="super 58%"/> </style:style>
+<style:style style:name="TOC_Item" style:family="paragraph" style:parent-style-name="Standard">
+ <style:paragraph-properties>
+  <style:tab-stops>
+   <style:tab-stop style:position="6.7283in" style:type="right" style:leader-style="dotted" style:leader-text="."/>
+  </style:tab-stops>
+ </style:paragraph-properties>
+</style:style>
+  <text:notes-configuration text:note-class="footnote" text:default-style-name="Footnote" text:citation-style-name="Footnote_20_Symbol" text:citation-body-style-name="Footnote_20_anchor" text:master-page-name="Footnote" style:num-format="a" text:start-value="0" text:footnotes-position="page" text:start-numbering-at="page"/>
+  <text:notes-configuration text:note-class="endnote" text:default-style-name="Endnote" text:citation-style-name="Endnote_20_Symbol" text:citation-body-style-name="Endnote_20_anchor" text:master-page-name="Endnote" style:num-format="1" text:start-value="0"/>
+</office:styles>
+<office:automatic-styles>   <style:style style:name="MMD-Italic" style:family="text">
+      <style:text-properties fo:font-style="italic" style:font-style-asian="italic"
+                             style:font-style-complex="italic"/>
+   </style:style>
+   <style:style style:name="MMD-Bold" style:family="text">
+      <style:text-properties fo:font-weight="bold" style:font-weight-asian="bold"
+                             style:font-weight-complex="bold"/>
+   </style:style>
+   <style:style style:name="MMD-Superscript" style:family="text">
+      <style:text-properties style:text-position="super 58%"/>
+   </style:style>
+   <style:style style:name="MMD-Subscript" style:family="text">
+      <style:text-properties style:text-position="sub 58%"/>
+   </style:style>
+   <style:style style:name="Strike" style:family="text">
+      <style:text-properties style:text-line-through-style="solid" />
+   </style:style>
+   <style:style style:name="Underline" style:family="text">
+      <style:text-properties style:text-underline-style="solid" style:text-underline-color="font-color"/>
+   </style:style>
+   <style:style style:name="Highlight" style:family="text">
+      <style:text-properties fo:background-color="#FFFF00" />
+   </style:style>
+   <style:style style:name="Comment" style:family="text">
+      <style:text-properties fo:color="#0000BB" />
+   </style:style>
+<style:style style:name="MMD-Table" style:family="paragraph" style:parent-style-name="Standard">
+   <style:paragraph-properties fo:margin-top="0in" fo:margin-bottom="0.05in"/>
+</style:style>
+<style:style style:name="MMD-Table-Center" style:family="paragraph" style:parent-style-name="MMD-Table">
+   <style:paragraph-properties fo:text-align="center" style:justify-single-word="false"/>
+</style:style>
+<style:style style:name="MMD-Table-Right" style:family="paragraph" style:parent-style-name="MMD-Table">
+   <style:paragraph-properties fo:text-align="right" style:justify-single-word="false"/>
+</style:style>
+<style:style style:name="P2" style:family="paragraph" style:parent-style-name="Standard"
+             style:list-style-name="L2">
+<style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
+</style:style>
+<style:style style:name="fr1" style:family="graphic" style:parent-style-name="Frame">
+   <style:graphic-properties style:print-content="true" style:vertical-pos="top"
+                             style:vertical-rel="baseline"
+                             fo:padding="0in"
+                             fo:border="none"
+                             style:shadow="none"/>
+</style:style>
+<style:style style:name="P1" style:family="paragraph" style:parent-style-name="Standard"
+             style:list-style-name="L1"/>
+<text:list-style style:name="L1">
+       <text:list-level-style-bullet text:level="1" text:style-name="Numbering_20_Symbols" style:num-suffix="." text:bullet-char="•">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="0.5in" fo:text-indent="-0.25in" fo:margin-left="0.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-bullet>
+       <text:list-level-style-bullet text:level="2" text:style-name="Numbering_20_Symbols" style:num-suffix="." text:bullet-char="◦">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="0.75in" fo:text-indent="-0.25in" fo:margin-left="0.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-bullet>
+       <text:list-level-style-bullet text:level="3" text:style-name="Numbering_20_Symbols" style:num-suffix="." text:bullet-char="▪">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1in" fo:text-indent="-0.25in" fo:margin-left="1in"/>
+               </style:list-level-properties>
+       </text:list-level-style-bullet>
+       <text:list-level-style-number text:level="4" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.25in" fo:text-indent="-0.25in" fo:margin-left="1.25in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="5" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.5in" fo:text-indent="-0.25in" fo:margin-left="1.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="6" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.75in" fo:text-indent="-0.25in" fo:margin-left="1.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="7" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2in" fo:text-indent="-0.25in" fo:margin-left="2in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="8" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.25in" fo:text-indent="-0.25in" fo:margin-left="2.25in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="9" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.5in" fo:text-indent="-0.25in" fo:margin-left="2.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="10" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.75in" fo:text-indent="-0.25in" fo:margin-left="2.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+</text:list-style>
+<text:list-style style:name="L2">
+       <text:list-level-style-number text:level="1" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="0.5in" fo:text-indent="-0.25in" fo:margin-left="0.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="2" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="0.75in" fo:text-indent="-0.25in" fo:margin-left="0.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="3" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1in" fo:text-indent="-0.25in" fo:margin-left="1in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="4" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.25in" fo:text-indent="-0.25in" fo:margin-left="1.25in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="5" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.5in" fo:text-indent="-0.25in" fo:margin-left="1.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="6" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="1.75in" fo:text-indent="-0.25in" fo:margin-left="1.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="7" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2in" fo:text-indent="-0.25in" fo:margin-left="2in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="8" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.25in" fo:text-indent="-0.25in" fo:margin-left="2.25in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="9" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.5in" fo:text-indent="-0.25in" fo:margin-left="2.5in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+       <text:list-level-style-number text:level="10" text:style-name="Standard" style:num-suffix="." style:num-format="1">
+               <style:list-level-properties text:list-level-position-and-space-mode="label-alignment">
+                       <style:list-level-label-alignment text:label-followed-by="listtab" text:list-tab-stop-position="2.75in" fo:text-indent="-0.25in" fo:margin-left="2.75in"/>
+               </style:list-level-properties>
+       </text:list-level-style-number>
+</text:list-style>
+</office:automatic-styles>
+ <office:master-styles>
+  <style:master-page style:name="Endnote" >
+    <style:header><text:h text:outline-level="2">Bibliography</text:h></style:header></style:master-page>
+  <style:master-page style:name="Footnote" style:page-layout-name="pm2"/>
+ </office:master-styles>
+<office:meta>
+       <dc:title>BOM</dc:title>
+</office:meta>
+<office:body>
+<office:text>
+<text:p text:style-name="Standard">This file leads with a Byte Order Marker (BOM).</text:p>
+</office:text>
+</office:body>
+</office:document>

diff --git a/tests/MMD6Tests/BOM.html b/tests/MMD6Tests/BOM.html
new file mode 100644 (file)
index 0000000..cf82022
--- /dev/null
+++ b/tests/MMD6Tests/BOM.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+<head>
+       <meta charset="utf-8"/>
+       <title>BOM</title>
+</head>
+<body>
+
+<p>This file leads with a Byte Order Marker (BOM).</p>
+
+</body>
+</html>
+

diff --git a/tests/MMD6Tests/BOM.htmlc b/tests/MMD6Tests/BOM.htmlc
new file mode 100644 (file)
index 0000000..109eea3
--- /dev/null
+++ b/tests/MMD6Tests/BOM.htmlc
@@ -0,0 +1,4 @@
+<p>Title:      BOM
+latex config:  article</p>
+
+<p>This file leads with a Byte Order Marker (BOM).</p>

diff --git a/tests/MMD6Tests/BOM.tex b/tests/MMD6Tests/BOM.tex
new file mode 100644 (file)
index 0000000..d67d849
--- /dev/null
+++ b/tests/MMD6Tests/BOM.tex
@@ -0,0 +1,8 @@
+\input{mmd6-article-leader}
+\def\mytitle{BOM}
+\input{mmd6-article-begin}
+
+This file leads with a Byte Order Marker (BOM).
+
+\input{mmd6-article-footer}
+\end{document}

diff --git a/tests/MMD6Tests/BOM.text b/tests/MMD6Tests/BOM.text
new file mode 100644 (file)
index 0000000..0c44adb
--- /dev/null
+++ b/tests/MMD6Tests/BOM.text
@@ -0,0 +1,4 @@
+Title:      BOM
+latex config:  article
+
+This file leads with a Byte Order Marker (BOM).