Backtestshugorun-20261009-153657#15355
tpl/transform: Strip BOM prior to unmarshaling a resource or string
Better
Our patch was judged better than the merged fix.
In this judgment, Patch B = our agent, Patch A = the merged fix.
Both patches fix the reported problem at the same root: they trim the UTF-8 BOM inside metadecoders.Decoder.Unmarshal, which transform.Unmarshal uses for both resources and strings. Format detection in FormatFromContentString is index-based, so the CSV/JSON/TOML/XML/YAML cases in the issue work under either patch. Patch B is more robust at the edges: - It also trims in UnmarshalTo and UnmarshalYaml, so callers that bypass Decoder.Unmarshal are covered. - It trims the string input in tpl/transform/unmarshal.go before the empty/whitespace check. A BOM-only string therefore returns nil, as an empty string would. Under Patch A, that string would fall through to format detection and likely error. - It extends the same handling to i18n bundles (TOML/JSON/YAML), which the issue's 'same/similar problem' framing suggests. The added scope is low risk and is factored into a small, well-named TrimBOM helper instead of a repeated inline byte literal. On tests, Patch B is clearly stronger: - Unit tests in decoder_test.go cover every format, including ORG and empty-after-BOM. - Integration tests cover data files, i18n, and transform.Unmarshal for both resources and strings in every format, plus the BOM-only case. - Patch A has a single integration test that exercises only one string format. Patch A also adds unrelated whitespace reformatting of existing test calls, which is noise in the diff. Overall, B is the better fix.
The judge saw only the two patches, labelled A and B in random order, with issue links and version numbers removed. The reasoning above is shown verbatim.
Solve
23m 58s
Wall-clock time in the container
Full test suite
No new failures
1,198 passed · 24 skipped · 1,222 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
159vs 88 merged
Changed lines, ours vs merged
The issue
View on GitHubReference: <https://discourse.gohugo.io/t/57561>
Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.
Failing example:
git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html
Expected:
city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|
Actual:
city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|
// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
t.Parallel()
bom := "\xef\xbb\xbf"
files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}The two patches
Our patch+157 −2
langs/i18n/i18n_integration_test.goindex 15912e6a0..341ae60aa 100644@@ -220,3 +220,37 @@ hello: Greetings from pt! runTest(strings.ReplaceAll(filesTemplate, "pt:", "PT:")) runTest(strings.ReplaceAll(filesTemplate, "-- i18n/pt.yml --", "-- i18n/pT.yml --")) }++// Issue 15355+func TestI18nWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+defaultContentLanguage = 'en'+defaultContentLanguageInSubdir = true+[languages.en]+weight = 1+[languages.de]+weight = 2+[languages.fr]+weight = 3+-- layouts/home.html --+hello: {{ T "hello" }}|+-- i18n/en.toml --+` + bom + `hello = "Hello"+-- i18n/de.json --+` + bom + `{"hello": "Hallo"}+-- i18n/fr.yaml --+` + bom + `hello: Bonjour+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/en/index.html", "hello: Hello|")+ b.AssertFileContent("public/de/index.html", "hello: Hallo|")+ b.AssertFileContent("public/fr/index.html", "hello: Bonjour|")+}langs/i18n/translationProvider.goindex 5fcaeb3fd..45a23cfe8 100644@@ -56,10 +56,14 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { } bundle := i18n.NewBundle(defaultLangTag) - bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)+ bundle.RegisterUnmarshalFunc("toml", func(data []byte, v any) error {+ return toml.Unmarshal(metadecoders.TrimBOM(data), v)+ }) bundle.RegisterUnmarshalFunc("yaml", metadecoders.UnmarshalYaml) bundle.RegisterUnmarshalFunc("yml", metadecoders.UnmarshalYaml)- bundle.RegisterUnmarshalFunc("json", json.Unmarshal)+ bundle.RegisterUnmarshalFunc("json", func(data []byte, v any) error {+ return json.Unmarshal(metadecoders.TrimBOM(data), v)+ }) var files []hugofs.FileMetaInfo w := hugofs.NewWalkway(parser/metadecoders/decoder.goindex 02e7dbdb7..f838a2cba 100644@@ -70,6 +70,14 @@ func (d Decoder) OptionsKey() string { return sb.String() } +// utf8BOM is the UTF-8 encoded Unicode byte order mark.+var utf8BOM = []byte("\xef\xbb\xbf")++// TrimBOM removes a leading UTF-8 byte order mark from data, if present.+func TrimBOM(data []byte) []byte {+ return bytes.TrimPrefix(data, utf8BOM)+}+ // Default is a Decoder in its default configuration. var Default = Decoder{ Delimiter: ',',@@ -142,6 +150,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = TrimBOM(data) if len(data) == 0 { switch f { case CSV:@@ -165,6 +174,7 @@ func (d Decoder) Unmarshal(data []byte, f Format) (any, error) { // UnmarshalYaml unmarshals data in YAML format into v. func UnmarshalYaml(data []byte, v any) error {+ data = TrimBOM(data) if err := yaml.Unmarshal(data, v); err != nil { return err }@@ -278,6 +288,8 @@ func validateAliasLimitForCollections(v any, limit int) error { func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error { var err error + data = TrimBOM(data)+ switch f { case ORG: err = d.unmarshalORG(data, v)parser/metadecoders/decoder_integration_test.goindex 9da1ee17d..eb0c3ddfb 100644@@ -77,3 +77,37 @@ Filtered: {{ $filtered }}| b.AssertFileContent("public/index.html", "[map[name:two weight:2] map[name:three weight:3] map[name:four weight:4]]|") }++// Issue 15355+func TestDataFilesWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+json: {{ site.Data.a.column_a }}|+toml: {{ site.Data.b.column_a }}|+yaml: {{ site.Data.c.column_a }}|+xml: {{ site.Data.d.column_a }}|+-- data/a.json --+` + bom + `{"column_a": "foo"}+-- data/b.toml --+` + bom + `column_a = "foo"+-- data/c.yaml --+` + bom + `column_a: foo+-- data/d.xml --+` + bom + `<root><column_a>foo</column_a></root>+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html",+ "json: foo|",+ "toml: foo|",+ "yaml: foo|",+ "xml: foo|",+ )+}parser/metadecoders/decoder_test.goindex 678197f7d..1f43ea72e 100644@@ -140,6 +140,15 @@ func TestUnmarshalToInterface(t *testing.T) { {[]byte(`<root><a>b</a></root>`), XML, expect}, {[]byte(`a,b,c`), CSV, [][]string{{"a", "b", "c"}}}, {[]byte("a: Easy!\nb:\n c: 2\n d: [3, 4]"), YAML, map[string]any{"a": "Easy!", "b": map[string]any{"c": uint64(2), "d": []any{uint64(3), uint64(4)}}}},+ // Leading UTF-8 byte order mark, see issue 15355.+ {[]byte("\xef\xbb\xbf{ \"a\": \"b\" }"), JSON, expect},+ {[]byte("\xef\xbb\xbfa = \"b\""), TOML, expect},+ {[]byte("\xef\xbb\xbfa: \"b\""), YAML, expect},+ {[]byte("\xef\xbb\xbf<root><a>b</a></root>"), XML, expect},+ {[]byte("\xef\xbb\xbfa,b,c"), CSV, [][]string{{"a", "b", "c"}}},+ {[]byte("\xef\xbb\xbf#+a: b"), ORG, expect},+ {[]byte("\xef\xbb\xbf"), JSON, map[string]any{}},+ {[]byte("\xef\xbb\xbf"), CSV, [][]string{}}, // errors {[]byte(`a = "`), TOML, false}, } {tpl/transform/transform_integration_test.goindex 9109e6c3e..db34a95d9 100644@@ -640,3 +640,63 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ range transform.Unmarshal $opts (resources.Get "a.csv") }}+csv: column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+{{ with transform.Unmarshal (resources.Get "a.json") }}json: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.toml") }}toml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.xml") }}xml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.yaml") }}yaml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ range transform.Unmarshal $opts ((resources.Get "a.csv").Content) }}+csv string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.json").Content) }}json string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.toml").Content) }}toml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.xml").Content) }}xml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.yaml").Content) }}yaml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+bom only string: {{ transform.Unmarshal ((resources.Get "bomonly.txt").Content) }}|+-- assets/bomonly.txt --+` + bom + `+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/a.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/a.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/a.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/a.yaml --+` + bom + `column_a: foo+column_b: bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html",+ "csv: column_a: foo|column_b: bar|",+ "json: column_a: foo|column_b: bar|",+ "toml: column_a: foo|column_b: bar|",+ "xml: column_a: foo|column_b: bar|",+ "yaml: column_a: foo|column_b: bar|",+ "csv string: column_a: foo|column_b: bar|",+ "json string: column_a: foo|column_b: bar|",+ "toml string: column_a: foo|column_b: bar|",+ "xml string: column_a: foo|column_b: bar|",+ "yaml string: column_a: foo|column_b: bar|",+ "bom only string: |",+ )+}tpl/transform/unmarshal.goindex 2a2a526dd..f0c4e88bd 100644@@ -121,6 +121,8 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) { return nil, fmt.Errorf("type %T not supported", data) } + dataStr = strings.TrimPrefix(dataStr, "\xef\xbb\xbf")+ if strings.TrimSpace(dataStr) == "" { return nil, nil }
Merged fix · PR #15357+80 −8
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM if len(data) == 0 { switch f { case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }} want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` - b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", "lang:"+want, "type:"+want, "code:"+want,@@ -418,11 +419,13 @@ DATA // targetType = map f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b := hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, ) @@ -434,11 +437,13 @@ DATA // targetType = slice f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -450,11 +455,13 @@ DATA // targetType not specified f = strings.ReplaceAll(files, "OPTS", "dict")- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -503,7 +510,8 @@ DATA // targetType = map (error: header row contains duplicate field names) f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) _, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+ {{ range . }}+ csv: {{ .column_a }}|{{ .column_b }}|+ {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+ json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+ toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+ xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+ yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+ string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent(+ "public/index.html",+ "csv: foo|bar|",+ "json: foo|bar|",+ "toml: foo|bar|",+ "xml: foo|bar|",+ "yaml: foo|bar|",+ "string: foo|bar|",+ )+}