Backtestshugorun-20261009-153657#15355

Issue #15355 Merged fix PR #15357 Run 9 Oct 2026 · run-20261009-153657 gohugoio/hugo

tpl/transform: Strip BOM prior to unmarshaling a resource or string

Better

Our patch was judged better than the merged fix.

In this judgment, Patch B = our agent, Patch A = the merged fix.

Both patches fix the reported problem at the same root: they trim the UTF-8 BOM inside metadecoders.Decoder.Unmarshal, which transform.Unmarshal uses for both resources and strings. Format detection in FormatFromContentString is index-based, so the CSV/JSON/TOML/XML/YAML cases in the issue work under either patch.

Patch B is more robust at the edges:
- It also trims in UnmarshalTo and UnmarshalYaml, so callers that bypass Decoder.Unmarshal are covered.
- It trims the string input in tpl/transform/unmarshal.go before the empty/whitespace check. A BOM-only string therefore returns nil, as an empty string would. Under Patch A, that string would fall through to format detection and likely error.
- It extends the same handling to i18n bundles (TOML/JSON/YAML), which the issue's 'same/similar problem' framing suggests. The added scope is low risk and is factored into a small, well-named TrimBOM helper instead of a repeated inline byte literal.

On tests, Patch B is clearly stronger:
- Unit tests in decoder_test.go cover every format, including ORG and empty-after-BOM.
- Integration tests cover data files, i18n, and transform.Unmarshal for both resources and strings in every format, plus the BOM-only case.
- Patch A has a single integration test that exercises only one string format.

Patch A also adds unrelated whitespace reformatting of existing test calls, which is noise in the diff. Overall, B is the better fix.

The judge saw only the two patches, labelled A and B in random order, with issue links and version numbers removed. The reasoning above is shown verbatim.

Solve

23m 58s

Wall-clock time in the container

Full test suite

No new failures

1,198 passed · 24 skipped · 1,222 total

Contamination audit

Clean

No sign the agent saw the real fix

Patch size

159vs 88 merged

Changed lines, ours vs merged

Reference: <https://discourse.gohugo.io/t/57561>

Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.

Failing example:

git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html

Expected:

city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|

Actual:

city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|

// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
	t.Parallel()

	bom := "\xef\xbb\xbf"

	files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`

	b := hugolib.Test(t, files)

	b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}

The two patches

Our patch+157 −2
langs/i18n/i18n_integration_test.goindex 15912e6a0..341ae60aa 100644@@ -220,3 +220,37 @@ hello: Greetings from pt! 	runTest(strings.ReplaceAll(filesTemplate, "pt:", "PT:")) 	runTest(strings.ReplaceAll(filesTemplate, "-- i18n/pt.yml --", "-- i18n/pT.yml --")) }++// Issue 15355+func TestI18nWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+defaultContentLanguage = 'en'+defaultContentLanguageInSubdir = true+[languages.en]+weight = 1+[languages.de]+weight = 2+[languages.fr]+weight = 3+-- layouts/home.html --+hello: {{ T "hello" }}|+-- i18n/en.toml --+` + bom + `hello = "Hello"+-- i18n/de.json --+` + bom + `{"hello": "Hallo"}+-- i18n/fr.yaml --+` + bom + `hello: Bonjour+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/en/index.html", "hello: Hello|")+	b.AssertFileContent("public/de/index.html", "hello: Hallo|")+	b.AssertFileContent("public/fr/index.html", "hello: Bonjour|")+}langs/i18n/translationProvider.goindex 5fcaeb3fd..45a23cfe8 100644@@ -56,10 +56,14 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { 	} 	bundle := i18n.NewBundle(defaultLangTag) -	bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)+	bundle.RegisterUnmarshalFunc("toml", func(data []byte, v any) error {+		return toml.Unmarshal(metadecoders.TrimBOM(data), v)+	}) 	bundle.RegisterUnmarshalFunc("yaml", metadecoders.UnmarshalYaml) 	bundle.RegisterUnmarshalFunc("yml", metadecoders.UnmarshalYaml)-	bundle.RegisterUnmarshalFunc("json", json.Unmarshal)+	bundle.RegisterUnmarshalFunc("json", func(data []byte, v any) error {+		return json.Unmarshal(metadecoders.TrimBOM(data), v)+	})  	var files []hugofs.FileMetaInfo 	w := hugofs.NewWalkway(parser/metadecoders/decoder.goindex 02e7dbdb7..f838a2cba 100644@@ -70,6 +70,14 @@ func (d Decoder) OptionsKey() string { 	return sb.String() } +// utf8BOM is the UTF-8 encoded Unicode byte order mark.+var utf8BOM = []byte("\xef\xbb\xbf")++// TrimBOM removes a leading UTF-8 byte order mark from data, if present.+func TrimBOM(data []byte) []byte {+	return bytes.TrimPrefix(data, utf8BOM)+}+ // Default is a Decoder in its default configuration. var Default = Decoder{ 	Delimiter:  ',',@@ -142,6 +150,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = TrimBOM(data) 	if len(data) == 0 { 		switch f { 		case CSV:@@ -165,6 +174,7 @@ func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {  // UnmarshalYaml unmarshals data in YAML format into v. func UnmarshalYaml(data []byte, v any) error {+	data = TrimBOM(data) 	if err := yaml.Unmarshal(data, v); err != nil { 		return err 	}@@ -278,6 +288,8 @@ func validateAliasLimitForCollections(v any, limit int) error { func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error { 	var err error +	data = TrimBOM(data)+ 	switch f { 	case ORG: 		err = d.unmarshalORG(data, v)parser/metadecoders/decoder_integration_test.goindex 9da1ee17d..eb0c3ddfb 100644@@ -77,3 +77,37 @@ Filtered: {{ $filtered }}|  	b.AssertFileContent("public/index.html", "[map[name:two weight:2] map[name:three weight:3] map[name:four weight:4]]|") }++// Issue 15355+func TestDataFilesWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+json: {{ site.Data.a.column_a }}|+toml: {{ site.Data.b.column_a }}|+yaml: {{ site.Data.c.column_a }}|+xml: {{ site.Data.d.column_a }}|+-- data/a.json --+` + bom + `{"column_a": "foo"}+-- data/b.toml --+` + bom + `column_a = "foo"+-- data/c.yaml --+` + bom + `column_a: foo+-- data/d.xml --+` + bom + `<root><column_a>foo</column_a></root>+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html",+		"json: foo|",+		"toml: foo|",+		"yaml: foo|",+		"xml: foo|",+	)+}parser/metadecoders/decoder_test.goindex 678197f7d..1f43ea72e 100644@@ -140,6 +140,15 @@ func TestUnmarshalToInterface(t *testing.T) { 		{[]byte(`<root><a>b</a></root>`), XML, expect}, 		{[]byte(`a,b,c`), CSV, [][]string{{"a", "b", "c"}}}, 		{[]byte("a: Easy!\nb:\n  c: 2\n  d: [3, 4]"), YAML, map[string]any{"a": "Easy!", "b": map[string]any{"c": uint64(2), "d": []any{uint64(3), uint64(4)}}}},+		// Leading UTF-8 byte order mark, see issue 15355.+		{[]byte("\xef\xbb\xbf{ \"a\": \"b\" }"), JSON, expect},+		{[]byte("\xef\xbb\xbfa = \"b\""), TOML, expect},+		{[]byte("\xef\xbb\xbfa: \"b\""), YAML, expect},+		{[]byte("\xef\xbb\xbf<root><a>b</a></root>"), XML, expect},+		{[]byte("\xef\xbb\xbfa,b,c"), CSV, [][]string{{"a", "b", "c"}}},+		{[]byte("\xef\xbb\xbf#+a: b"), ORG, expect},+		{[]byte("\xef\xbb\xbf"), JSON, map[string]any{}},+		{[]byte("\xef\xbb\xbf"), CSV, [][]string{}}, 		// errors 		{[]byte(`a = "`), TOML, false}, 	} {tpl/transform/transform_integration_test.goindex 9109e6c3e..db34a95d9 100644@@ -640,3 +640,63 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ range transform.Unmarshal $opts (resources.Get "a.csv") }}+csv: column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+{{ with transform.Unmarshal (resources.Get "a.json") }}json: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.toml") }}toml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.xml") }}xml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.yaml") }}yaml: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ range transform.Unmarshal $opts ((resources.Get "a.csv").Content) }}+csv string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.json").Content) }}json string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.toml").Content) }}toml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.xml").Content) }}xml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+{{ with transform.Unmarshal ((resources.Get "a.yaml").Content) }}yaml string: column_a: {{ .column_a }}|column_b: {{ .column_b }}|{{ end }}+bom only string: {{ transform.Unmarshal ((resources.Get "bomonly.txt").Content) }}|+-- assets/bomonly.txt --+` + bom + `+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/a.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/a.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/a.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/a.yaml --+` + bom + `column_a: foo+column_b: bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html",+		"csv: column_a: foo|column_b: bar|",+		"json: column_a: foo|column_b: bar|",+		"toml: column_a: foo|column_b: bar|",+		"xml: column_a: foo|column_b: bar|",+		"yaml: column_a: foo|column_b: bar|",+		"csv string: column_a: foo|column_b: bar|",+		"json string: column_a: foo|column_b: bar|",+		"toml string: column_a: foo|column_b: bar|",+		"xml string: column_a: foo|column_b: bar|",+		"yaml string: column_a: foo|column_b: bar|",+		"bom only string: |",+	)+}tpl/transform/unmarshal.goindex 2a2a526dd..f0c4e88bd 100644@@ -121,6 +121,8 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) { 		return nil, fmt.Errorf("type %T not supported", data) 	} +	dataStr = strings.TrimPrefix(dataStr, "\xef\xbb\xbf")+ 	if strings.TrimSpace(dataStr) == "" { 		return nil, nil 	} 
Merged fix · PR #15357+80 −8
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM 	if len(data) == 0 { 		switch f { 		case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }}  	want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` -	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		"lang:"+want, 		"type:"+want, 		"code:"+want,@@ -418,11 +419,13 @@ DATA  	// targetType = map 	f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b := hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, 	) @@ -434,11 +437,13 @@ DATA  	// targetType = slice 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -450,11 +455,13 @@ DATA  	// targetType not specified 	f = strings.ReplaceAll(files, "OPTS", "dict")-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -503,7 +510,8 @@ DATA  	// targetType = map (error: header row contains duplicate field names) 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	_, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+  {{ range . }}+    csv: {{ .column_a }}|{{ .column_b }}|+  {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+  json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+  toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+  xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+  yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+  string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent(+		"public/index.html",+		"csv: foo|bar|",+		"json: foo|bar|",+		"toml: foo|bar|",+		"xml: foo|bar|",+		"yaml: foo|bar|",+		"string: foo|bar|",+	)+}