Backtestshugorun-20261013-011256#15355

Issue #15355 Merged fix PR #15357 Run 13 Oct 2026 · run-20261013-011256 gohugoio/hugo

tpl/transform: Strip BOM prior to unmarshaling a resource or string

Better

Our patch was judged better than the merged fix.

In this judgment, Patch B = our agent, Patch A = the merged fix.

Both patches strip the UTF-8 BOM in `metadecoders.Decoder.Unmarshal`, which fixes the reported resource cases.

Patch B is more robust in three ways:

1. **String inputs.** It strips the BOM in `tpl/transform/unmarshal.go` before the `TrimSpace` emptiness check and before format detection on the raw string. A BOM-only string is then treated as empty, and detection does not see a leading BOM. Patch A relies only on the decoder-level strip, so string format detection may still see the BOM.
2. **Other entry points.** It also strips the BOM in `UnmarshalTo`, `UnmarshalYaml`, and `Remarshal`, so all decoding paths behave the same.
3. **Shared helper.** It adds a named `StripBOM` helper instead of an inline magic byte slice.

The i18n change is adjacent hardening. It is low-risk and tested.

**Tests.** Patch B's tests are much stronger:
- Unit tests across JSON, TOML, YAML, XML, CSV, ORG and empty input, plus `UnmarshalToMap` and `UnmarshalYaml`.
- Integration tests for every format via both resource and string input, Remarshal, BOM-only edge cases, and the issue's exact CSV test.
- Data-file and i18n tests.

Patch A has a single integration test, which is reasonable. However, it also adds unrelated formatting churn to existing tests by reflowing `AssertFileContent` and `ReplaceAll` calls.

On correctness, test coverage and craft, Patch B is better.

The judge saw only the two patches, labelled A and B in random order, with issue links and version numbers removed. The reasoning above is shown verbatim.

Solve

40m 14s

Wall-clock time in the container

Full test suite

No new failures

1,195 passed · 24 skipped · 1,224 total

Contamination audit

Clean

No sign the agent saw the real fix

Patch size

216vs 88 merged

Changed lines, ours vs merged

Reference: <https://discourse.gohugo.io/t/57561>

Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.

Failing example:

git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html

Expected:

city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|

Actual:

city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|

// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
	t.Parallel()

	bom := "\xef\xbb\xbf"

	files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`

	b := hugolib.Test(t, files)

	b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}

The two patches

Our patch+213 −3
langs/i18n/i18n_integration_test.goindex 15912e6a0..936868439 100644@@ -220,3 +220,36 @@ hello: Greetings from pt! 	runTest(strings.ReplaceAll(filesTemplate, "pt:", "PT:")) 	runTest(strings.ReplaceAll(filesTemplate, "-- i18n/pt.yml --", "-- i18n/pT.yml --")) }++// Issue 15355+func TestI18nFilesWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+defaultContentLanguage = 'en'+[languages.en]+weight = 1+[languages.de]+weight = 2+[languages.fr]+weight = 3+-- i18n/en.toml --+` + bom + `hello = "Hello from en!"+-- i18n/de.json --+` + bom + `{"hello": "Hello from de!"}+-- i18n/fr.yaml --+` + bom + `hello: Hello from fr!+-- layouts/home.html --+{{ T "hello" }}|+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html", "Hello from en!|")+	b.AssertFileContent("public/de/index.html", "Hello from de!|")+	b.AssertFileContent("public/fr/index.html", "Hello from fr!|")+}langs/i18n/translationProvider.goindex 5fcaeb3fd..133735a39 100644@@ -56,10 +56,14 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { 	} 	bundle := i18n.NewBundle(defaultLangTag) -	bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)+	bundle.RegisterUnmarshalFunc("toml", func(data []byte, v any) error {+		return toml.Unmarshal(metadecoders.StripBOM(data), v)+	}) 	bundle.RegisterUnmarshalFunc("yaml", metadecoders.UnmarshalYaml) 	bundle.RegisterUnmarshalFunc("yml", metadecoders.UnmarshalYaml)-	bundle.RegisterUnmarshalFunc("json", json.Unmarshal)+	bundle.RegisterUnmarshalFunc("json", func(data []byte, v any) error {+		return json.Unmarshal(metadecoders.StripBOM(data), v)+	})  	var files []hugofs.FileMetaInfo 	w := hugofs.NewWalkway(parser/metadecoders/decoder.goindex 02e7dbdb7..79372a37d 100644@@ -139,9 +139,18 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { 	} } +// utf8BOM is the UTF-8 encoded byte order mark.+var utf8BOM = []byte("\xef\xbb\xbf")++// StripBOM removes a leading UTF-8 byte order mark from data, if present.+func StripBOM(data []byte) []byte {+	return bytes.TrimPrefix(data, utf8BOM)+}+ // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = StripBOM(data) 	if len(data) == 0 { 		switch f { 		case CSV:@@ -165,6 +174,7 @@ func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {  // UnmarshalYaml unmarshals data in YAML format into v. func UnmarshalYaml(data []byte, v any) error {+	data = StripBOM(data) 	if err := yaml.Unmarshal(data, v); err != nil { 		return err 	}@@ -276,6 +286,8 @@ func validateAliasLimitForCollections(v any, limit int) error {  // UnmarshalTo unmarshals data in format f into v. func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error {+	data = StripBOM(data)+ 	var err error  	switch f {parser/metadecoders/decoder_test.goindex 678197f7d..92d10eca0 100644@@ -281,3 +281,37 @@ params: 		}) 	} }++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+	c := qt.New(t)++	bom := "\xef\xbb\xbf"++	for _, test := range []struct {+		data   string+		format Format+		expect any+	}{+		{`{"a": "b"}`, JSON, map[string]any{"a": "b"}},+		{`a = "b"`, TOML, map[string]any{"a": "b"}},+		{`a: b`, YAML, map[string]any{"a": "b"}},+		{`<root><a>b</a></root>`, XML, map[string]any{"a": "b"}},+		{"a,b\nc,d", CSV, [][]string{{"a", "b"}, {"c", "d"}}},+		{"#+a: b", ORG, map[string]any{"a": "b"}},+		{"", JSON, map[string]any{}},+	} {+		msg := qt.Commentf("format %s", test.format)+		v, err := Default.Unmarshal([]byte(bom+test.data), test.format)+		c.Assert(err, qt.IsNil, msg)+		c.Assert(v, qt.DeepEquals, test.expect, msg)+	}++	m, err := Default.UnmarshalToMap([]byte(bom+`a = "b"`), TOML)+	c.Assert(err, qt.IsNil)+	c.Assert(m, qt.DeepEquals, map[string]any{"a": "b"})++	var y map[string]any+	c.Assert(UnmarshalYaml([]byte(bom+"a: b"), &y), qt.IsNil)+	c.Assert(y, qt.DeepEquals, map[string]any{"a": "b"})+}tpl/transform/remarshal.goindex 2fbdacd2d..9e5e44b08 100644@@ -34,7 +34,7 @@ func (ns *Namespace) Remarshal(format string, data any) (string, error) { 			return "", err 		} -		from = strings.TrimSpace(from)+		from = strings.TrimSpace(strings.TrimPrefix(from, "\ufeff")) 		if from == "" { 			return "", nil 		}tpl/transform/transform_integration_test.goindex 9109e6c3e..f58ec60dd 100644@@ -640,3 +640,127 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// Issue 15355+func TestUnmarshalCSVWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $r := resources.Get "a.csv" }}+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ $data := transform.Unmarshal $opts $r }}+{{ range $data }}+column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")+}++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $csvOpts := dict "targetType" "map" }}+{{ with transform.Unmarshal $csvOpts (resources.Get "a.csv") }}{{ range . }}csv: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with transform.Unmarshal (resources.Get "a.json") }}json: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.toml") }}toml: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.yaml") }}yaml: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.xml") }}xml: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.json").Content) }}json string: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.toml").Content) }}toml string: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.yaml").Content) }}yaml string: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.xml").Content) }}xml string: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with transform.Unmarshal $csvOpts (printf "%s" (resources.Get "a.csv").Content) }}{{ range . }}csv string: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with transform.Remarshal "json" (printf "%s" (resources.Get "a.toml").Content) }}remarshal: {{ . | safeHTML }}{{ end }}+bom only unmarshal: {{ transform.Unmarshal (printf "%s" (resources.Get "bom.txt").Content) }}|+bom only remarshal: {{ transform.Remarshal "json" (printf "%s" (resources.Get "bom.txt").Content) }}|+-- assets/bom.txt --+` + bom + `+-- assets/a.csv --+` + bom + `city,country+tokyo,japan+-- assets/a.json --+` + bom + `{"fruit": "apple", "color": "red"}+-- assets/a.toml --+` + bom + `id = 1234+role = "developer"+-- assets/a.yaml --+` + bom + `vehicle: car+type: sedan+-- assets/a.xml --+` + bom + `<?xml version="1.0" encoding="UTF-8"?>+<root><name>laptop</name><price>999.99</price></root>+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html",+		"csv: city: tokyo|country: japan|",+		"json: fruit: apple|color: red|",+		"toml: id: 1234|role: developer|",+		"yaml: vehicle: car|type: sedan|",+		"xml: name: laptop|price: 999.99|",+		"json string: fruit: apple|color: red|",+		"toml string: id: 1234|role: developer|",+		"yaml string: vehicle: car|type: sedan|",+		"xml string: name: laptop|price: 999.99|",+		"csv string: city: tokyo|country: japan|",+		`remarshal: {+  "id": 1234,+  "role": "developer"+}`,+		"bom only unmarshal: |",+		"bom only remarshal: |",+	)+}++// Issue 15355+func TestDataFilesWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+json: fruit: {{ hugo.Data.j.fruit }}|+toml: id: {{ hugo.Data.t.id }}|+yaml: vehicle: {{ hugo.Data.y.vehicle }}|+xml: name: {{ hugo.Data.x.name }}|+-- data/j.json --+` + bom + `{"fruit": "apple"}+-- data/t.toml --+` + bom + `id = 1234+-- data/y.yaml --+` + bom + `vehicle: car+-- data/x.xml --+` + bom + `<root><name>laptop</name></root>+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html",+		"json: fruit: apple|",+		"toml: id: 1234|",+		"yaml: vehicle: car|",+		"xml: name: laptop|",+	)+}tpl/transform/unmarshal.goindex 2a2a526dd..14ded618a 100644@@ -121,6 +121,9 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) { 		return nil, fmt.Errorf("type %T not supported", data) 	} +	// Strip any leading UTF-8 BOM so that a BOM-only string is treated as empty.+	dataStr = strings.TrimPrefix(dataStr, "\ufeff")+ 	if strings.TrimSpace(dataStr) == "" { 		return nil, nil 	} 
Merged fix · PR #15357+80 −8
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM 	if len(data) == 0 { 		switch f { 		case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }}  	want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` -	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		"lang:"+want, 		"type:"+want, 		"code:"+want,@@ -418,11 +419,13 @@ DATA  	// targetType = map 	f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b := hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, 	) @@ -434,11 +437,13 @@ DATA  	// targetType = slice 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -450,11 +455,13 @@ DATA  	// targetType not specified 	f = strings.ReplaceAll(files, "OPTS", "dict")-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -503,7 +510,8 @@ DATA  	// targetType = map (error: header row contains duplicate field names) 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	_, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+  {{ range . }}+    csv: {{ .column_a }}|{{ .column_b }}|+  {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+  json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+  toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+  xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+  yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+  string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent(+		"public/index.html",+		"csv: foo|bar|",+		"json: foo|bar|",+		"toml: foo|bar|",+		"xml: foo|bar|",+		"yaml: foo|bar|",+		"string: foo|bar|",+	)+}