Backtestshugorun-20261007-020926#15355

Issue #15355 Merged fix PR #15357 Run 7 Oct 2026 · run-20261007-020926 gohugoio/hugo

tpl/transform: Strip BOM prior to unmarshaling a resource or string

Better

Our patch was judged better than the merged fix.

In this judgment, Patch A = our agent, Patch B = the merged fix.

Both patches fix the root cause: they strip the UTF-8 BOM in `metadecoders.Decoder.Unmarshal`, which is the path `transform.Unmarshal` uses for both resources and strings. That resolves the reported CSV/JSON/TOML/XML/YAML failures.

Patch A is more thorough. It also strips the BOM in `UnmarshalTo` (used by `UnmarshalToMap` and `UnmarshalFileToMap`, so front matter, config and data paths are covered) and in `UnmarshalStringTo`, so every decoder entry point behaves the same. The helper and constant are named clearly and cite the issue.

Patch A's tests are stronger. It adds unit tests for `UnmarshalStringTo` with a BOM, the issue's CSV integration test, and a broader integration test. That test covers every format as a resource, every format as a string via `.Content`, and BOM-prefixed files in the `data/` directory through `hugo.Data`. Patch B's test covers each format as a resource and only JSON as a string.

Patch B also reformats several unrelated existing test calls (`AssertFileContent` and `strings.ReplaceAll` argument wrapping). That churn obscures the diff without adding value. Its fix is minimal and correct for the issue, but narrower.

Overall Patch A has comparable correctness, broader consistency, better tests and no unrelated churn.

The judge saw only the two patches, labelled A and B in random order, with issue links and version numbers removed. The reasoning above is shown verbatim.

Solve

46m 28s

Wall-clock time in the container

Full test suite

No new failures

3,772 passed · 101 skipped · 3,875 total

Contamination audit

Clean

No sign the agent saw the real fix

Patch size

109vs 88 merged

Changed lines, ours vs merged

Reference: <https://discourse.gohugo.io/t/57561>

Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.

Failing example:

git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html

Expected:

city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|

Actual:

city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|

// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
	t.Parallel()

	bom := "\xef\xbb\xbf"

	files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`

	b := hugolib.Test(t, files)

	b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}

The two patches

Our patch+108 −1
parser/metadecoders/decoder.goindex 02e7dbdb7..818bd45f7 100644@@ -70,6 +70,15 @@ func (d Decoder) OptionsKey() string { 	return sb.String() } +// utf8BOM is the UTF-8 encoded byte order mark.+const utf8BOM = "\ufeff"++// trimBOM removes a leading UTF-8 byte order mark from data, if present.+// See issue 15335.+func trimBOM(data []byte) []byte {+	return bytes.TrimPrefix(data, []byte(utf8BOM))+}+ // Default is a Decoder in its default configuration. var Default = Decoder{ 	Delimiter:  ',',@@ -115,7 +124,7 @@ func (d Decoder) UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]an  // UnmarshalStringTo tries to unmarshal data to a new instance of type typ. func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) {-	data = strings.TrimSpace(data)+	data = strings.TrimSpace(strings.TrimPrefix(data, utf8BOM)) 	// We only check for the possible types in YAML, JSON and TOML. 	switch typ.(type) { 	case string:@@ -142,6 +151,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = trimBOM(data) 	if len(data) == 0 { 		switch f { 		case CSV:@@ -278,6 +288,8 @@ func validateAliasLimitForCollections(v any, limit int) error { func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error { 	var err error +	data = trimBOM(data)+ 	switch f { 	case ORG: 		err = d.unmarshalORG(data, v)parser/metadecoders/decoder_test.goindex 678197f7d..6b3941a23 100644@@ -174,6 +174,10 @@ func TestUnmarshalStringTo(t *testing.T) { 		{"3.14159", float64(1), float64(3.14159)}, 		{"[3,7,9]", []any{}, []any{uint64(3), uint64(7), uint64(9)}}, 		{"[3.1,7.2,9.3]", []any{}, []any{3.1, 7.2, 9.3}},+		// See issue 15335.+		{"\ufeff{ \"a\": \"b\" }", make(map[string]any), expectMap},+		{"\ufeffa string", "string", "a string"},+		{"\ufeff32", int(1234), int(32)}, 	} { 		msg := qt.Commentf("%d: %T", i, test.to) 		m, err := d.UnmarshalStringTo(test.data, test.to)tpl/transform/transform_integration_test.goindex 9109e6c3e..4efe18ef0 100644@@ -640,3 +640,94 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15335.+func TestUnmarshalCSVWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $r := resources.Get "a.csv" }}+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ $data := transform.Unmarshal $opts $r }}+{{ range $data }}+column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")+}++// See issue 15335.+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ range $f := slice "a.csv" "a.json" "a.toml" "a.xml" "a.yaml" }}+{{ $r := resources.Get $f }}+{{ $opts := dict }}+{{ if eq $f "a.csv" }}{{ $opts = dict "targetType" "map" }}{{ end }}+{{ $data := transform.Unmarshal $opts $r }}+{{ if eq $f "a.csv" }}{{ $data = index $data 0 }}{{ end }}+resource {{ $f }}: city: {{ $data.city }}|country: {{ $data.country }}|+{{ $data = transform.Unmarshal $opts $r.Content }}+{{ if eq $f "a.csv" }}{{ $data = index $data 0 }}{{ end }}+string {{ $f }}: city: {{ $data.city }}|country: {{ $data.country }}|+{{ end }}+{{ range $k, $v := hugo.Data }}+data {{ $k }}: city: {{ $v.city }}|country: {{ $v.country }}|+{{ end }}+-- assets/a.csv --+` + bom + `city,country+tokyo,japan+-- assets/a.json --+` + bom + `{"city": "tokyo", "country": "japan"}+-- assets/a.toml --+` + bom + `city = "tokyo"+country = "japan"+-- assets/a.xml --+` + bom + `<root><city>tokyo</city><country>japan</country></root>+-- assets/a.yaml --+` + bom + `city: tokyo+country: japan+-- data/djson.json --+` + bom + `{"city": "tokyo", "country": "japan"}+-- data/dtoml.toml --+` + bom + `city = "tokyo"+country = "japan"+-- data/dxml.xml --+` + bom + `<root><city>tokyo</city><country>japan</country></root>+-- data/dyaml.yaml --+` + bom + `city: tokyo+country: japan+`++	b := hugolib.Test(t, files)++	var expect []string+	for _, f := range []string{"a.csv", "a.json", "a.toml", "a.xml", "a.yaml"} {+		expect = append(expect,+			"resource "+f+": city: tokyo|country: japan|",+			"string "+f+": city: tokyo|country: japan|",+		)+	}+	for _, k := range []string{"djson", "dtoml", "dxml", "dyaml"} {+		expect = append(expect, "data "+k+": city: tokyo|country: japan|")+	}++	b.AssertFileContent("public/index.html", expect...)+} 
Merged fix · PR #15357+80 −8
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+	data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM 	if len(data) == 0 { 		switch f { 		case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }}  	want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` -	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		"lang:"+want, 		"type:"+want, 		"code:"+want,@@ -418,11 +419,13 @@ DATA  	// targetType = map 	f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b := hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, 	) @@ -434,11 +437,13 @@ DATA  	// targetType = slice 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -450,11 +455,13 @@ DATA  	// targetType not specified 	f = strings.ReplaceAll(files, "OPTS", "dict")-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	b = hugolib.Test(t, f)-	b.AssertFileContent("public/index.html",+	b.AssertFileContent(+		"public/index.html", 		`[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, 	) @@ -503,7 +510,8 @@ DATA  	// targetType = map (error: header row contains duplicate field names) 	f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)-	f = strings.ReplaceAll(f, "DATA",+	f = strings.ReplaceAll(+		f, "DATA", 		"name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", 	) 	_, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] 	b.Assert(err, qt.IsNotNil) 	b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+	t.Parallel()++	bom := "\xef\xbb\xbf"++	files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+  {{ range . }}+    csv: {{ .column_a }}|{{ .column_b }}|+  {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+  json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+  toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+  xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+  yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+  string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++	b := hugolib.Test(t, files)++	b.AssertFileContent(+		"public/index.html",+		"csv: foo|bar|",+		"json: foo|bar|",+		"toml: foo|bar|",+		"xml: foo|bar|",+		"yaml: foo|bar|",+		"string: foo|bar|",+	)+}