Backtestshugorun-20261013-011256#15355
tpl/transform: Strip BOM prior to unmarshaling a resource or string
Better
Our patch was judged better than the merged fix.
In this judgment, Patch B = our agent, Patch A = the merged fix.
Both patches strip the UTF-8 BOM in `metadecoders.Decoder.Unmarshal`, which fixes the reported resource cases. Patch B is more robust in three ways: 1. **String inputs.** It strips the BOM in `tpl/transform/unmarshal.go` before the `TrimSpace` emptiness check and before format detection on the raw string. A BOM-only string is then treated as empty, and detection does not see a leading BOM. Patch A relies only on the decoder-level strip, so string format detection may still see the BOM. 2. **Other entry points.** It also strips the BOM in `UnmarshalTo`, `UnmarshalYaml`, and `Remarshal`, so all decoding paths behave the same. 3. **Shared helper.** It adds a named `StripBOM` helper instead of an inline magic byte slice. The i18n change is adjacent hardening. It is low-risk and tested. **Tests.** Patch B's tests are much stronger: - Unit tests across JSON, TOML, YAML, XML, CSV, ORG and empty input, plus `UnmarshalToMap` and `UnmarshalYaml`. - Integration tests for every format via both resource and string input, Remarshal, BOM-only edge cases, and the issue's exact CSV test. - Data-file and i18n tests. Patch A has a single integration test, which is reasonable. However, it also adds unrelated formatting churn to existing tests by reflowing `AssertFileContent` and `ReplaceAll` calls. On correctness, test coverage and craft, Patch B is better.
The judge saw only the two patches, labelled A and B in random order, with issue links and version numbers removed. The reasoning above is shown verbatim.
Solve
40m 14s
Wall-clock time in the container
Full test suite
No new failures
1,195 passed · 24 skipped · 1,224 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
216vs 88 merged
Changed lines, ours vs merged
The issue
View on GitHubReference: <https://discourse.gohugo.io/t/57561>
Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.
Failing example:
git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html
Expected:
city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|
Actual:
city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|
// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
t.Parallel()
bom := "\xef\xbb\xbf"
files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}The two patches
Our patch+213 −3
langs/i18n/i18n_integration_test.goindex 15912e6a0..936868439 100644@@ -220,3 +220,36 @@ hello: Greetings from pt! runTest(strings.ReplaceAll(filesTemplate, "pt:", "PT:")) runTest(strings.ReplaceAll(filesTemplate, "-- i18n/pt.yml --", "-- i18n/pT.yml --")) }++// Issue 15355+func TestI18nFilesWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+defaultContentLanguage = 'en'+[languages.en]+weight = 1+[languages.de]+weight = 2+[languages.fr]+weight = 3+-- i18n/en.toml --+` + bom + `hello = "Hello from en!"+-- i18n/de.json --+` + bom + `{"hello": "Hello from de!"}+-- i18n/fr.yaml --+` + bom + `hello: Hello from fr!+-- layouts/home.html --+{{ T "hello" }}|+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html", "Hello from en!|")+ b.AssertFileContent("public/de/index.html", "Hello from de!|")+ b.AssertFileContent("public/fr/index.html", "Hello from fr!|")+}langs/i18n/translationProvider.goindex 5fcaeb3fd..133735a39 100644@@ -56,10 +56,14 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { } bundle := i18n.NewBundle(defaultLangTag) - bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)+ bundle.RegisterUnmarshalFunc("toml", func(data []byte, v any) error {+ return toml.Unmarshal(metadecoders.StripBOM(data), v)+ }) bundle.RegisterUnmarshalFunc("yaml", metadecoders.UnmarshalYaml) bundle.RegisterUnmarshalFunc("yml", metadecoders.UnmarshalYaml)- bundle.RegisterUnmarshalFunc("json", json.Unmarshal)+ bundle.RegisterUnmarshalFunc("json", func(data []byte, v any) error {+ return json.Unmarshal(metadecoders.StripBOM(data), v)+ }) var files []hugofs.FileMetaInfo w := hugofs.NewWalkway(parser/metadecoders/decoder.goindex 02e7dbdb7..79372a37d 100644@@ -139,9 +139,18 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { } } +// utf8BOM is the UTF-8 encoded byte order mark.+var utf8BOM = []byte("\xef\xbb\xbf")++// StripBOM removes a leading UTF-8 byte order mark from data, if present.+func StripBOM(data []byte) []byte {+ return bytes.TrimPrefix(data, utf8BOM)+}+ // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = StripBOM(data) if len(data) == 0 { switch f { case CSV:@@ -165,6 +174,7 @@ func (d Decoder) Unmarshal(data []byte, f Format) (any, error) { // UnmarshalYaml unmarshals data in YAML format into v. func UnmarshalYaml(data []byte, v any) error {+ data = StripBOM(data) if err := yaml.Unmarshal(data, v); err != nil { return err }@@ -276,6 +286,8 @@ func validateAliasLimitForCollections(v any, limit int) error { // UnmarshalTo unmarshals data in format f into v. func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error {+ data = StripBOM(data)+ var err error switch f {parser/metadecoders/decoder_test.goindex 678197f7d..92d10eca0 100644@@ -281,3 +281,37 @@ params: }) } }++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+ c := qt.New(t)++ bom := "\xef\xbb\xbf"++ for _, test := range []struct {+ data string+ format Format+ expect any+ }{+ {`{"a": "b"}`, JSON, map[string]any{"a": "b"}},+ {`a = "b"`, TOML, map[string]any{"a": "b"}},+ {`a: b`, YAML, map[string]any{"a": "b"}},+ {`<root><a>b</a></root>`, XML, map[string]any{"a": "b"}},+ {"a,b\nc,d", CSV, [][]string{{"a", "b"}, {"c", "d"}}},+ {"#+a: b", ORG, map[string]any{"a": "b"}},+ {"", JSON, map[string]any{}},+ } {+ msg := qt.Commentf("format %s", test.format)+ v, err := Default.Unmarshal([]byte(bom+test.data), test.format)+ c.Assert(err, qt.IsNil, msg)+ c.Assert(v, qt.DeepEquals, test.expect, msg)+ }++ m, err := Default.UnmarshalToMap([]byte(bom+`a = "b"`), TOML)+ c.Assert(err, qt.IsNil)+ c.Assert(m, qt.DeepEquals, map[string]any{"a": "b"})++ var y map[string]any+ c.Assert(UnmarshalYaml([]byte(bom+"a: b"), &y), qt.IsNil)+ c.Assert(y, qt.DeepEquals, map[string]any{"a": "b"})+}tpl/transform/remarshal.goindex 2fbdacd2d..9e5e44b08 100644@@ -34,7 +34,7 @@ func (ns *Namespace) Remarshal(format string, data any) (string, error) { return "", err } - from = strings.TrimSpace(from)+ from = strings.TrimSpace(strings.TrimPrefix(from, "\ufeff")) if from == "" { return "", nil }tpl/transform/transform_integration_test.goindex 9109e6c3e..f58ec60dd 100644@@ -640,3 +640,127 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// Issue 15355+func TestUnmarshalCSVWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $r := resources.Get "a.csv" }}+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ $data := transform.Unmarshal $opts $r }}+{{ range $data }}+column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")+}++// Issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $csvOpts := dict "targetType" "map" }}+{{ with transform.Unmarshal $csvOpts (resources.Get "a.csv") }}{{ range . }}csv: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with transform.Unmarshal (resources.Get "a.json") }}json: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.toml") }}toml: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.yaml") }}yaml: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with transform.Unmarshal (resources.Get "a.xml") }}xml: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.json").Content) }}json string: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.toml").Content) }}toml string: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.yaml").Content) }}yaml string: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with transform.Unmarshal (printf "%s" (resources.Get "a.xml").Content) }}xml string: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with transform.Unmarshal $csvOpts (printf "%s" (resources.Get "a.csv").Content) }}{{ range . }}csv string: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with transform.Remarshal "json" (printf "%s" (resources.Get "a.toml").Content) }}remarshal: {{ . | safeHTML }}{{ end }}+bom only unmarshal: {{ transform.Unmarshal (printf "%s" (resources.Get "bom.txt").Content) }}|+bom only remarshal: {{ transform.Remarshal "json" (printf "%s" (resources.Get "bom.txt").Content) }}|+-- assets/bom.txt --+` + bom + `+-- assets/a.csv --+` + bom + `city,country+tokyo,japan+-- assets/a.json --+` + bom + `{"fruit": "apple", "color": "red"}+-- assets/a.toml --+` + bom + `id = 1234+role = "developer"+-- assets/a.yaml --+` + bom + `vehicle: car+type: sedan+-- assets/a.xml --+` + bom + `<?xml version="1.0" encoding="UTF-8"?>+<root><name>laptop</name><price>999.99</price></root>+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html",+ "csv: city: tokyo|country: japan|",+ "json: fruit: apple|color: red|",+ "toml: id: 1234|role: developer|",+ "yaml: vehicle: car|type: sedan|",+ "xml: name: laptop|price: 999.99|",+ "json string: fruit: apple|color: red|",+ "toml string: id: 1234|role: developer|",+ "yaml string: vehicle: car|type: sedan|",+ "xml string: name: laptop|price: 999.99|",+ "csv string: city: tokyo|country: japan|",+ `remarshal: {+ "id": 1234,+ "role": "developer"+}`,+ "bom only unmarshal: |",+ "bom only remarshal: |",+ )+}++// Issue 15355+func TestDataFilesWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+json: fruit: {{ hugo.Data.j.fruit }}|+toml: id: {{ hugo.Data.t.id }}|+yaml: vehicle: {{ hugo.Data.y.vehicle }}|+xml: name: {{ hugo.Data.x.name }}|+-- data/j.json --+` + bom + `{"fruit": "apple"}+-- data/t.toml --+` + bom + `id = 1234+-- data/y.yaml --+` + bom + `vehicle: car+-- data/x.xml --+` + bom + `<root><name>laptop</name></root>+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html",+ "json: fruit: apple|",+ "toml: id: 1234|",+ "yaml: vehicle: car|",+ "xml: name: laptop|",+ )+}tpl/transform/unmarshal.goindex 2a2a526dd..14ded618a 100644@@ -121,6 +121,9 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) { return nil, fmt.Errorf("type %T not supported", data) } + // Strip any leading UTF-8 BOM so that a BOM-only string is treated as empty.+ dataStr = strings.TrimPrefix(dataStr, "\ufeff")+ if strings.TrimSpace(dataStr) == "" { return nil, nil }
Merged fix · PR #15357+80 −8
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM if len(data) == 0 { switch f { case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }} want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` - b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", "lang:"+want, "type:"+want, "code:"+want,@@ -418,11 +419,13 @@ DATA // targetType = map f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b := hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, ) @@ -434,11 +437,13 @@ DATA // targetType = slice f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -450,11 +455,13 @@ DATA // targetType not specified f = strings.ReplaceAll(files, "OPTS", "dict")- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -503,7 +510,8 @@ DATA // targetType = map (error: header row contains duplicate field names) f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) _, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+ {{ range . }}+ csv: {{ .column_a }}|{{ .column_b }}|+ {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+ json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+ toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+ xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+ yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+ string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent(+ "public/index.html",+ "csv: foo|bar|",+ "json: foo|bar|",+ "toml: foo|bar|",+ "xml: foo|bar|",+ "yaml: foo|bar|",+ "string: foo|bar|",+ )+}