Backtestshugorun-20261024-162305#15355
tpl/transform: Strip BOM prior to unmarshaling a resource or string
Our patch was judged better than the merged fix.
In this judgment, Patch B = our agent, Patch A = the merged fix.
Both patches fix the issue at the same root: trimming the UTF-8 BOM at the top of metadecoders Decoder.Unmarshal, which is the path transform.Unmarshal uses for both resources and strings. The independent tests confirm both go from FAIL-ASSERT on base to PASS for CSV, JSON, TOML, YAML and XML, as resources and as strings, so correctness on the issue's scenarios is equal. The differences are in scope hygiene, tests, and craft. Patch A mixes in unrelated reformatting of existing test calls (splitting AssertFileContent and ReplaceAll argument lists across lines), which removes eight lines and adds noise that obscures the actual fix. Patch B keeps the existing tests untouched. Patch B also factors the BOM into a named constant and a small stripBOM helper instead of an inline byte literal with a trailing comment, and applies it consistently to UnmarshalTo and UnmarshalYaml and the i18n bundle; that extra coverage is beyond the ask and earns no credit on its own, but it is low risk (a trailing-whitespace-style prefix trim) and does not complicate the core change. On tests, both add a multi-format integration test that would fail before and pass after. B additionally includes the issue's own reproduction verbatim and exercises the site.Data path and i18n files, so its tests are broader without being tautological. A's test is adequate but narrower and ships alongside churn. Weighing scope (A's unrelated churn counts against it) and craft (B's named helper and constant), B is the better patch by a modest margin.
Four judges ruled on this ticket in parallel, and the verdict comes from counting their votes. Each saw the issue and the two patches, labelled A and B in random order (two of them saw ours as A), with issue links and version numbers removed. Where tests could check the fix, they also saw the results of tests a separate agent wrote for the issue, run on both patches. None was told which patch was ours. The reasoning above is one judge's, shown verbatim.
Solve
39m 53s
Wall-clock time in the container
Full test suite
No new failures
1,241 passed · 1 skipped · 1,253 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
157vs 88 merged
Changed lines, ours vs merged
The issue
View on GitHubReference: <https://discourse.gohugo.io/t/57561>
Although the forum topic above is about CSV data, I think we'd have the same/similar problem with JSON, XML, and TOML data.
Failing example:
git clone --single-branch -b hugo-github-issue-15335 https://github.com/jmooring/hugo-testing hugo-github-issue-15335
cd hugo-github-issue-15335
hugo --ignoreCache && cat public/index.html
Expected:
city: tokyo|country: japan|
fruit: apple|color: red|
id: 1234|role: developer|
vehicle: car|type: sedan|
name: laptop|price: 999.99|
Actual:
city: |country: japan|
fruit: |color: red|
id: |role: developer|
vehicle: |type: sedan|
name: |price: 999.99|
// See issue 15355
func TestUnmarshalCSVWithBOM(t *testing.T) {
t.Parallel()
bom := "\xef\xbb\xbf"
files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/home.html --
{{ $r := resources.Get "a.csv" }}
{{ $opts := dict "targetType" "map" "format" "csv" }}
{{ $data := transform.Unmarshal $opts $r }}
{{ range $data }}
column_a: {{ .column_a }}|column_b: {{ .column_b }}|
{{ end }}
-- assets/a.csv --
` + bom + `column_a,column_b
foo,bar
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")
}The two patches
langs/i18n/i18n_integration_test.goindex 15912e6a0..653ab03c4 100644@@ -220,3 +220,37 @@ hello: Greetings from pt! runTest(strings.ReplaceAll(filesTemplate, "pt:", "PT:")) runTest(strings.ReplaceAll(filesTemplate, "-- i18n/pt.yml --", "-- i18n/pT.yml --")) }++// See issue 15355+func TestI18nWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+defaultContentLanguage = 'en'+defaultContentLanguageInSubdir = true+[languages.en]+weight = 1+[languages.de]+weight = 2+[languages.fr]+weight = 3+-- i18n/en.toml --+` + bom + `hello = 'hello from toml'+-- i18n/de.json --+` + bom + `{"hello": "hello from json"}+-- i18n/fr.yaml --+` + bom + `hello: hello from yaml+-- layouts/home.html --+{{ T "hello" }}|+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/en/index.html", "hello from toml|")+ b.AssertFileContent("public/de/index.html", "hello from json|")+ b.AssertFileContent("public/fr/index.html", "hello from yaml|")+}langs/i18n/translationProvider.goindex 5fcaeb3fd..e45afe3c8 100644@@ -14,6 +14,7 @@ package i18n import (+ "bytes" "context" "encoding/json" "fmt"@@ -48,6 +49,14 @@ func NewTranslationProvider() *TranslationProvider { return &TranslationProvider{} } +// stripBOM wraps fn so that a leading UTF-8 byte order mark is removed+// before unmarshaling.+func stripBOM(fn func([]byte, any) error) func([]byte, any) error {+ return func(data []byte, v any) error {+ return fn(bytes.TrimPrefix(data, []byte("\xef\xbb\xbf")), v)+ }+}+ // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { defaultLangTag, err := language.Parse(dst.Conf.DefaultContentLanguage())@@ -56,10 +65,10 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { } bundle := i18n.NewBundle(defaultLangTag) - bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)+ bundle.RegisterUnmarshalFunc("toml", stripBOM(toml.Unmarshal)) bundle.RegisterUnmarshalFunc("yaml", metadecoders.UnmarshalYaml) bundle.RegisterUnmarshalFunc("yml", metadecoders.UnmarshalYaml)- bundle.RegisterUnmarshalFunc("json", json.Unmarshal)+ bundle.RegisterUnmarshalFunc("json", stripBOM(json.Unmarshal)) var files []hugofs.FileMetaInfo w := hugofs.NewWalkway(parser/metadecoders/decoder.goindex 02e7dbdb7..32cf8da27 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = stripBOM(data) if len(data) == 0 { switch f { case CSV:@@ -165,6 +166,7 @@ func (d Decoder) Unmarshal(data []byte, f Format) (any, error) { // UnmarshalYaml unmarshals data in YAML format into v. func UnmarshalYaml(data []byte, v any) error {+ data = stripBOM(data) if err := yaml.Unmarshal(data, v); err != nil { return err }@@ -274,10 +276,20 @@ func validateAliasLimitForCollections(v any, limit int) error { return validate(v) } +// utf8BOM is the UTF-8 encoded byte order mark.+var utf8BOM = []byte("\xef\xbb\xbf")++// stripBOM removes a leading UTF-8 byte order mark from data, if present.+func stripBOM(data []byte) []byte {+ return bytes.TrimPrefix(data, utf8BOM)+}+ // UnmarshalTo unmarshals data in format f into v. func (d Decoder) UnmarshalTo(data []byte, f Format, v any) error { var err error + data = stripBOM(data)+ switch f { case ORG: err = d.unmarshalORG(data, v)tpl/transform/transform_integration_test.goindex 9109e6c3e..c80f9618f 100644@@ -640,3 +640,101 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355+func TestUnmarshalCSVWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $r := resources.Get "a.csv" }}+{{ $opts := dict "targetType" "map" "format" "csv" }}+{{ $data := transform.Unmarshal $opts $r }}+{{ range $data }}+column_a: {{ .column_a }}|column_b: {{ .column_b }}|+{{ end }}+-- assets/a.csv --+` + bom + `column_a,column_b+foo,bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html", "column_a: foo|column_b: bar|")+}++// See issue 15355+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ $csvOpts := dict "targetType" "map" }}+{{ with resources.Get "a.csv" | transform.Unmarshal $csvOpts }}{{ range . }}csv resource: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with resources.Get "a.json" | transform.Unmarshal }}json resource: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with resources.Get "a.toml" | transform.Unmarshal }}toml resource: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with resources.Get "a.yaml" | transform.Unmarshal }}yaml resource: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with resources.Get "a.xml" | transform.Unmarshal }}xml resource: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with (resources.Get "a.csv").Content | transform.Unmarshal $csvOpts }}{{ range . }}csv string: city: {{ .city }}|country: {{ .country }}|{{ end }}{{ end }}+{{ with (resources.Get "a.json").Content | transform.Unmarshal }}json string: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with (resources.Get "a.toml").Content | transform.Unmarshal }}toml string: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with (resources.Get "a.yaml").Content | transform.Unmarshal }}yaml string: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with (resources.Get "a.xml").Content | transform.Unmarshal }}xml string: name: {{ .name }}|price: {{ .price }}|{{ end }}+{{ with site.Data.d.json }}json data: fruit: {{ .fruit }}|color: {{ .color }}|{{ end }}+{{ with site.Data.d.toml }}toml data: id: {{ .id }}|role: {{ .role }}|{{ end }}+{{ with site.Data.d.yaml }}yaml data: vehicle: {{ .vehicle }}|type: {{ .type }}|{{ end }}+{{ with site.Data.d.xml }}xml data: name: {{ .name }}|price: {{ .price }}|{{ end }}+-- assets/a.csv --+` + bom + `city,country+tokyo,japan+-- assets/a.json --+` + bom + `{"fruit": "apple", "color": "red"}+-- assets/a.toml --+` + bom + `id = 1234+role = "developer"+-- assets/a.yaml --+` + bom + `vehicle: car+type: sedan+-- assets/a.xml --+` + bom + `<?xml version="1.0" encoding="UTF-8"?>+<root><name>laptop</name><price>999.99</price></root>+-- data/d/json.json --+` + bom + `{"fruit": "apple", "color": "red"}+-- data/d/toml.toml --+` + bom + `id = 1234+role = "developer"+-- data/d/yaml.yaml --+` + bom + `vehicle: car+type: sedan+-- data/d/xml.xml --+` + bom + `<?xml version="1.0" encoding="UTF-8"?>+<root><name>laptop</name><price>999.99</price></root>+`++ b := hugolib.Test(t, files)++ b.AssertFileContent("public/index.html",+ "csv resource: city: tokyo|country: japan|",+ "json resource: fruit: apple|color: red|",+ "toml resource: id: 1234|role: developer|",+ "yaml resource: vehicle: car|type: sedan|",+ "xml resource: name: laptop|price: 999.99|",+ "csv string: city: tokyo|country: japan|",+ "json string: fruit: apple|color: red|",+ "toml string: id: 1234|role: developer|",+ "yaml string: vehicle: car|type: sedan|",+ "xml string: name: laptop|price: 999.99|",+ "json data: fruit: apple|color: red|",+ "toml data: id: 1234|role: developer|",+ "yaml data: vehicle: car|type: sedan|",+ "xml data: name: laptop|price: 999.99|",+ )+}
parser/metadecoders/decoder.goindex 02e7dbdb7..dda9576c6 100644@@ -142,6 +142,7 @@ func (d Decoder) UnmarshalStringTo(data string, typ any) (any, error) { // Unmarshal will unmarshall data in format f into an interface{}. // This is what's needed for Hugo's /data handling. func (d Decoder) Unmarshal(data []byte, f Format) (any, error) {+ data = bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) // UTF-8 BOM if len(data) == 0 { switch f { case CSV:tpl/transform/transform_integration_test.goindex 9109e6c3e..057a6cc9c 100644@@ -124,7 +124,8 @@ code:{{ transform.Highlight "" (dict "type" "go" "code" "i = 42") }} want := `<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go"><span class="line"><span class="cl"><span class="nx">i</span><span class="w"> </span><span class="p">=</span><span class="w"> </span><span class="mi">42</span></span></span></code></pre></div>` - b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", "lang:"+want, "type:"+want, "code:"+want,@@ -418,11 +419,13 @@ DATA // targetType = map f := strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b := hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[{"age":"3","breed":"Collie","name":"Spot","type":"dog"},{"age":"7","breed":"Malicious","name":"Felix","type":"cat"}]`, ) @@ -434,11 +437,13 @@ DATA // targetType = slice f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "slice"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -450,11 +455,13 @@ DATA // targetType not specified f = strings.ReplaceAll(files, "OPTS", "dict")- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,type,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) b = hugolib.Test(t, f)- b.AssertFileContent("public/index.html",+ b.AssertFileContent(+ "public/index.html", `[["name","type","breed","age"],["Spot","dog","Collie","3"],["Felix","cat","Malicious","7"]]`, ) @@ -503,7 +510,8 @@ DATA // targetType = map (error: header row contains duplicate field names) f = strings.ReplaceAll(files, "OPTS", `dict "targetType" "map"`)- f = strings.ReplaceAll(f, "DATA",+ f = strings.ReplaceAll(+ f, "DATA", "name,name,breed,age\nSpot,dog,Collie,3\nFelix,cat,Malicious,7", ) _, err = hugolib.TestE(t, f)@@ -640,3 +648,66 @@ i: &i [*h, *h, *h, *h, *h, *h, *h, *h, *h, *h] b.Assert(err, qt.IsNotNil) b.Assert(err.Error(), qt.Contains, "too many YAML aliases for non-scalar nodes") }++// See issue 15355.+func TestUnmarshalWithBOM(t *testing.T) {+ t.Parallel()++ bom := "\xef\xbb\xbf"++ files := `+-- hugo.toml --+disableKinds = ['page','rss','section','sitemap','taxonomy','term']+-- layouts/home.html --+{{ with resources.Get "data.csv" | transform.Unmarshal (dict "targetType" "map") }}+ {{ range . }}+ csv: {{ .column_a }}|{{ .column_b }}|+ {{ end }}+{{ end }}++{{ with resources.Get "data.json" | transform.Unmarshal }}+ json: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.toml" | transform.Unmarshal }}+ toml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.xml" | transform.Unmarshal }}+ xml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with resources.Get "data.yaml" | transform.Unmarshal }}+ yaml: {{ .column_a }}|{{ .column_b }}|+{{ end }}++{{ with (resources.Get "data.json").Content | transform.Unmarshal }}+ string: {{ .column_a }}|{{ .column_b }}|+{{ end }}+-- assets/data.csv --+` + bom + `column_a,column_b+foo,bar+-- assets/data.json --+` + bom + `{"column_a": "foo", "column_b": "bar"}+-- assets/data.toml --+` + bom + `column_a = "foo"+column_b = "bar"+-- assets/data.xml --+` + bom + `<root><column_a>foo</column_a><column_b>bar</column_b></root>+-- assets/data.yaml --+` + bom + `column_a: foo+column_b: bar+`++ b := hugolib.Test(t, files)++ b.AssertFileContent(+ "public/index.html",+ "csv: foo|bar|",+ "json: foo|bar|",+ "toml: foo|bar|",+ "xml: foo|bar|",+ "yaml: foo|bar|",+ "string: foo|bar|",+ )+}