Browse Source
* update go text, include text/display * [feature] Set instance langs, show post lang on frontend * go fmt * WebGet * set language for whole article, don't use FA icon * mention instance languages + other optional config vars * little tweak * put languages in config properly * warn log language parse * change some naming around * tidy up validate a bit * lint * rename LanguageTmpl in templatepull/2365/head
tobi
2 years ago
committed by
GitHub
73 changed files with 55005 additions and 141 deletions
@ -0,0 +1,184 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package language |
||||
|
||||
import ( |
||||
"github.com/superseriousbusiness/gotosocial/internal/gtserror" |
||||
"golang.org/x/text/language" |
||||
"golang.org/x/text/language/display" |
||||
) |
||||
|
||||
var namer display.Namer |
||||
|
||||
// InitLangs parses languages from the
|
||||
// given slice of tags, and sets the `namer`
|
||||
// display.Namer for the instance.
|
||||
//
|
||||
// This function should only be called once,
|
||||
// since setting the namer is not thread safe.
|
||||
func InitLangs(tagStrs []string) (Languages, error) { |
||||
var ( |
||||
languages = make(Languages, len(tagStrs)) |
||||
tags = make([]language.Tag, len(tagStrs)) |
||||
) |
||||
|
||||
// Reset namer.
|
||||
namer = nil |
||||
|
||||
// Parse all tags first.
|
||||
for i, tagStr := range tagStrs { |
||||
tag, err := language.Parse(tagStr) |
||||
if err != nil { |
||||
return nil, gtserror.Newf( |
||||
"error parsing %s as BCP47 language tag: %w", |
||||
tagStr, err, |
||||
) |
||||
} |
||||
tags[i] = tag |
||||
} |
||||
|
||||
// Check if we can set a namer.
|
||||
if len(tags) != 0 { |
||||
namer = display.Languages(tags[0]) |
||||
} |
||||
|
||||
// Fall namer back to English.
|
||||
if namer == nil { |
||||
namer = display.Languages(language.English) |
||||
} |
||||
|
||||
// Parse nice language models from tags
|
||||
// (this will use the namer we just set).
|
||||
for i, tag := range tags { |
||||
languages[i] = ParseTag(tag) |
||||
} |
||||
|
||||
return languages, nil |
||||
} |
||||
|
||||
// Language models a BCP47 language tag
|
||||
// along with helper strings for the tag.
|
||||
type Language struct { |
||||
// BCP47 language tag
|
||||
Tag language.Tag |
||||
// Normalized string
|
||||
// of BCP47 tag.
|
||||
TagStr string |
||||
// Human-readable
|
||||
// language name(s).
|
||||
DisplayStr string |
||||
} |
||||
|
||||
// MarshalText implements encoding.TextMarshaler{}.
|
||||
func (l *Language) MarshalText() ([]byte, error) { |
||||
return []byte(l.TagStr), nil |
||||
} |
||||
|
||||
// UnmarshalText implements encoding.TextUnmarshaler{}.
|
||||
func (l *Language) UnmarshalText(text []byte) error { |
||||
lang, err := Parse(string(text)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
*l = *lang |
||||
return nil |
||||
} |
||||
|
||||
type Languages []*Language |
||||
|
||||
func (l Languages) Tags() []language.Tag { |
||||
tags := make([]language.Tag, len(l)) |
||||
for i, lang := range l { |
||||
tags[i] = lang.Tag |
||||
} |
||||
|
||||
return tags |
||||
} |
||||
|
||||
func (l Languages) TagStrs() []string { |
||||
tagStrs := make([]string, len(l)) |
||||
for i, lang := range l { |
||||
tagStrs[i] = lang.TagStr |
||||
} |
||||
|
||||
return tagStrs |
||||
} |
||||
|
||||
func (l Languages) DisplayStrs() []string { |
||||
displayStrs := make([]string, len(l)) |
||||
for i, lang := range l { |
||||
displayStrs[i] = lang.DisplayStr |
||||
} |
||||
|
||||
return displayStrs |
||||
} |
||||
|
||||
// ParseTag parses and nicely formats the input language BCP47 tag,
|
||||
// returning a Language with ready-to-use display and tag strings.
|
||||
func ParseTag(tag language.Tag) *Language { |
||||
l := new(Language) |
||||
l.Tag = tag |
||||
l.TagStr = tag.String() |
||||
|
||||
var ( |
||||
// Our name for the language.
|
||||
name string |
||||
// Language's name for itself.
|
||||
selfName = display.Self.Name(tag) |
||||
) |
||||
|
||||
// Try to use namer
|
||||
// (if initialized).
|
||||
if namer != nil { |
||||
name = namer.Name(tag) |
||||
} |
||||
|
||||
switch { |
||||
case name == "": |
||||
// We don't have a name for
|
||||
// this language, just use
|
||||
// its own name for itself.
|
||||
l.DisplayStr = selfName |
||||
|
||||
case name == selfName: |
||||
// Avoid repeating ourselves:
|
||||
// showing "English (English)"
|
||||
// is not useful.
|
||||
l.DisplayStr = name |
||||
|
||||
default: |
||||
// Include our name for the
|
||||
// language, and its own
|
||||
// name for itself.
|
||||
l.DisplayStr = name + " " + "(" + selfName + ")" |
||||
} |
||||
|
||||
return l |
||||
} |
||||
|
||||
// Parse parses and nicely formats the input language BCP47 tag,
|
||||
// returning a Language with ready-to-use display and tag strings.
|
||||
func Parse(lang string) (*Language, error) { |
||||
tag, err := language.Parse(lang) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return ParseTag(tag), nil |
||||
} |
||||
@ -0,0 +1,142 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package language_test |
||||
|
||||
import ( |
||||
"slices" |
||||
"testing" |
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/language" |
||||
golanguage "golang.org/x/text/language" |
||||
) |
||||
|
||||
func TestInstanceLangs(t *testing.T) { |
||||
for i, test := range []struct { |
||||
InstanceLangs []string |
||||
expectedLangs []golanguage.Tag |
||||
expectedLangStrs []string |
||||
expectedErr error |
||||
parseDisplayLang string |
||||
expectedDisplayLang string |
||||
}{ |
||||
{ |
||||
InstanceLangs: []string{"en-us", "fr"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.AmericanEnglish, |
||||
golanguage.French, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"American English", |
||||
"French (français)", |
||||
}, |
||||
parseDisplayLang: "de", |
||||
expectedDisplayLang: "German (Deutsch)", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{"fr", "en-us"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.French, |
||||
golanguage.AmericanEnglish, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"français", |
||||
"anglais américain (American English)", |
||||
}, |
||||
parseDisplayLang: "de", |
||||
expectedDisplayLang: "allemand (Deutsch)", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{}, |
||||
expectedLangs: []golanguage.Tag{}, |
||||
expectedLangStrs: []string{}, |
||||
parseDisplayLang: "de", |
||||
expectedDisplayLang: "German (Deutsch)", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{"zh"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.Chinese, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"中文", |
||||
}, |
||||
parseDisplayLang: "de", |
||||
expectedDisplayLang: "德语 (Deutsch)", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{"ar", "en"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.Arabic, |
||||
golanguage.English, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"العربية", |
||||
"الإنجليزية (English)", |
||||
}, |
||||
parseDisplayLang: "fi", |
||||
expectedDisplayLang: "الفنلندية (suomi)", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{"en-us"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.AmericanEnglish, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"American English", |
||||
}, |
||||
parseDisplayLang: "en-us", |
||||
expectedDisplayLang: "American English", |
||||
}, |
||||
{ |
||||
InstanceLangs: []string{"en-us"}, |
||||
expectedLangs: []golanguage.Tag{ |
||||
golanguage.AmericanEnglish, |
||||
}, |
||||
expectedLangStrs: []string{ |
||||
"American English", |
||||
}, |
||||
parseDisplayLang: "en-gb", |
||||
expectedDisplayLang: "British English", |
||||
}, |
||||
} { |
||||
languages, err := language.InitLangs(test.InstanceLangs) |
||||
if err != test.expectedErr { |
||||
t.Errorf("test %d expected error %v, got %v", i, test.expectedErr, err) |
||||
} |
||||
|
||||
parsedTags := languages.Tags() |
||||
if !slices.Equal(test.expectedLangs, parsedTags) { |
||||
t.Errorf("test %d expected language tags %v, got %v", i, test.expectedLangs, parsedTags) |
||||
} |
||||
|
||||
parsedLangStrs := languages.DisplayStrs() |
||||
if !slices.Equal(test.expectedLangStrs, parsedLangStrs) { |
||||
t.Errorf("test %d expected language strings %v, got %v", i, test.expectedLangStrs, parsedLangStrs) |
||||
} |
||||
|
||||
parsedLang, err := language.Parse(test.parseDisplayLang) |
||||
if err != nil { |
||||
t.Errorf("unexpected error %v", err) |
||||
return |
||||
} |
||||
|
||||
if test.expectedDisplayLang != parsedLang.DisplayStr { |
||||
t.Errorf("test %d expected to parse language %v, got %v", i, test.expectedDisplayLang, parsedLang.DisplayStr) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,41 @@
|
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package format contains types for defining language-specific formatting of
|
||||
// values.
|
||||
//
|
||||
// This package is internal now, but will eventually be exposed after the API
|
||||
// settles.
|
||||
package format // import "golang.org/x/text/internal/format"
|
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"golang.org/x/text/language" |
||||
) |
||||
|
||||
// State represents the printer state passed to custom formatters. It provides
|
||||
// access to the fmt.State interface and the sentence and language-related
|
||||
// context.
|
||||
type State interface { |
||||
fmt.State |
||||
|
||||
// Language reports the requested language in which to render a message.
|
||||
Language() language.Tag |
||||
|
||||
// TODO: consider this and removing rune from the Format method in the
|
||||
// Formatter interface.
|
||||
//
|
||||
// Verb returns the format variant to render, analogous to the types used
|
||||
// in fmt. Use 'v' for the default or only variant.
|
||||
// Verb() rune
|
||||
|
||||
// TODO: more info:
|
||||
// - sentence context such as linguistic features passed by the translator.
|
||||
} |
||||
|
||||
// Formatter is analogous to fmt.Formatter.
|
||||
type Formatter interface { |
||||
Format(state State, verb rune) |
||||
} |
||||
@ -0,0 +1,358 @@
|
||||
// Copyright 2017 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package format |
||||
|
||||
import ( |
||||
"reflect" |
||||
"unicode/utf8" |
||||
) |
||||
|
||||
// A Parser parses a format string. The result from the parse are set in the
|
||||
// struct fields.
|
||||
type Parser struct { |
||||
Verb rune |
||||
|
||||
WidthPresent bool |
||||
PrecPresent bool |
||||
Minus bool |
||||
Plus bool |
||||
Sharp bool |
||||
Space bool |
||||
Zero bool |
||||
|
||||
// For the formats %+v %#v, we set the plusV/sharpV flags
|
||||
// and clear the plus/sharp flags since %+v and %#v are in effect
|
||||
// different, flagless formats set at the top level.
|
||||
PlusV bool |
||||
SharpV bool |
||||
|
||||
HasIndex bool |
||||
|
||||
Width int |
||||
Prec int // precision
|
||||
|
||||
// retain arguments across calls.
|
||||
Args []interface{} |
||||
// retain current argument number across calls
|
||||
ArgNum int |
||||
|
||||
// reordered records whether the format string used argument reordering.
|
||||
Reordered bool |
||||
// goodArgNum records whether the most recent reordering directive was valid.
|
||||
goodArgNum bool |
||||
|
||||
// position info
|
||||
format string |
||||
startPos int |
||||
endPos int |
||||
Status Status |
||||
} |
||||
|
||||
// Reset initializes a parser to scan format strings for the given args.
|
||||
func (p *Parser) Reset(args []interface{}) { |
||||
p.Args = args |
||||
p.ArgNum = 0 |
||||
p.startPos = 0 |
||||
p.Reordered = false |
||||
} |
||||
|
||||
// Text returns the part of the format string that was parsed by the last call
|
||||
// to Scan. It returns the original substitution clause if the current scan
|
||||
// parsed a substitution.
|
||||
func (p *Parser) Text() string { return p.format[p.startPos:p.endPos] } |
||||
|
||||
// SetFormat sets a new format string to parse. It does not reset the argument
|
||||
// count.
|
||||
func (p *Parser) SetFormat(format string) { |
||||
p.format = format |
||||
p.startPos = 0 |
||||
p.endPos = 0 |
||||
} |
||||
|
||||
// Status indicates the result type of a call to Scan.
|
||||
type Status int |
||||
|
||||
const ( |
||||
StatusText Status = iota |
||||
StatusSubstitution |
||||
StatusBadWidthSubstitution |
||||
StatusBadPrecSubstitution |
||||
StatusNoVerb |
||||
StatusBadArgNum |
||||
StatusMissingArg |
||||
) |
||||
|
||||
// ClearFlags reset the parser to default behavior.
|
||||
func (p *Parser) ClearFlags() { |
||||
p.WidthPresent = false |
||||
p.PrecPresent = false |
||||
p.Minus = false |
||||
p.Plus = false |
||||
p.Sharp = false |
||||
p.Space = false |
||||
p.Zero = false |
||||
|
||||
p.PlusV = false |
||||
p.SharpV = false |
||||
|
||||
p.HasIndex = false |
||||
} |
||||
|
||||
// Scan scans the next part of the format string and sets the status to
|
||||
// indicate whether it scanned a string literal, substitution or error.
|
||||
func (p *Parser) Scan() bool { |
||||
p.Status = StatusText |
||||
format := p.format |
||||
end := len(format) |
||||
if p.endPos >= end { |
||||
return false |
||||
} |
||||
afterIndex := false // previous item in format was an index like [3].
|
||||
|
||||
p.startPos = p.endPos |
||||
p.goodArgNum = true |
||||
i := p.startPos |
||||
for i < end && format[i] != '%' { |
||||
i++ |
||||
} |
||||
if i > p.startPos { |
||||
p.endPos = i |
||||
return true |
||||
} |
||||
// Process one verb
|
||||
i++ |
||||
|
||||
p.Status = StatusSubstitution |
||||
|
||||
// Do we have flags?
|
||||
p.ClearFlags() |
||||
|
||||
simpleFormat: |
||||
for ; i < end; i++ { |
||||
c := p.format[i] |
||||
switch c { |
||||
case '#': |
||||
p.Sharp = true |
||||
case '0': |
||||
p.Zero = !p.Minus // Only allow zero padding to the left.
|
||||
case '+': |
||||
p.Plus = true |
||||
case '-': |
||||
p.Minus = true |
||||
p.Zero = false // Do not pad with zeros to the right.
|
||||
case ' ': |
||||
p.Space = true |
||||
default: |
||||
// Fast path for common case of ascii lower case simple verbs
|
||||
// without precision or width or argument indices.
|
||||
if 'a' <= c && c <= 'z' && p.ArgNum < len(p.Args) { |
||||
if c == 'v' { |
||||
// Go syntax
|
||||
p.SharpV = p.Sharp |
||||
p.Sharp = false |
||||
// Struct-field syntax
|
||||
p.PlusV = p.Plus |
||||
p.Plus = false |
||||
} |
||||
p.Verb = rune(c) |
||||
p.ArgNum++ |
||||
p.endPos = i + 1 |
||||
return true |
||||
} |
||||
// Format is more complex than simple flags and a verb or is malformed.
|
||||
break simpleFormat |
||||
} |
||||
} |
||||
|
||||
// Do we have an explicit argument index?
|
||||
i, afterIndex = p.updateArgNumber(format, i) |
||||
|
||||
// Do we have width?
|
||||
if i < end && format[i] == '*' { |
||||
i++ |
||||
p.Width, p.WidthPresent = p.intFromArg() |
||||
|
||||
if !p.WidthPresent { |
||||
p.Status = StatusBadWidthSubstitution |
||||
} |
||||
|
||||
// We have a negative width, so take its value and ensure
|
||||
// that the minus flag is set
|
||||
if p.Width < 0 { |
||||
p.Width = -p.Width |
||||
p.Minus = true |
||||
p.Zero = false // Do not pad with zeros to the right.
|
||||
} |
||||
afterIndex = false |
||||
} else { |
||||
p.Width, p.WidthPresent, i = parsenum(format, i, end) |
||||
if afterIndex && p.WidthPresent { // "%[3]2d"
|
||||
p.goodArgNum = false |
||||
} |
||||
} |
||||
|
||||
// Do we have precision?
|
||||
if i+1 < end && format[i] == '.' { |
||||
i++ |
||||
if afterIndex { // "%[3].2d"
|
||||
p.goodArgNum = false |
||||
} |
||||
i, afterIndex = p.updateArgNumber(format, i) |
||||
if i < end && format[i] == '*' { |
||||
i++ |
||||
p.Prec, p.PrecPresent = p.intFromArg() |
||||
// Negative precision arguments don't make sense
|
||||
if p.Prec < 0 { |
||||
p.Prec = 0 |
||||
p.PrecPresent = false |
||||
} |
||||
if !p.PrecPresent { |
||||
p.Status = StatusBadPrecSubstitution |
||||
} |
||||
afterIndex = false |
||||
} else { |
||||
p.Prec, p.PrecPresent, i = parsenum(format, i, end) |
||||
if !p.PrecPresent { |
||||
p.Prec = 0 |
||||
p.PrecPresent = true |
||||
} |
||||
} |
||||
} |
||||
|
||||
if !afterIndex { |
||||
i, afterIndex = p.updateArgNumber(format, i) |
||||
} |
||||
p.HasIndex = afterIndex |
||||
|
||||
if i >= end { |
||||
p.endPos = i |
||||
p.Status = StatusNoVerb |
||||
return true |
||||
} |
||||
|
||||
verb, w := utf8.DecodeRuneInString(format[i:]) |
||||
p.endPos = i + w |
||||
p.Verb = verb |
||||
|
||||
switch { |
||||
case verb == '%': // Percent does not absorb operands and ignores f.wid and f.prec.
|
||||
p.startPos = p.endPos - 1 |
||||
p.Status = StatusText |
||||
case !p.goodArgNum: |
||||
p.Status = StatusBadArgNum |
||||
case p.ArgNum >= len(p.Args): // No argument left over to print for the current verb.
|
||||
p.Status = StatusMissingArg |
||||
p.ArgNum++ |
||||
case verb == 'v': |
||||
// Go syntax
|
||||
p.SharpV = p.Sharp |
||||
p.Sharp = false |
||||
// Struct-field syntax
|
||||
p.PlusV = p.Plus |
||||
p.Plus = false |
||||
fallthrough |
||||
default: |
||||
p.ArgNum++ |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// intFromArg gets the ArgNumth element of Args. On return, isInt reports
|
||||
// whether the argument has integer type.
|
||||
func (p *Parser) intFromArg() (num int, isInt bool) { |
||||
if p.ArgNum < len(p.Args) { |
||||
arg := p.Args[p.ArgNum] |
||||
num, isInt = arg.(int) // Almost always OK.
|
||||
if !isInt { |
||||
// Work harder.
|
||||
switch v := reflect.ValueOf(arg); v.Kind() { |
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: |
||||
n := v.Int() |
||||
if int64(int(n)) == n { |
||||
num = int(n) |
||||
isInt = true |
||||
} |
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: |
||||
n := v.Uint() |
||||
if int64(n) >= 0 && uint64(int(n)) == n { |
||||
num = int(n) |
||||
isInt = true |
||||
} |
||||
default: |
||||
// Already 0, false.
|
||||
} |
||||
} |
||||
p.ArgNum++ |
||||
if tooLarge(num) { |
||||
num = 0 |
||||
isInt = false |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// parseArgNumber returns the value of the bracketed number, minus 1
|
||||
// (explicit argument numbers are one-indexed but we want zero-indexed).
|
||||
// The opening bracket is known to be present at format[0].
|
||||
// The returned values are the index, the number of bytes to consume
|
||||
// up to the closing paren, if present, and whether the number parsed
|
||||
// ok. The bytes to consume will be 1 if no closing paren is present.
|
||||
func parseArgNumber(format string) (index int, wid int, ok bool) { |
||||
// There must be at least 3 bytes: [n].
|
||||
if len(format) < 3 { |
||||
return 0, 1, false |
||||
} |
||||
|
||||
// Find closing bracket.
|
||||
for i := 1; i < len(format); i++ { |
||||
if format[i] == ']' { |
||||
width, ok, newi := parsenum(format, 1, i) |
||||
if !ok || newi != i { |
||||
return 0, i + 1, false |
||||
} |
||||
return width - 1, i + 1, true // arg numbers are one-indexed and skip paren.
|
||||
} |
||||
} |
||||
return 0, 1, false |
||||
} |
||||
|
||||
// updateArgNumber returns the next argument to evaluate, which is either the value of the passed-in
|
||||
// argNum or the value of the bracketed integer that begins format[i:]. It also returns
|
||||
// the new value of i, that is, the index of the next byte of the format to process.
|
||||
func (p *Parser) updateArgNumber(format string, i int) (newi int, found bool) { |
||||
if len(format) <= i || format[i] != '[' { |
||||
return i, false |
||||
} |
||||
p.Reordered = true |
||||
index, wid, ok := parseArgNumber(format[i:]) |
||||
if ok && 0 <= index && index < len(p.Args) { |
||||
p.ArgNum = index |
||||
return i + wid, true |
||||
} |
||||
p.goodArgNum = false |
||||
return i + wid, ok |
||||
} |
||||
|
||||
// tooLarge reports whether the magnitude of the integer is
|
||||
// too large to be used as a formatting width or precision.
|
||||
func tooLarge(x int) bool { |
||||
const max int = 1e6 |
||||
return x > max || x < -max |
||||
} |
||||
|
||||
// parsenum converts ASCII to integer. num is 0 (and isnum is false) if no number present.
|
||||
func parsenum(s string, start, end int) (num int, isnum bool, newi int) { |
||||
if start >= end { |
||||
return 0, false, end |
||||
} |
||||
for newi = start; newi < end && '0' <= s[newi] && s[newi] <= '9'; newi++ { |
||||
if tooLarge(num) { |
||||
return 0, false, end // Overflow; crazy long number most likely.
|
||||
} |
||||
num = num*10 + int(s[newi]-'0') |
||||
isnum = true |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,92 @@
|
||||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package display |
||||
|
||||
// This file contains sets of data for specific languages. Users can use these
|
||||
// to create smaller collections of supported languages and reduce total table
|
||||
// size.
|
||||
|
||||
// The variable names defined here correspond to those in package language.
|
||||
|
||||
var ( |
||||
Afrikaans *Dictionary = &af // af
|
||||
Amharic *Dictionary = &am // am
|
||||
Arabic *Dictionary = &ar // ar
|
||||
ModernStandardArabic *Dictionary = Arabic // ar-001
|
||||
Azerbaijani *Dictionary = &az // az
|
||||
Bulgarian *Dictionary = &bg // bg
|
||||
Bengali *Dictionary = &bn // bn
|
||||
Catalan *Dictionary = &ca // ca
|
||||
Czech *Dictionary = &cs // cs
|
||||
Danish *Dictionary = &da // da
|
||||
German *Dictionary = &de // de
|
||||
Greek *Dictionary = &el // el
|
||||
English *Dictionary = &en // en
|
||||
AmericanEnglish *Dictionary = English // en-US
|
||||
BritishEnglish *Dictionary = English // en-GB
|
||||
Spanish *Dictionary = &es // es
|
||||
EuropeanSpanish *Dictionary = Spanish // es-ES
|
||||
LatinAmericanSpanish *Dictionary = Spanish // es-419
|
||||
Estonian *Dictionary = &et // et
|
||||
Persian *Dictionary = &fa // fa
|
||||
Finnish *Dictionary = &fi // fi
|
||||
Filipino *Dictionary = &fil // fil
|
||||
French *Dictionary = &fr // fr
|
||||
Gujarati *Dictionary = &gu // gu
|
||||
Hebrew *Dictionary = &he // he
|
||||
Hindi *Dictionary = &hi // hi
|
||||
Croatian *Dictionary = &hr // hr
|
||||
Hungarian *Dictionary = &hu // hu
|
||||
Armenian *Dictionary = &hy // hy
|
||||
Indonesian *Dictionary = &id // id
|
||||
Icelandic *Dictionary = &is // is
|
||||
Italian *Dictionary = &it // it
|
||||
Japanese *Dictionary = &ja // ja
|
||||
Georgian *Dictionary = &ka // ka
|
||||
Kazakh *Dictionary = &kk // kk
|
||||
Khmer *Dictionary = &km // km
|
||||
Kannada *Dictionary = &kn // kn
|
||||
Korean *Dictionary = &ko // ko
|
||||
Kirghiz *Dictionary = &ky // ky
|
||||
Lao *Dictionary = &lo // lo
|
||||
Lithuanian *Dictionary = < // lt
|
||||
Latvian *Dictionary = &lv // lv
|
||||
Macedonian *Dictionary = &mk // mk
|
||||
Malayalam *Dictionary = &ml // ml
|
||||
Mongolian *Dictionary = &mn // mn
|
||||
Marathi *Dictionary = &mr // mr
|
||||
Malay *Dictionary = &ms // ms
|
||||
Burmese *Dictionary = &my // my
|
||||
Nepali *Dictionary = &ne // ne
|
||||
Dutch *Dictionary = &nl // nl
|
||||
Norwegian *Dictionary = &no // no
|
||||
Punjabi *Dictionary = &pa // pa
|
||||
Polish *Dictionary = &pl // pl
|
||||
Portuguese *Dictionary = &pt // pt
|
||||
BrazilianPortuguese *Dictionary = Portuguese // pt-BR
|
||||
EuropeanPortuguese *Dictionary = &ptPT // pt-PT
|
||||
Romanian *Dictionary = &ro // ro
|
||||
Russian *Dictionary = &ru // ru
|
||||
Sinhala *Dictionary = &si // si
|
||||
Slovak *Dictionary = &sk // sk
|
||||
Slovenian *Dictionary = &sl // sl
|
||||
Albanian *Dictionary = &sq // sq
|
||||
Serbian *Dictionary = &sr // sr
|
||||
SerbianLatin *Dictionary = &srLatn // sr
|
||||
Swedish *Dictionary = &sv // sv
|
||||
Swahili *Dictionary = &sw // sw
|
||||
Tamil *Dictionary = &ta // ta
|
||||
Telugu *Dictionary = &te // te
|
||||
Thai *Dictionary = &th // th
|
||||
Turkish *Dictionary = &tr // tr
|
||||
Ukrainian *Dictionary = &uk // uk
|
||||
Urdu *Dictionary = &ur // ur
|
||||
Uzbek *Dictionary = &uz // uz
|
||||
Vietnamese *Dictionary = &vi // vi
|
||||
Chinese *Dictionary = &zh // zh
|
||||
SimplifiedChinese *Dictionary = Chinese // zh-Hans
|
||||
TraditionalChinese *Dictionary = &zhHant // zh-Hant
|
||||
Zulu *Dictionary = &zu // zu
|
||||
) |
||||
@ -0,0 +1,420 @@
|
||||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:generate go run maketables.go -output tables.go
|
||||
|
||||
// Package display provides display names for languages, scripts and regions in
|
||||
// a requested language.
|
||||
//
|
||||
// The data is based on CLDR's localeDisplayNames. It includes the names of the
|
||||
// draft level "contributed" or "approved". The resulting tables are quite
|
||||
// large. The display package is designed so that users can reduce the linked-in
|
||||
// table sizes by cherry picking the languages one wishes to support. There is a
|
||||
// Dictionary defined for a selected set of common languages for this purpose.
|
||||
package display // import "golang.org/x/text/language/display"
|
||||
|
||||
import ( |
||||
"fmt" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/internal/format" |
||||
"golang.org/x/text/language" |
||||
) |
||||
|
||||
/* |
||||
TODO: |
||||
All fairly low priority at the moment: |
||||
- Include alternative and variants as an option (using func options). |
||||
- Option for returning the empty string for undefined values. |
||||
- Support variants, currencies, time zones, option names and other data |
||||
provided in CLDR. |
||||
- Do various optimizations: |
||||
- Reduce size of offset tables. |
||||
- Consider compressing infrequently used languages and decompress on demand. |
||||
*/ |
||||
|
||||
// A Formatter formats a tag in the current language. It is used in conjunction
|
||||
// with the message package.
|
||||
type Formatter struct { |
||||
lookup func(tag int, x interface{}) string |
||||
x interface{} |
||||
} |
||||
|
||||
// Format implements "golang.org/x/text/internal/format".Formatter.
|
||||
func (f Formatter) Format(state format.State, verb rune) { |
||||
// TODO: there are a lot of inefficiencies in this code. Fix it when we
|
||||
// language.Tag has embedded compact tags.
|
||||
t := state.Language() |
||||
_, index, _ := matcher.Match(t) |
||||
str := f.lookup(index, f.x) |
||||
if str == "" { |
||||
// TODO: use language-specific punctuation.
|
||||
// TODO: use codePattern instead of language?
|
||||
if unknown := f.lookup(index, language.Und); unknown != "" { |
||||
fmt.Fprintf(state, "%v (%v)", unknown, f.x) |
||||
} else { |
||||
fmt.Fprintf(state, "[language: %v]", f.x) |
||||
} |
||||
} else { |
||||
state.Write([]byte(str)) |
||||
} |
||||
} |
||||
|
||||
// Language returns a Formatter that renders the name for lang in the
|
||||
// current language. x may be a language.Base or a language.Tag.
|
||||
// It renders lang in the default language if no translation for the current
|
||||
// language is supported.
|
||||
func Language(lang interface{}) Formatter { |
||||
return Formatter{langFunc, lang} |
||||
} |
||||
|
||||
// Region returns a Formatter that renders the name for region in the current
|
||||
// language. region may be a language.Region or a language.Tag.
|
||||
// It renders region in the default language if no translation for the current
|
||||
// language is supported.
|
||||
func Region(region interface{}) Formatter { |
||||
return Formatter{regionFunc, region} |
||||
} |
||||
|
||||
// Script returns a Formatter that renders the name for script in the current
|
||||
// language. script may be a language.Script or a language.Tag.
|
||||
// It renders script in the default language if no translation for the current
|
||||
// language is supported.
|
||||
func Script(script interface{}) Formatter { |
||||
return Formatter{scriptFunc, script} |
||||
} |
||||
|
||||
// Tag returns a Formatter that renders the name for tag in the current
|
||||
// language. tag may be a language.Tag.
|
||||
// It renders tag in the default language if no translation for the current
|
||||
// language is supported.
|
||||
func Tag(tag interface{}) Formatter { |
||||
return Formatter{tagFunc, tag} |
||||
} |
||||
|
||||
// A Namer is used to get the name for a given value, such as a Tag, Language,
|
||||
// Script or Region.
|
||||
type Namer interface { |
||||
// Name returns a display string for the given value. A Namer returns an
|
||||
// empty string for values it does not support. A Namer may support naming
|
||||
// an unspecified value. For example, when getting the name for a region for
|
||||
// a tag that does not have a defined Region, it may return the name for an
|
||||
// unknown region. It is up to the user to filter calls to Name for values
|
||||
// for which one does not want to have a name string.
|
||||
Name(x interface{}) string |
||||
} |
||||
|
||||
var ( |
||||
// Supported lists the languages for which names are defined.
|
||||
Supported language.Coverage |
||||
|
||||
// The set of all possible values for which names are defined. Note that not
|
||||
// all Namer implementations will cover all the values of a given type.
|
||||
// A Namer will return the empty string for unsupported values.
|
||||
Values language.Coverage |
||||
|
||||
matcher language.Matcher |
||||
) |
||||
|
||||
func init() { |
||||
tags := make([]language.Tag, numSupported) |
||||
s := supported |
||||
for i := range tags { |
||||
p := strings.IndexByte(s, '|') |
||||
tags[i] = language.Raw.Make(s[:p]) |
||||
s = s[p+1:] |
||||
} |
||||
matcher = language.NewMatcher(tags) |
||||
Supported = language.NewCoverage(tags) |
||||
|
||||
Values = language.NewCoverage(langTagSet.Tags, supportedScripts, supportedRegions) |
||||
} |
||||
|
||||
// Languages returns a Namer for naming languages. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either language.Base
|
||||
// or language.Tag. Note that the result may differ between passing a tag or its
|
||||
// base language. For example, for English, passing "nl-BE" would return Flemish
|
||||
// whereas passing "nl" returns "Dutch".
|
||||
func Languages(t language.Tag) Namer { |
||||
if _, index, conf := matcher.Match(t); conf != language.No { |
||||
return languageNamer(index) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
type languageNamer int |
||||
|
||||
func langFunc(i int, x interface{}) string { |
||||
return nameLanguage(languageNamer(i), x) |
||||
} |
||||
|
||||
func (n languageNamer) name(i int) string { |
||||
return lookup(langHeaders[:], int(n), i) |
||||
} |
||||
|
||||
// Name implements the Namer interface for language names.
|
||||
func (n languageNamer) Name(x interface{}) string { |
||||
return nameLanguage(n, x) |
||||
} |
||||
|
||||
// nonEmptyIndex walks up the parent chain until a non-empty header is found.
|
||||
// It returns -1 if no index could be found.
|
||||
func nonEmptyIndex(h []header, index int) int { |
||||
for ; index != -1 && h[index].data == ""; index = int(parents[index]) { |
||||
} |
||||
return index |
||||
} |
||||
|
||||
// Scripts returns a Namer for naming scripts. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either a
|
||||
// language.Script or a language.Tag. It will not attempt to infer a script for
|
||||
// tags with an unspecified script.
|
||||
func Scripts(t language.Tag) Namer { |
||||
if _, index, conf := matcher.Match(t); conf != language.No { |
||||
if index = nonEmptyIndex(scriptHeaders[:], index); index != -1 { |
||||
return scriptNamer(index) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
type scriptNamer int |
||||
|
||||
func scriptFunc(i int, x interface{}) string { |
||||
return nameScript(scriptNamer(i), x) |
||||
} |
||||
|
||||
func (n scriptNamer) name(i int) string { |
||||
return lookup(scriptHeaders[:], int(n), i) |
||||
} |
||||
|
||||
// Name implements the Namer interface for script names.
|
||||
func (n scriptNamer) Name(x interface{}) string { |
||||
return nameScript(n, x) |
||||
} |
||||
|
||||
// Regions returns a Namer for naming regions. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either a
|
||||
// language.Region or a language.Tag. It will not attempt to infer a region for
|
||||
// tags with an unspecified region.
|
||||
func Regions(t language.Tag) Namer { |
||||
if _, index, conf := matcher.Match(t); conf != language.No { |
||||
if index = nonEmptyIndex(regionHeaders[:], index); index != -1 { |
||||
return regionNamer(index) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
type regionNamer int |
||||
|
||||
func regionFunc(i int, x interface{}) string { |
||||
return nameRegion(regionNamer(i), x) |
||||
} |
||||
|
||||
func (n regionNamer) name(i int) string { |
||||
return lookup(regionHeaders[:], int(n), i) |
||||
} |
||||
|
||||
// Name implements the Namer interface for region names.
|
||||
func (n regionNamer) Name(x interface{}) string { |
||||
return nameRegion(n, x) |
||||
} |
||||
|
||||
// Tags returns a Namer for giving a full description of a tag. The names of
|
||||
// scripts and regions that are not already implied by the language name will
|
||||
// in appended within parentheses. It returns nil if there is not data for the
|
||||
// given tag. The type passed to Name must be a tag.
|
||||
func Tags(t language.Tag) Namer { |
||||
if _, index, conf := matcher.Match(t); conf != language.No { |
||||
return tagNamer(index) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
type tagNamer int |
||||
|
||||
func tagFunc(i int, x interface{}) string { |
||||
return nameTag(languageNamer(i), scriptNamer(i), regionNamer(i), x) |
||||
} |
||||
|
||||
// Name implements the Namer interface for tag names.
|
||||
func (n tagNamer) Name(x interface{}) string { |
||||
return nameTag(languageNamer(n), scriptNamer(n), regionNamer(n), x) |
||||
} |
||||
|
||||
// lookup finds the name for an entry in a global table, traversing the
|
||||
// inheritance hierarchy if needed.
|
||||
func lookup(table []header, dict, want int) string { |
||||
for dict != -1 { |
||||
if s := table[dict].name(want); s != "" { |
||||
return s |
||||
} |
||||
dict = int(parents[dict]) |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// A Dictionary holds a collection of Namers for a single language. One can
|
||||
// reduce the amount of data linked in to a binary by only referencing
|
||||
// Dictionaries for the languages one needs to support instead of using the
|
||||
// generic Namer factories.
|
||||
type Dictionary struct { |
||||
parent *Dictionary |
||||
lang header |
||||
script header |
||||
region header |
||||
} |
||||
|
||||
// Tags returns a Namer for giving a full description of a tag. The names of
|
||||
// scripts and regions that are not already implied by the language name will
|
||||
// in appended within parentheses. It returns nil if there is not data for the
|
||||
// given tag. The type passed to Name must be a tag.
|
||||
func (d *Dictionary) Tags() Namer { |
||||
return dictTags{d} |
||||
} |
||||
|
||||
type dictTags struct { |
||||
d *Dictionary |
||||
} |
||||
|
||||
// Name implements the Namer interface for tag names.
|
||||
func (n dictTags) Name(x interface{}) string { |
||||
return nameTag(dictLanguages{n.d}, dictScripts{n.d}, dictRegions{n.d}, x) |
||||
} |
||||
|
||||
// Languages returns a Namer for naming languages. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either language.Base
|
||||
// or language.Tag. Note that the result may differ between passing a tag or its
|
||||
// base language. For example, for English, passing "nl-BE" would return Flemish
|
||||
// whereas passing "nl" returns "Dutch".
|
||||
func (d *Dictionary) Languages() Namer { |
||||
return dictLanguages{d} |
||||
} |
||||
|
||||
type dictLanguages struct { |
||||
d *Dictionary |
||||
} |
||||
|
||||
func (n dictLanguages) name(i int) string { |
||||
for d := n.d; d != nil; d = d.parent { |
||||
if s := d.lang.name(i); s != "" { |
||||
return s |
||||
} |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// Name implements the Namer interface for language names.
|
||||
func (n dictLanguages) Name(x interface{}) string { |
||||
return nameLanguage(n, x) |
||||
} |
||||
|
||||
// Scripts returns a Namer for naming scripts. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either a
|
||||
// language.Script or a language.Tag. It will not attempt to infer a script for
|
||||
// tags with an unspecified script.
|
||||
func (d *Dictionary) Scripts() Namer { |
||||
return dictScripts{d} |
||||
} |
||||
|
||||
type dictScripts struct { |
||||
d *Dictionary |
||||
} |
||||
|
||||
func (n dictScripts) name(i int) string { |
||||
for d := n.d; d != nil; d = d.parent { |
||||
if s := d.script.name(i); s != "" { |
||||
return s |
||||
} |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// Name implements the Namer interface for script names.
|
||||
func (n dictScripts) Name(x interface{}) string { |
||||
return nameScript(n, x) |
||||
} |
||||
|
||||
// Regions returns a Namer for naming regions. It returns nil if there is no
|
||||
// data for the given tag. The type passed to Name must be either a
|
||||
// language.Region or a language.Tag. It will not attempt to infer a region for
|
||||
// tags with an unspecified region.
|
||||
func (d *Dictionary) Regions() Namer { |
||||
return dictRegions{d} |
||||
} |
||||
|
||||
type dictRegions struct { |
||||
d *Dictionary |
||||
} |
||||
|
||||
func (n dictRegions) name(i int) string { |
||||
for d := n.d; d != nil; d = d.parent { |
||||
if s := d.region.name(i); s != "" { |
||||
return s |
||||
} |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// Name implements the Namer interface for region names.
|
||||
func (n dictRegions) Name(x interface{}) string { |
||||
return nameRegion(n, x) |
||||
} |
||||
|
||||
// A SelfNamer implements a Namer that returns the name of language in this same
|
||||
// language. It provides a very compact mechanism to provide a comprehensive
|
||||
// list of languages to users in their native language.
|
||||
type SelfNamer struct { |
||||
// Supported defines the values supported by this Namer.
|
||||
Supported language.Coverage |
||||
} |
||||
|
||||
var ( |
||||
// Self is a shared instance of a SelfNamer.
|
||||
Self *SelfNamer = &self |
||||
|
||||
self = SelfNamer{language.NewCoverage(selfTagSet.Tags)} |
||||
) |
||||
|
||||
// Name returns the name of a given language tag in the language identified by
|
||||
// this tag. It supports both the language.Base and language.Tag types.
|
||||
func (n SelfNamer) Name(x interface{}) string { |
||||
t, _ := language.All.Compose(x) |
||||
base, scr, reg := t.Raw() |
||||
baseScript := language.Script{} |
||||
if (scr == language.Script{} && reg != language.Region{}) { |
||||
// For looking up in the self dictionary, we need to select the
|
||||
// maximized script. This is even the case if the script isn't
|
||||
// specified.
|
||||
s1, _ := t.Script() |
||||
if baseScript = getScript(base); baseScript != s1 { |
||||
scr = s1 |
||||
} |
||||
} |
||||
|
||||
i, scr, reg := selfTagSet.index(base, scr, reg) |
||||
if i == -1 { |
||||
return "" |
||||
} |
||||
|
||||
// Only return the display name if the script matches the expected script.
|
||||
if (scr != language.Script{}) { |
||||
if (baseScript == language.Script{}) { |
||||
baseScript = getScript(base) |
||||
} |
||||
if baseScript != scr { |
||||
return "" |
||||
} |
||||
} |
||||
|
||||
return selfHeaders[0].name(i) |
||||
} |
||||
|
||||
// getScript returns the maximized script for a base language.
|
||||
func getScript(b language.Base) language.Script { |
||||
tag, _ := language.Raw.Compose(b) |
||||
scr, _ := tag.Script() |
||||
return scr |
||||
} |
||||
@ -0,0 +1,253 @@
|
||||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package display |
||||
|
||||
// This file contains common lookup code that is shared between the various
|
||||
// implementations of Namer and Dictionaries.
|
||||
|
||||
import ( |
||||
"fmt" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/language" |
||||
) |
||||
|
||||
type namer interface { |
||||
// name gets the string for the given index. It should walk the
|
||||
// inheritance chain if a value is not present in the base index.
|
||||
name(idx int) string |
||||
} |
||||
|
||||
func nameLanguage(n namer, x interface{}) string { |
||||
t, _ := language.All.Compose(x) |
||||
for { |
||||
i, _, _ := langTagSet.index(t.Raw()) |
||||
if s := n.name(i); s != "" { |
||||
return s |
||||
} |
||||
if t = t.Parent(); t == language.Und { |
||||
return "" |
||||
} |
||||
} |
||||
} |
||||
|
||||
func nameScript(n namer, x interface{}) string { |
||||
t, _ := language.DeprecatedScript.Compose(x) |
||||
_, s, _ := t.Raw() |
||||
return n.name(scriptIndex.index(s.String())) |
||||
} |
||||
|
||||
func nameRegion(n namer, x interface{}) string { |
||||
t, _ := language.DeprecatedRegion.Compose(x) |
||||
_, _, r := t.Raw() |
||||
return n.name(regionIndex.index(r.String())) |
||||
} |
||||
|
||||
func nameTag(langN, scrN, regN namer, x interface{}) string { |
||||
t, ok := x.(language.Tag) |
||||
if !ok { |
||||
return "" |
||||
} |
||||
const form = language.All &^ language.SuppressScript |
||||
if c, err := form.Canonicalize(t); err == nil { |
||||
t = c |
||||
} |
||||
_, sRaw, rRaw := t.Raw() |
||||
i, scr, reg := langTagSet.index(t.Raw()) |
||||
for i != -1 { |
||||
if str := langN.name(i); str != "" { |
||||
if hasS, hasR := (scr != language.Script{}), (reg != language.Region{}); hasS || hasR { |
||||
ss, sr := "", "" |
||||
if hasS { |
||||
ss = scrN.name(scriptIndex.index(scr.String())) |
||||
} |
||||
if hasR { |
||||
sr = regN.name(regionIndex.index(reg.String())) |
||||
} |
||||
// TODO: use patterns in CLDR or at least confirm they are the
|
||||
// same for all languages.
|
||||
if ss != "" && sr != "" { |
||||
return fmt.Sprintf("%s (%s, %s)", str, ss, sr) |
||||
} |
||||
if ss != "" || sr != "" { |
||||
return fmt.Sprintf("%s (%s%s)", str, ss, sr) |
||||
} |
||||
} |
||||
return str |
||||
} |
||||
scr, reg = sRaw, rRaw |
||||
if t = t.Parent(); t == language.Und { |
||||
return "" |
||||
} |
||||
i, _, _ = langTagSet.index(t.Raw()) |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// header contains the data and indexes for a single namer.
|
||||
// data contains a series of strings concatenated into one. index contains the
|
||||
// offsets for a string in data. For example, consider a header that defines
|
||||
// strings for the languages de, el, en, fi, and nl:
|
||||
//
|
||||
// header{
|
||||
// data: "GermanGreekEnglishDutch",
|
||||
// index: []uint16{0, 6, 11, 18, 18, 23},
|
||||
// }
|
||||
//
|
||||
// For a language with index i, the string is defined by
|
||||
// data[index[i]:index[i+1]]. So the number of elements in index is always one
|
||||
// greater than the number of languages for which header defines a value.
|
||||
// A string for a language may be empty, which means the name is undefined. In
|
||||
// the above example, the name for fi (Finnish) is undefined.
|
||||
type header struct { |
||||
data string |
||||
index []uint16 |
||||
} |
||||
|
||||
// name looks up the name for a tag in the dictionary, given its index.
|
||||
func (h *header) name(i int) string { |
||||
if 0 <= i && i < len(h.index)-1 { |
||||
return h.data[h.index[i]:h.index[i+1]] |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// tagSet is used to find the index of a language in a set of tags.
|
||||
type tagSet struct { |
||||
single tagIndex |
||||
long []string |
||||
} |
||||
|
||||
var ( |
||||
langTagSet = tagSet{ |
||||
single: langIndex, |
||||
long: langTagsLong, |
||||
} |
||||
|
||||
// selfTagSet is used for indexing the language strings in their own
|
||||
// language.
|
||||
selfTagSet = tagSet{ |
||||
single: selfIndex, |
||||
long: selfTagsLong, |
||||
} |
||||
|
||||
zzzz = language.MustParseScript("Zzzz") |
||||
zz = language.MustParseRegion("ZZ") |
||||
) |
||||
|
||||
// index returns the index of the tag for the given base, script and region or
|
||||
// its parent if the tag is not available. If the match is for a parent entry,
|
||||
// the excess script and region are returned.
|
||||
func (ts *tagSet) index(base language.Base, scr language.Script, reg language.Region) (int, language.Script, language.Region) { |
||||
lang := base.String() |
||||
index := -1 |
||||
if (scr != language.Script{} || reg != language.Region{}) { |
||||
if scr == zzzz { |
||||
scr = language.Script{} |
||||
} |
||||
if reg == zz { |
||||
reg = language.Region{} |
||||
} |
||||
|
||||
i := sort.SearchStrings(ts.long, lang) |
||||
// All entries have either a script or a region and not both.
|
||||
scrStr, regStr := scr.String(), reg.String() |
||||
for ; i < len(ts.long) && strings.HasPrefix(ts.long[i], lang); i++ { |
||||
if s := ts.long[i][len(lang)+1:]; s == scrStr { |
||||
scr = language.Script{} |
||||
index = i + ts.single.len() |
||||
break |
||||
} else if s == regStr { |
||||
reg = language.Region{} |
||||
index = i + ts.single.len() |
||||
break |
||||
} |
||||
} |
||||
} |
||||
if index == -1 { |
||||
index = ts.single.index(lang) |
||||
} |
||||
return index, scr, reg |
||||
} |
||||
|
||||
func (ts *tagSet) Tags() []language.Tag { |
||||
tags := make([]language.Tag, 0, ts.single.len()+len(ts.long)) |
||||
ts.single.keys(func(s string) { |
||||
tags = append(tags, language.Raw.MustParse(s)) |
||||
}) |
||||
for _, s := range ts.long { |
||||
tags = append(tags, language.Raw.MustParse(s)) |
||||
} |
||||
return tags |
||||
} |
||||
|
||||
func supportedScripts() []language.Script { |
||||
scr := make([]language.Script, 0, scriptIndex.len()) |
||||
scriptIndex.keys(func(s string) { |
||||
scr = append(scr, language.MustParseScript(s)) |
||||
}) |
||||
return scr |
||||
} |
||||
|
||||
func supportedRegions() []language.Region { |
||||
reg := make([]language.Region, 0, regionIndex.len()) |
||||
regionIndex.keys(func(s string) { |
||||
reg = append(reg, language.MustParseRegion(s)) |
||||
}) |
||||
return reg |
||||
} |
||||
|
||||
// tagIndex holds a concatenated lists of subtags of length 2 to 4, one string
|
||||
// for each length, which can be used in combination with binary search to get
|
||||
// the index associated with a tag.
|
||||
// For example, a tagIndex{
|
||||
//
|
||||
// "arenesfrruzh", // 6 2-byte tags.
|
||||
// "barwae", // 2 3-byte tags.
|
||||
// "",
|
||||
//
|
||||
// }
|
||||
// would mean that the 2-byte tag "fr" had an index of 3, and the 3-byte tag
|
||||
// "wae" had an index of 7.
|
||||
type tagIndex [3]string |
||||
|
||||
func (t *tagIndex) index(s string) int { |
||||
sz := len(s) |
||||
if sz < 2 || 4 < sz { |
||||
return -1 |
||||
} |
||||
a := t[sz-2] |
||||
index := sort.Search(len(a)/sz, func(i int) bool { |
||||
p := i * sz |
||||
return a[p:p+sz] >= s |
||||
}) |
||||
p := index * sz |
||||
if end := p + sz; end > len(a) || a[p:end] != s { |
||||
return -1 |
||||
} |
||||
// Add the number of tags for smaller sizes.
|
||||
for i := 0; i < sz-2; i++ { |
||||
index += len(t[i]) / (i + 2) |
||||
} |
||||
return index |
||||
} |
||||
|
||||
// len returns the number of tags that are contained in the tagIndex.
|
||||
func (t *tagIndex) len() (n int) { |
||||
for i, s := range t { |
||||
n += len(s) / (i + 2) |
||||
} |
||||
return n |
||||
} |
||||
|
||||
// keys calls f for each tag.
|
||||
func (t *tagIndex) keys(f func(key string)) { |
||||
for i, s := range *t { |
||||
for ; s != ""; s = s[i+2:] { |
||||
f(s[:i+2]) |
||||
} |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue