From 982daeb1ecdced87bbef6c11783a640eb88a193a Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sat, 18 Jan 2020 00:32:50 +0800 Subject: [PATCH] =?UTF-8?q?Use=20zero-copy=20approach=20to=20convert=20typ?= =?UTF-8?q?es=20between=20string=20and=20byte=E2=80=A6=20(#2206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use zero-copy approach to convert types between string and byte slice * Rename argument to a eligible one Benchmark: BenchmarkBytesConvBytesToStrRaw-4 21003800 70.9 ns/op 96 B/op 1 allocs/op BenchmarkBytesConvBytesToStr-4 1000000000 0.333 ns/op 0 B/op 0 allocs/op BenchmarkBytesConvStrToBytesRaw-4 18478059 59.3 ns/op 96 B/op 1 allocs/op BenchmarkBytesConvStrToBytes-4 1000000000 0.373 ns/op 0 B/op 0 allocs/op Co-authored-by: Bo-Yi Wu --- auth.go | 4 +- binding/form_mapping.go | 5 +- gin.go | 3 +- internal/bytesconv/bytesconv.go | 19 ++++++ internal/bytesconv/bytesconv_test.go | 95 ++++++++++++++++++++++++++++ render/json.go | 14 ++-- 6 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 internal/bytesconv/bytesconv.go create mode 100644 internal/bytesconv/bytesconv_test.go diff --git a/auth.go b/auth.go index c96b1e2..9e5d4cf 100644 --- a/auth.go +++ b/auth.go @@ -8,6 +8,8 @@ import ( "encoding/base64" "net/http" "strconv" + + "github.com/gin-gonic/gin/internal/bytesconv" ) // AuthUserKey is the cookie name for user credential in basic auth. @@ -83,5 +85,5 @@ func processAccounts(accounts Accounts) authPairs { func authorizationHeader(user, password string) string { base := user + ":" + password - return "Basic " + base64.StdEncoding.EncodeToString([]byte(base)) + return "Basic " + base64.StdEncoding.EncodeToString(bytesconv.StringToBytes(base)) } diff --git a/binding/form_mapping.go b/binding/form_mapping.go index d6199c4..b81ad19 100644 --- a/binding/form_mapping.go +++ b/binding/form_mapping.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/gin-gonic/gin/internal/bytesconv" "github.com/gin-gonic/gin/internal/json" ) @@ -208,9 +209,9 @@ func setWithProperType(val string, value reflect.Value, field reflect.StructFiel case time.Time: return setTimeField(val, field, value) } - return json.Unmarshal([]byte(val), value.Addr().Interface()) + return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface()) case reflect.Map: - return json.Unmarshal([]byte(val), value.Addr().Interface()) + return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface()) default: return errUnknownType } diff --git a/gin.go b/gin.go index 71f3fd5..0244f18 100644 --- a/gin.go +++ b/gin.go @@ -13,6 +13,7 @@ import ( "path" "sync" + "github.com/gin-gonic/gin/internal/bytesconv" "github.com/gin-gonic/gin/render" ) @@ -477,7 +478,7 @@ func redirectFixedPath(c *Context, root *node, trailingSlash bool) bool { rPath := req.URL.Path if fixedPath, ok := root.findCaseInsensitivePath(cleanPath(rPath), trailingSlash); ok { - req.URL.Path = string(fixedPath) + req.URL.Path = bytesconv.BytesToString(fixedPath) redirectRequest(c) return true } diff --git a/internal/bytesconv/bytesconv.go b/internal/bytesconv/bytesconv.go new file mode 100644 index 0000000..32c2b59 --- /dev/null +++ b/internal/bytesconv/bytesconv.go @@ -0,0 +1,19 @@ +package bytesconv + +import ( + "reflect" + "unsafe" +) + +// StringToBytes converts string to byte slice without a memory allocation. +func StringToBytes(s string) (b []byte) { + sh := *(*reflect.StringHeader)(unsafe.Pointer(&s)) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data, bh.Len, bh.Cap = sh.Data, sh.Len, sh.Len + return b +} + +// BytesToString converts byte slice to string without a memory allocation. +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/internal/bytesconv/bytesconv_test.go b/internal/bytesconv/bytesconv_test.go new file mode 100644 index 0000000..ee2c8ab --- /dev/null +++ b/internal/bytesconv/bytesconv_test.go @@ -0,0 +1,95 @@ +package bytesconv + +import ( + "bytes" + "math/rand" + "strings" + "testing" + "time" +) + +var testString = "Albert Einstein: Logic will get you from A to B. Imagination will take you everywhere." +var testBytes = []byte(testString) + +func rawBytesToStr(b []byte) string { + return string(b) +} + +func rawStrToBytes(s string) []byte { + return []byte(s) +} + +// go test -v + +func TestBytesToString(t *testing.T) { + data := make([]byte, 1024) + for i := 0; i < 100; i++ { + rand.Read(data) + if rawBytesToStr(data) != BytesToString(data) { + t.Fatal("don't match") + } + } +} + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + sb.WriteByte(letterBytes[idx]) + i-- + } + cache >>= letterIdxBits + remain-- + } + + return sb.String() +} + +func TestStringToBytes(t *testing.T) { + for i := 0; i < 100; i++ { + s := RandStringBytesMaskImprSrcSB(64) + if !bytes.Equal(rawStrToBytes(s), StringToBytes(s)) { + t.Fatal("don't match") + } + } +} + +// go test -v -run=none -bench=^BenchmarkBytesConv -benchmem=true + +func BenchmarkBytesConvBytesToStrRaw(b *testing.B) { + for i := 0; i < b.N; i++ { + rawBytesToStr(testBytes) + } +} + +func BenchmarkBytesConvBytesToStr(b *testing.B) { + for i := 0; i < b.N; i++ { + BytesToString(testBytes) + } +} + +func BenchmarkBytesConvStrToBytesRaw(b *testing.B) { + for i := 0; i < b.N; i++ { + rawStrToBytes(testString) + } +} + +func BenchmarkBytesConvStrToBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + StringToBytes(testString) + } +} diff --git a/render/json.go b/render/json.go index 70506f7..a6fd311 100644 --- a/render/json.go +++ b/render/json.go @@ -10,6 +10,7 @@ import ( "html/template" "net/http" + "github.com/gin-gonic/gin/internal/bytesconv" "github.com/gin-gonic/gin/internal/json" ) @@ -97,8 +98,9 @@ func (r SecureJSON) Render(w http.ResponseWriter) error { return err } // if the jsonBytes is array values - if bytes.HasPrefix(jsonBytes, []byte("[")) && bytes.HasSuffix(jsonBytes, []byte("]")) { - _, err = w.Write([]byte(r.Prefix)) + if bytes.HasPrefix(jsonBytes, bytesconv.StringToBytes("[")) && bytes.HasSuffix(jsonBytes, + bytesconv.StringToBytes("]")) { + _, err = w.Write(bytesconv.StringToBytes(r.Prefix)) if err != nil { return err } @@ -126,11 +128,11 @@ func (r JsonpJSON) Render(w http.ResponseWriter) (err error) { } callback := template.JSEscapeString(r.Callback) - _, err = w.Write([]byte(callback)) + _, err = w.Write(bytesconv.StringToBytes(callback)) if err != nil { return err } - _, err = w.Write([]byte("(")) + _, err = w.Write(bytesconv.StringToBytes("(")) if err != nil { return err } @@ -138,7 +140,7 @@ func (r JsonpJSON) Render(w http.ResponseWriter) (err error) { if err != nil { return err } - _, err = w.Write([]byte(");")) + _, err = w.Write(bytesconv.StringToBytes(");")) if err != nil { return err } @@ -160,7 +162,7 @@ func (r AsciiJSON) Render(w http.ResponseWriter) (err error) { } var buffer bytes.Buffer - for _, r := range string(ret) { + for _, r := range bytesconv.BytesToString(ret) { cvt := string(r) if r >= 128 { cvt = fmt.Sprintf("\\u%04x", int64(r))