Use zero-copy approach to convert types between string and byte… (#2206)

* Use zero-copy approach to convert types between string and byte slice

* Rename argument to a eligible one

Benchmark:

BenchmarkBytesConvBytesToStrRaw-4   	21003800	        70.9 ns/op	      96 B/op	       1 allocs/op
BenchmarkBytesConvBytesToStr-4      	1000000000	         0.333 ns/op	       0 B/op	       0 allocs/op
BenchmarkBytesConvStrToBytesRaw-4   	18478059	        59.3 ns/op	      96 B/op	       1 allocs/op
BenchmarkBytesConvStrToBytes-4      	1000000000	         0.373 ns/op	       0 B/op	       0 allocs/op


Co-authored-by: Bo-Yi Wu <appleboy.tw@gmail.com>
This commit is contained in:
Andy Pan 2020-01-18 00:32:50 +08:00 committed by Bo-Yi Wu
parent ace6e4c2ea
commit 982daeb1ec
6 changed files with 130 additions and 10 deletions

View File

@ -8,6 +8,8 @@ import (
"encoding/base64" "encoding/base64"
"net/http" "net/http"
"strconv" "strconv"
"github.com/gin-gonic/gin/internal/bytesconv"
) )
// AuthUserKey is the cookie name for user credential in basic auth. // AuthUserKey is the cookie name for user credential in basic auth.
@ -83,5 +85,5 @@ func processAccounts(accounts Accounts) authPairs {
func authorizationHeader(user, password string) string { func authorizationHeader(user, password string) string {
base := user + ":" + password base := user + ":" + password
return "Basic " + base64.StdEncoding.EncodeToString([]byte(base)) return "Basic " + base64.StdEncoding.EncodeToString(bytesconv.StringToBytes(base))
} }

View File

@ -12,6 +12,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/gin-gonic/gin/internal/bytesconv"
"github.com/gin-gonic/gin/internal/json" "github.com/gin-gonic/gin/internal/json"
) )
@ -208,9 +209,9 @@ func setWithProperType(val string, value reflect.Value, field reflect.StructFiel
case time.Time: case time.Time:
return setTimeField(val, field, value) return setTimeField(val, field, value)
} }
return json.Unmarshal([]byte(val), value.Addr().Interface()) return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
case reflect.Map: case reflect.Map:
return json.Unmarshal([]byte(val), value.Addr().Interface()) return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
default: default:
return errUnknownType return errUnknownType
} }

3
gin.go
View File

@ -13,6 +13,7 @@ import (
"path" "path"
"sync" "sync"
"github.com/gin-gonic/gin/internal/bytesconv"
"github.com/gin-gonic/gin/render" "github.com/gin-gonic/gin/render"
) )
@ -477,7 +478,7 @@ func redirectFixedPath(c *Context, root *node, trailingSlash bool) bool {
rPath := req.URL.Path rPath := req.URL.Path
if fixedPath, ok := root.findCaseInsensitivePath(cleanPath(rPath), trailingSlash); ok { if fixedPath, ok := root.findCaseInsensitivePath(cleanPath(rPath), trailingSlash); ok {
req.URL.Path = string(fixedPath) req.URL.Path = bytesconv.BytesToString(fixedPath)
redirectRequest(c) redirectRequest(c)
return true return true
} }

View File

@ -0,0 +1,19 @@
package bytesconv
import (
"reflect"
"unsafe"
)
// StringToBytes converts string to byte slice without a memory allocation.
func StringToBytes(s string) (b []byte) {
sh := *(*reflect.StringHeader)(unsafe.Pointer(&s))
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
bh.Data, bh.Len, bh.Cap = sh.Data, sh.Len, sh.Len
return b
}
// BytesToString converts byte slice to string without a memory allocation.
func BytesToString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}

View File

@ -0,0 +1,95 @@
package bytesconv
import (
"bytes"
"math/rand"
"strings"
"testing"
"time"
)
var testString = "Albert Einstein: Logic will get you from A to B. Imagination will take you everywhere."
var testBytes = []byte(testString)
func rawBytesToStr(b []byte) string {
return string(b)
}
func rawStrToBytes(s string) []byte {
return []byte(s)
}
// go test -v
func TestBytesToString(t *testing.T) {
data := make([]byte, 1024)
for i := 0; i < 100; i++ {
rand.Read(data)
if rawBytesToStr(data) != BytesToString(data) {
t.Fatal("don't match")
}
}
}
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
var src = rand.NewSource(time.Now().UnixNano())
func RandStringBytesMaskImprSrcSB(n int) string {
sb := strings.Builder{}
sb.Grow(n)
// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
sb.WriteByte(letterBytes[idx])
i--
}
cache >>= letterIdxBits
remain--
}
return sb.String()
}
func TestStringToBytes(t *testing.T) {
for i := 0; i < 100; i++ {
s := RandStringBytesMaskImprSrcSB(64)
if !bytes.Equal(rawStrToBytes(s), StringToBytes(s)) {
t.Fatal("don't match")
}
}
}
// go test -v -run=none -bench=^BenchmarkBytesConv -benchmem=true
func BenchmarkBytesConvBytesToStrRaw(b *testing.B) {
for i := 0; i < b.N; i++ {
rawBytesToStr(testBytes)
}
}
func BenchmarkBytesConvBytesToStr(b *testing.B) {
for i := 0; i < b.N; i++ {
BytesToString(testBytes)
}
}
func BenchmarkBytesConvStrToBytesRaw(b *testing.B) {
for i := 0; i < b.N; i++ {
rawStrToBytes(testString)
}
}
func BenchmarkBytesConvStrToBytes(b *testing.B) {
for i := 0; i < b.N; i++ {
StringToBytes(testString)
}
}

View File

@ -10,6 +10,7 @@ import (
"html/template" "html/template"
"net/http" "net/http"
"github.com/gin-gonic/gin/internal/bytesconv"
"github.com/gin-gonic/gin/internal/json" "github.com/gin-gonic/gin/internal/json"
) )
@ -97,8 +98,9 @@ func (r SecureJSON) Render(w http.ResponseWriter) error {
return err return err
} }
// if the jsonBytes is array values // if the jsonBytes is array values
if bytes.HasPrefix(jsonBytes, []byte("[")) && bytes.HasSuffix(jsonBytes, []byte("]")) { if bytes.HasPrefix(jsonBytes, bytesconv.StringToBytes("[")) && bytes.HasSuffix(jsonBytes,
_, err = w.Write([]byte(r.Prefix)) bytesconv.StringToBytes("]")) {
_, err = w.Write(bytesconv.StringToBytes(r.Prefix))
if err != nil { if err != nil {
return err return err
} }
@ -126,11 +128,11 @@ func (r JsonpJSON) Render(w http.ResponseWriter) (err error) {
} }
callback := template.JSEscapeString(r.Callback) callback := template.JSEscapeString(r.Callback)
_, err = w.Write([]byte(callback)) _, err = w.Write(bytesconv.StringToBytes(callback))
if err != nil { if err != nil {
return err return err
} }
_, err = w.Write([]byte("(")) _, err = w.Write(bytesconv.StringToBytes("("))
if err != nil { if err != nil {
return err return err
} }
@ -138,7 +140,7 @@ func (r JsonpJSON) Render(w http.ResponseWriter) (err error) {
if err != nil { if err != nil {
return err return err
} }
_, err = w.Write([]byte(");")) _, err = w.Write(bytesconv.StringToBytes(");"))
if err != nil { if err != nil {
return err return err
} }
@ -160,7 +162,7 @@ func (r AsciiJSON) Render(w http.ResponseWriter) (err error) {
} }
var buffer bytes.Buffer var buffer bytes.Buffer
for _, r := range string(ret) { for _, r := range bytesconv.BytesToString(ret) {
cvt := string(r) cvt := string(r)
if r >= 128 { if r >= 128 {
cvt = fmt.Sprintf("\\u%04x", int64(r)) cvt = fmt.Sprintf("\\u%04x", int64(r))