mirror of
https://github.com/mailgun/groupcache.git
synced 2024-09-28 23:00:56 +00:00
initial commit.
This commit is contained in:
commit
6dad98a783
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
*~
|
191
LICENSE
Normal file
191
LICENSE
Normal file
@ -0,0 +1,191 @@
|
|||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction, and
|
||||||
|
distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by the copyright
|
||||||
|
owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all other entities
|
||||||
|
that control, are controlled by, or are under common control with that entity.
|
||||||
|
For the purposes of this definition, "control" means (i) the power, direct or
|
||||||
|
indirect, to cause the direction or management of such entity, whether by
|
||||||
|
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity exercising
|
||||||
|
permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications, including
|
||||||
|
but not limited to software source code, documentation source, and configuration
|
||||||
|
files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical transformation or
|
||||||
|
translation of a Source form, including but not limited to compiled object code,
|
||||||
|
generated documentation, and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or Object form, made
|
||||||
|
available under the License, as indicated by a copyright notice that is included
|
||||||
|
in or attached to the work (an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object form, that
|
||||||
|
is based on (or derived from) the Work and for which the editorial revisions,
|
||||||
|
annotations, elaborations, or other modifications represent, as a whole, an
|
||||||
|
original work of authorship. For the purposes of this License, Derivative Works
|
||||||
|
shall not include works that remain separable from, or merely link (or bind by
|
||||||
|
name) to the interfaces of, the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including the original version
|
||||||
|
of the Work and any modifications or additions to that Work or Derivative Works
|
||||||
|
thereof, that is intentionally submitted to Licensor for inclusion in the Work
|
||||||
|
by the copyright owner or by an individual or Legal Entity authorized to submit
|
||||||
|
on behalf of the copyright owner. For the purposes of this definition,
|
||||||
|
"submitted" means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems, and
|
||||||
|
issue tracking systems that are managed by, or on behalf of, the Licensor for
|
||||||
|
the purpose of discussing and improving the Work, but excluding communication
|
||||||
|
that is conspicuously marked or otherwise designated in writing by the copyright
|
||||||
|
owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
|
||||||
|
of whom a Contribution has been received by Licensor and subsequently
|
||||||
|
incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License.
|
||||||
|
|
||||||
|
Subject to the terms and conditions of this License, each Contributor hereby
|
||||||
|
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
||||||
|
irrevocable copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the Work and such
|
||||||
|
Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License.
|
||||||
|
|
||||||
|
Subject to the terms and conditions of this License, each Contributor hereby
|
||||||
|
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
||||||
|
irrevocable (except as stated in this section) patent license to make, have
|
||||||
|
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
|
||||||
|
such license applies only to those patent claims licensable by such Contributor
|
||||||
|
that are necessarily infringed by their Contribution(s) alone or by combination
|
||||||
|
of their Contribution(s) with the Work to which such Contribution(s) was
|
||||||
|
submitted. If You institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
|
||||||
|
Contribution incorporated within the Work constitutes direct or contributory
|
||||||
|
patent infringement, then any patent licenses granted to You under this License
|
||||||
|
for that Work shall terminate as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution.
|
||||||
|
|
||||||
|
You may reproduce and distribute copies of the Work or Derivative Works thereof
|
||||||
|
in any medium, with or without modifications, and in Source or Object form,
|
||||||
|
provided that You meet the following conditions:
|
||||||
|
|
||||||
|
You must give any other recipients of the Work or Derivative Works a copy of
|
||||||
|
this License; and
|
||||||
|
You must cause any modified files to carry prominent notices stating that You
|
||||||
|
changed the files; and
|
||||||
|
You must retain, in the Source form of any Derivative Works that You distribute,
|
||||||
|
all copyright, patent, trademark, and attribution notices from the Source form
|
||||||
|
of the Work, excluding those notices that do not pertain to any part of the
|
||||||
|
Derivative Works; and
|
||||||
|
If the Work includes a "NOTICE" text file as part of its distribution, then any
|
||||||
|
Derivative Works that You distribute must include a readable copy of the
|
||||||
|
attribution notices contained within such NOTICE file, excluding those notices
|
||||||
|
that do not pertain to any part of the Derivative Works, in at least one of the
|
||||||
|
following places: within a NOTICE text file distributed as part of the
|
||||||
|
Derivative Works; within the Source form or documentation, if provided along
|
||||||
|
with the Derivative Works; or, within a display generated by the Derivative
|
||||||
|
Works, if and wherever such third-party notices normally appear. The contents of
|
||||||
|
the NOTICE file are for informational purposes only and do not modify the
|
||||||
|
License. You may add Your own attribution notices within Derivative Works that
|
||||||
|
You distribute, alongside or as an addendum to the NOTICE text from the Work,
|
||||||
|
provided that such additional attribution notices cannot be construed as
|
||||||
|
modifying the License.
|
||||||
|
You may add Your own copyright statement to Your modifications and may provide
|
||||||
|
additional or different license terms and conditions for use, reproduction, or
|
||||||
|
distribution of Your modifications, or for any such Derivative Works as a whole,
|
||||||
|
provided Your use, reproduction, and distribution of the Work otherwise complies
|
||||||
|
with the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions.
|
||||||
|
|
||||||
|
Unless You explicitly state otherwise, any Contribution intentionally submitted
|
||||||
|
for inclusion in the Work by You to the Licensor shall be under the terms and
|
||||||
|
conditions of this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify the terms of
|
||||||
|
any separate license agreement you may have executed with Licensor regarding
|
||||||
|
such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks.
|
||||||
|
|
||||||
|
This License does not grant permission to use the trade names, trademarks,
|
||||||
|
service marks, or product names of the Licensor, except as required for
|
||||||
|
reasonable and customary use in describing the origin of the Work and
|
||||||
|
reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty.
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, Licensor provides the
|
||||||
|
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
|
||||||
|
including, without limitation, any warranties or conditions of TITLE,
|
||||||
|
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
|
||||||
|
solely responsible for determining the appropriateness of using or
|
||||||
|
redistributing the Work and assume any risks associated with Your exercise of
|
||||||
|
permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability.
|
||||||
|
|
||||||
|
In no event and under no legal theory, whether in tort (including negligence),
|
||||||
|
contract, or otherwise, unless required by applicable law (such as deliberate
|
||||||
|
and grossly negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special, incidental,
|
||||||
|
or consequential damages of any character arising as a result of this License or
|
||||||
|
out of the use or inability to use the Work (including but not limited to
|
||||||
|
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
|
||||||
|
any and all other commercial damages or losses), even if such Contributor has
|
||||||
|
been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability.
|
||||||
|
|
||||||
|
While redistributing the Work or Derivative Works thereof, You may choose to
|
||||||
|
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
|
||||||
|
other liability obligations and/or rights consistent with this License. However,
|
||||||
|
in accepting such obligations, You may act only on Your own behalf and on Your
|
||||||
|
sole responsibility, not on behalf of any other Contributor, and only if You
|
||||||
|
agree to indemnify, defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason of your
|
||||||
|
accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following boilerplate
|
||||||
|
notice, with the fields enclosed by brackets "[]" replaced with your own
|
||||||
|
identifying information. (Don't include the brackets!) The text should be
|
||||||
|
enclosed in the appropriate comment syntax for the file format. We also
|
||||||
|
recommend that a file or class name and description of purpose be included on
|
||||||
|
the same "printed page" as the copyright notice for easier identification within
|
||||||
|
third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
73
README.md
Normal file
73
README.md
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
# groupcache
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
groupcache is a caching and cache-filling library, intended as a
|
||||||
|
replacement for memcached in many cases.
|
||||||
|
|
||||||
|
For API docs and examples, see http://godoc.org/github.com/golang/groupcache
|
||||||
|
|
||||||
|
## Comparison to memcached
|
||||||
|
|
||||||
|
### **Like memcached**, groupcache:
|
||||||
|
|
||||||
|
* shards by key to select which peer is responsible for that key
|
||||||
|
|
||||||
|
### **Unlike memcached**, groupcache:
|
||||||
|
|
||||||
|
* does not require running a separate set of servers, thus massively
|
||||||
|
reducing deployment/configuration pain. groupcache is a client
|
||||||
|
library as well as a server. It connects to its own peers.
|
||||||
|
|
||||||
|
* comes with a cache filling mechanism. Whereas memcached just says
|
||||||
|
"Sorry, cache miss", often resulting in a thundering herd of
|
||||||
|
database (or whatever) loads from an unbounded number of clients
|
||||||
|
(which has resulted in several fun outages), groupcache coordinates
|
||||||
|
cache fills such that only one load in one process of an entire
|
||||||
|
replicated set of processes populates the cache, then multiplexes
|
||||||
|
the loaded value to all callers.
|
||||||
|
|
||||||
|
* does not support versioned values. If key "foo" is value "bar",
|
||||||
|
key "foo" must always be "bar". There are neither cache expiration
|
||||||
|
times, nor explicit cache evictions. Thus there is also no CAS,
|
||||||
|
nor Increment/Decrement. This also means that groupcache....
|
||||||
|
|
||||||
|
* ... supports automatic mirroring of super-hot items to multiple
|
||||||
|
processes. This prevents memcached hot spotting where a machine's
|
||||||
|
CPU and/or NIC are overloaded by very popular keys/values.
|
||||||
|
|
||||||
|
* is currently only available for Go. It's very unlikely that I
|
||||||
|
(bradfitz@) will port the code to any other language.
|
||||||
|
|
||||||
|
## Loading process
|
||||||
|
|
||||||
|
In a nutshell, a groupcache lookup of **Get("foo")** looks like:
|
||||||
|
|
||||||
|
(On machine #5 of a set of N machines running the same code)
|
||||||
|
|
||||||
|
1. Is the value of "foo" in local memory because it's super hot? If so, use it.
|
||||||
|
|
||||||
|
2. Is the value of "foo" in local memory because peer #5 (the current
|
||||||
|
peer) is the owner of it? If so, use it.
|
||||||
|
|
||||||
|
3. Amongst all the peers in my set of N, am I the owner of the key
|
||||||
|
"foo"? (e.g. does it consistent hash to 5?) If so, load it. If
|
||||||
|
other callers come in (via the same process or via RPC requests
|
||||||
|
from peers, they block waiting for the load to finish and get the
|
||||||
|
same answer). If not, RPC to the peer that's the owner and get
|
||||||
|
the answer. If the RPC fails, just load it locally (still with
|
||||||
|
local dup suppresion).
|
||||||
|
|
||||||
|
## Users
|
||||||
|
|
||||||
|
groupcache is in production use by dl.google.com (its original user),
|
||||||
|
parts of Blogger, parts of Google Code, parts of Google Fiber, parts
|
||||||
|
of Google production monitoring systems, etc.
|
||||||
|
|
||||||
|
## Presentations
|
||||||
|
|
||||||
|
See http://talks.golang.org/2013/oscon-dl.slide
|
||||||
|
|
||||||
|
## Help
|
||||||
|
|
||||||
|
Use the golang-nuts mailing list for any discussion or questions.
|
160
byteview.go
Normal file
160
byteview.go
Normal file
@ -0,0 +1,160 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A ByteView holds an immutable view of bytes.
|
||||||
|
// Internally it wraps either a []byte or a string,
|
||||||
|
// but that detail is invisible to callers.
|
||||||
|
//
|
||||||
|
// A ByteView is meant to be used as a value type, not
|
||||||
|
// a pointer (like a time.Time).
|
||||||
|
type ByteView struct {
|
||||||
|
// If b is non-nil, b is used, else s is used.
|
||||||
|
b []byte
|
||||||
|
s string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Len returns the view's length.
|
||||||
|
func (v ByteView) Len() int {
|
||||||
|
if v.b != nil {
|
||||||
|
return len(v.b)
|
||||||
|
}
|
||||||
|
return len(v.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ByteSlice returns a copy of the data as a byte slice.
|
||||||
|
func (v ByteView) ByteSlice() []byte {
|
||||||
|
if v.b != nil {
|
||||||
|
return cloneBytes(v.b)
|
||||||
|
}
|
||||||
|
return []byte(v.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns the data as a string, making a copy if necessary.
|
||||||
|
func (v ByteView) String() string {
|
||||||
|
if v.b != nil {
|
||||||
|
return string(v.b)
|
||||||
|
}
|
||||||
|
return v.s
|
||||||
|
}
|
||||||
|
|
||||||
|
// At returns the byte at index i.
|
||||||
|
func (v ByteView) At(i int) byte {
|
||||||
|
if v.b != nil {
|
||||||
|
return v.b[i]
|
||||||
|
}
|
||||||
|
return v.s[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slice slices the view between the provided from and to indices.
|
||||||
|
func (v ByteView) Slice(from, to int) ByteView {
|
||||||
|
if v.b != nil {
|
||||||
|
return ByteView{b: v.b[from:to]}
|
||||||
|
}
|
||||||
|
return ByteView{s: v.s[from:to]}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SliceFrom slices the view from the provided index until the end.
|
||||||
|
func (v ByteView) SliceFrom(from int) ByteView {
|
||||||
|
if v.b != nil {
|
||||||
|
return ByteView{b: v.b[from:]}
|
||||||
|
}
|
||||||
|
return ByteView{s: v.s[from:]}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies b into dest and returns the number of bytes copied.
|
||||||
|
func (v ByteView) Copy(dest []byte) int {
|
||||||
|
if v.b != nil {
|
||||||
|
return copy(dest, v.b)
|
||||||
|
}
|
||||||
|
return copy(dest, v.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Equal returns whether the bytes in b are the same as the bytes in
|
||||||
|
// b2.
|
||||||
|
func (v ByteView) Equal(b2 ByteView) bool {
|
||||||
|
if b2.b == nil {
|
||||||
|
return v.EqualString(b2.s)
|
||||||
|
}
|
||||||
|
return v.EqualBytes(b2.b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EqualString returns whether the bytes in b are the same as the bytes
|
||||||
|
// in s.
|
||||||
|
func (v ByteView) EqualString(s string) bool {
|
||||||
|
if v.b == nil {
|
||||||
|
return v.s == s
|
||||||
|
}
|
||||||
|
l := v.Len()
|
||||||
|
if len(s) != l {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i, bi := range v.b {
|
||||||
|
if bi != s[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// EqualBytes returns whether the bytes in b are the same as the bytes
|
||||||
|
// in b2.
|
||||||
|
func (v ByteView) EqualBytes(b2 []byte) bool {
|
||||||
|
if v.b != nil {
|
||||||
|
return bytes.Equal(v.b, b2)
|
||||||
|
}
|
||||||
|
l := v.Len()
|
||||||
|
if len(b2) != l {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i, bi := range b2 {
|
||||||
|
if bi != v.s[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader returns an io.ReadSeeker for the bytes in v.
|
||||||
|
func (v ByteView) Reader() io.ReadSeeker {
|
||||||
|
if v.b != nil {
|
||||||
|
return bytes.NewReader(v.b)
|
||||||
|
}
|
||||||
|
return strings.NewReader(v.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAt implements io.ReaderAt on the bytes in v.
|
||||||
|
func (v ByteView) ReadAt(p []byte, off int64) (n int, err error) {
|
||||||
|
if off < 0 {
|
||||||
|
return 0, errors.New("view: invalid offset")
|
||||||
|
}
|
||||||
|
if off >= int64(v.Len()) {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
n = v.SliceFrom(int(off)).Copy(p)
|
||||||
|
if n < len(p) {
|
||||||
|
err = io.EOF
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
142
byteview_test.go
Normal file
142
byteview_test.go
Normal file
@ -0,0 +1,142 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestByteView(t *testing.T) {
|
||||||
|
for _, s := range []string{"", "x", "yy"} {
|
||||||
|
for _, v := range []ByteView{of([]byte(s)), of(s)} {
|
||||||
|
name := fmt.Sprintf("string %q, view %+v", s, v)
|
||||||
|
if v.Len() != len(s) {
|
||||||
|
t.Errorf("%s: Len = %d; want %d", name, v.Len(), len(s))
|
||||||
|
}
|
||||||
|
if v.String() != s {
|
||||||
|
t.Errorf("%s: String = %q; want %q", name, v.String(), s)
|
||||||
|
}
|
||||||
|
var longDest [3]byte
|
||||||
|
if n := v.Copy(longDest[:]); n != len(s) {
|
||||||
|
t.Errorf("%s: long Copy = %d; want %d", name, n, len(s))
|
||||||
|
}
|
||||||
|
var shortDest [1]byte
|
||||||
|
if n := v.Copy(shortDest[:]); n != min(len(s), 1) {
|
||||||
|
t.Errorf("%s: short Copy = %d; want %d", name, n, min(len(s), 1))
|
||||||
|
}
|
||||||
|
if got, err := ioutil.ReadAll(v.Reader()); err != nil || string(got) != s {
|
||||||
|
t.Errorf("%s: Reader = %q, %v; want %q", name, got, err, s)
|
||||||
|
}
|
||||||
|
if got, err := ioutil.ReadAll(io.NewSectionReader(v, 0, int64(len(s)))); err != nil || string(got) != s {
|
||||||
|
t.Errorf("%s: SectionReader of ReaderAt = %q, %v; want %q", name, got, err, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// of returns a byte view of the []byte or string in x.
|
||||||
|
func of(x interface{}) ByteView {
|
||||||
|
if bytes, ok := x.([]byte); ok {
|
||||||
|
return ByteView{b: bytes}
|
||||||
|
}
|
||||||
|
return ByteView{s: x.(string)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestByteViewEqual(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
a interface{} // string or []byte
|
||||||
|
b interface{} // string or []byte
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"x", "x", true},
|
||||||
|
{"x", "y", false},
|
||||||
|
{"x", "yy", false},
|
||||||
|
{[]byte("x"), []byte("x"), true},
|
||||||
|
{[]byte("x"), []byte("y"), false},
|
||||||
|
{[]byte("x"), []byte("yy"), false},
|
||||||
|
{[]byte("x"), "x", true},
|
||||||
|
{[]byte("x"), "y", false},
|
||||||
|
{[]byte("x"), "yy", false},
|
||||||
|
{"x", []byte("x"), true},
|
||||||
|
{"x", []byte("y"), false},
|
||||||
|
{"x", []byte("yy"), false},
|
||||||
|
}
|
||||||
|
for i, tt := range tests {
|
||||||
|
va := of(tt.a)
|
||||||
|
if bytes, ok := tt.b.([]byte); ok {
|
||||||
|
if got := va.EqualBytes(bytes); got != tt.want {
|
||||||
|
t.Errorf("%d. EqualBytes = %v; want %v", i, got, tt.want)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if got := va.EqualString(tt.b.(string)); got != tt.want {
|
||||||
|
t.Errorf("%d. EqualString = %v; want %v", i, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := va.Equal(of(tt.b)); got != tt.want {
|
||||||
|
t.Errorf("%d. Equal = %v; want %v", i, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestByteViewSlice(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
in string
|
||||||
|
from int
|
||||||
|
to interface{} // nil to mean the end (SliceFrom); else int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
in: "abc",
|
||||||
|
from: 1,
|
||||||
|
to: 2,
|
||||||
|
want: "b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
in: "abc",
|
||||||
|
from: 1,
|
||||||
|
want: "bc",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
in: "abc",
|
||||||
|
to: 2,
|
||||||
|
want: "ab",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for i, tt := range tests {
|
||||||
|
for _, v := range []ByteView{of([]byte(tt.in)), of(tt.in)} {
|
||||||
|
name := fmt.Sprintf("test %d, view %+v", i, v)
|
||||||
|
if tt.to != nil {
|
||||||
|
v = v.Slice(tt.from, tt.to.(int))
|
||||||
|
} else {
|
||||||
|
v = v.SliceFrom(tt.from)
|
||||||
|
}
|
||||||
|
if v.String() != tt.want {
|
||||||
|
t.Errorf("%s: got %q; want %q", name, v.String(), tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func min(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
455
groupcache.go
Normal file
455
groupcache.go
Normal file
@ -0,0 +1,455 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package groupcache provides a data loading mechanism with caching
|
||||||
|
// and de-duplication that works across a set of peer processes.
|
||||||
|
//
|
||||||
|
// Each data Get first consults its local cache, otherwise delegates
|
||||||
|
// to the requested key's canonical owner, which then checks its cache
|
||||||
|
// or finally gets the data. In the common case, many concurrent
|
||||||
|
// cache misses across a set of peers for the same key result in just
|
||||||
|
// one cache fill.
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math/rand"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
pb "github.com/golang/groupcache/groupcachepb"
|
||||||
|
"github.com/golang/groupcache/lru"
|
||||||
|
"github.com/golang/groupcache/singleflight"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Getter loads data for a key.
|
||||||
|
type Getter interface {
|
||||||
|
// Get returns the value identified by key, populating dest.
|
||||||
|
//
|
||||||
|
// The returned data must be unversioned. That is, key must
|
||||||
|
// uniquely describe the loaded data, without an implicit
|
||||||
|
// current time, and without relying on cache expiration
|
||||||
|
// mechanisms.
|
||||||
|
Get(ctx Context, key string, dest Sink) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// A GetterFunc implements Getter with a function.
|
||||||
|
type GetterFunc func(ctx Context, key string, dest Sink) error
|
||||||
|
|
||||||
|
func (f GetterFunc) Get(ctx Context, key string, dest Sink) error {
|
||||||
|
return f(ctx, key, dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.RWMutex
|
||||||
|
groups = make(map[string]*Group)
|
||||||
|
|
||||||
|
initPeerServerOnce sync.Once
|
||||||
|
initPeerServer func()
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetGroup returns the named group previously created with NewGroup, or
|
||||||
|
// nil if there's no such group.
|
||||||
|
func GetGroup(name string) *Group {
|
||||||
|
mu.RLock()
|
||||||
|
g := groups[name]
|
||||||
|
mu.RUnlock()
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGroup creates a coordinated group-aware Getter from a Getter.
|
||||||
|
//
|
||||||
|
// The returned Getter tries (but does not guarantee) to run only one
|
||||||
|
// Get call at once for a given key across an entire set of peer
|
||||||
|
// processes. Concurrent callers both in the local process and in
|
||||||
|
// other processes receive copies of the answer once the original Get
|
||||||
|
// completes.
|
||||||
|
//
|
||||||
|
// The group name must be unique for each getter.
|
||||||
|
func NewGroup(name string, cacheBytes int64, getter Getter) *Group {
|
||||||
|
return newGroup(name, cacheBytes, getter, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If peers is nil, the peerPicker is called via a sync.Once to initialize it.
|
||||||
|
func newGroup(name string, cacheBytes int64, getter Getter, peers PeerPicker) *Group {
|
||||||
|
if getter == nil {
|
||||||
|
panic("nil Getter")
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
initPeerServerOnce.Do(callInitPeerServer)
|
||||||
|
if _, dup := groups[name]; dup {
|
||||||
|
panic("duplicate registration of group " + name)
|
||||||
|
}
|
||||||
|
g := &Group{
|
||||||
|
name: name,
|
||||||
|
getter: getter,
|
||||||
|
peers: peers,
|
||||||
|
cacheBytes: cacheBytes,
|
||||||
|
}
|
||||||
|
if fn := newGroupHook; fn != nil {
|
||||||
|
fn(g)
|
||||||
|
}
|
||||||
|
groups[name] = g
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
// newGroupHook, if non-nil, is called right after a new group is created.
|
||||||
|
var newGroupHook func(*Group)
|
||||||
|
|
||||||
|
// RegisterNewGroupHook registers a hook that is run each time
|
||||||
|
// a group is created.
|
||||||
|
func RegisterNewGroupHook(fn func(*Group)) {
|
||||||
|
if newGroupHook != nil {
|
||||||
|
panic("RegisterNewGroupHook called more than once")
|
||||||
|
}
|
||||||
|
newGroupHook = fn
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterServerStart registers a hook that is run when the first
|
||||||
|
// group is created.
|
||||||
|
func RegisterServerStart(fn func()) {
|
||||||
|
if initPeerServer != nil {
|
||||||
|
panic("RegisterServerStart called more than once")
|
||||||
|
}
|
||||||
|
initPeerServer = fn
|
||||||
|
}
|
||||||
|
|
||||||
|
func callInitPeerServer() {
|
||||||
|
if initPeerServer != nil {
|
||||||
|
initPeerServer()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Group is a cache namespace and associated data loaded spread over
|
||||||
|
// a group of 1 or more machines.
|
||||||
|
type Group struct {
|
||||||
|
name string
|
||||||
|
getter Getter
|
||||||
|
peersOnce sync.Once
|
||||||
|
peers PeerPicker
|
||||||
|
cacheBytes int64 // limit for sum of mainCache and hotCache size
|
||||||
|
|
||||||
|
// mainCache is a cache of the keys for which this process
|
||||||
|
// (amongst its peers) is authorative. That is, this cache
|
||||||
|
// contains keys which consistent hash on to this process's
|
||||||
|
// peer number.
|
||||||
|
mainCache cache
|
||||||
|
|
||||||
|
// hotCache contains keys/values for which this peer is not
|
||||||
|
// authorative (otherwise they would be in mainCache), but
|
||||||
|
// are popular enough to warrant mirroring in this process to
|
||||||
|
// avoid going over the network to fetch from a peer. Having
|
||||||
|
// a hotCache avoids network hotspotting, where a peer's
|
||||||
|
// network card could become the bottleneck on a popular key.
|
||||||
|
// This cache is used sparingly to maximize the total number
|
||||||
|
// of key/value pairs that can stored globally.
|
||||||
|
hotCache cache
|
||||||
|
|
||||||
|
// loadGroup ensures that each key is only fetched once
|
||||||
|
// (either locally or remotely), regardless of the number of
|
||||||
|
// concurrent callers.
|
||||||
|
loadGroup singleflight.Group
|
||||||
|
|
||||||
|
// Stats are statistics on the group.
|
||||||
|
Stats Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats are per-group statistics.
|
||||||
|
type Stats struct {
|
||||||
|
Gets AtomicInt // any Get request, including from peers
|
||||||
|
CacheHits AtomicInt // either cache was good
|
||||||
|
PeerLoads AtomicInt // either remote load or remote cache hit (not an error)
|
||||||
|
PeerErrors AtomicInt
|
||||||
|
Loads AtomicInt // (gets - cacheHits)
|
||||||
|
LoadsDeduped AtomicInt // after singleflight
|
||||||
|
LocalLoads AtomicInt // total good local loads
|
||||||
|
LocalLoadErrs AtomicInt // total bad local loads
|
||||||
|
ServerRequests AtomicInt // gets that came over the network from peers
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the group.
|
||||||
|
func (g *Group) Name() string {
|
||||||
|
return g.name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) initPeers() {
|
||||||
|
if g.peers == nil {
|
||||||
|
g.peers = getPeers()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) Get(ctx Context, key string, dest Sink) error {
|
||||||
|
g.peersOnce.Do(g.initPeers)
|
||||||
|
g.Stats.Gets.Add(1)
|
||||||
|
if dest == nil {
|
||||||
|
return errors.New("groupcache: nil dest Sink")
|
||||||
|
}
|
||||||
|
value, cacheHit := g.lookupCache(key)
|
||||||
|
|
||||||
|
if cacheHit {
|
||||||
|
g.Stats.CacheHits.Add(1)
|
||||||
|
return setSinkView(dest, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Optimization to avoid double unmarshalling or copying: keep
|
||||||
|
// track of whether the dest was already populated. One caller
|
||||||
|
// (if local) will set this; the losers will not. The common
|
||||||
|
// case will likely be one caller.
|
||||||
|
destPopulated := false
|
||||||
|
value, destPopulated, err := g.load(ctx, key, dest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if destPopulated {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return setSinkView(dest, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// load loads key either by invoking the getter locally or by sending it to another machine.
|
||||||
|
func (g *Group) load(ctx Context, key string, dest Sink) (value ByteView, destPopulated bool, err error) {
|
||||||
|
g.Stats.Loads.Add(1)
|
||||||
|
viewi, err := g.loadGroup.Do(key, func() (interface{}, error) {
|
||||||
|
g.Stats.LoadsDeduped.Add(1)
|
||||||
|
var value ByteView
|
||||||
|
var err error
|
||||||
|
if peer, ok := g.peers.PickPeer(key); ok {
|
||||||
|
value, err = g.getFromPeer(ctx, peer, key)
|
||||||
|
if err == nil {
|
||||||
|
g.Stats.PeerLoads.Add(1)
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
g.Stats.PeerErrors.Add(1)
|
||||||
|
// TODO(bradfitz): log the peer's error? keep
|
||||||
|
// log of the past few for /groupcachez? It's
|
||||||
|
// probably boring (normal task movement), so not
|
||||||
|
// worth logging I imagine.
|
||||||
|
}
|
||||||
|
value, err = g.getLocally(ctx, key, dest)
|
||||||
|
if err != nil {
|
||||||
|
g.Stats.LocalLoadErrs.Add(1)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
g.Stats.LocalLoads.Add(1)
|
||||||
|
destPopulated = true // only one caller of load gets this return value
|
||||||
|
g.populateCache(key, value, &g.mainCache)
|
||||||
|
return value, nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
value = viewi.(ByteView)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) getLocally(ctx Context, key string, dest Sink) (ByteView, error) {
|
||||||
|
err := g.getter.Get(ctx, key, dest)
|
||||||
|
if err != nil {
|
||||||
|
return ByteView{}, err
|
||||||
|
}
|
||||||
|
return dest.view()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) getFromPeer(ctx Context, peer ProtoGetter, key string) (ByteView, error) {
|
||||||
|
req := &pb.GetRequest{
|
||||||
|
Group: &g.name,
|
||||||
|
Key: &key,
|
||||||
|
}
|
||||||
|
res := &pb.GetResponse{}
|
||||||
|
err := peer.Get(ctx, req, res)
|
||||||
|
if err != nil {
|
||||||
|
return ByteView{}, err
|
||||||
|
}
|
||||||
|
value := ByteView{b: res.Value}
|
||||||
|
// TODO(bradfitz): use res.MinuteQps or something smart to
|
||||||
|
// conditionally populate hotCache. For now just do it some
|
||||||
|
// percentage of the time.
|
||||||
|
if rand.Intn(10) == 0 {
|
||||||
|
g.populateCache(key, value, &g.hotCache)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) lookupCache(key string) (value ByteView, ok bool) {
|
||||||
|
if g.cacheBytes <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
value, ok = g.mainCache.get(key)
|
||||||
|
if ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
value, ok = g.hotCache.get(key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Group) populateCache(key string, value ByteView, cache *cache) {
|
||||||
|
if g.cacheBytes <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cache.add(key, value)
|
||||||
|
|
||||||
|
// Evict items from cache(s) if necessary.
|
||||||
|
for {
|
||||||
|
mainBytes := g.mainCache.bytes()
|
||||||
|
hotBytes := g.hotCache.bytes()
|
||||||
|
if mainBytes+hotBytes <= g.cacheBytes {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO(bradfitz): this is good-enough-for-now logic.
|
||||||
|
// It should be something based on measurements and/or
|
||||||
|
// respecting the costs of different resources.
|
||||||
|
victim := &g.mainCache
|
||||||
|
if hotBytes > mainBytes/8 {
|
||||||
|
victim = &g.hotCache
|
||||||
|
}
|
||||||
|
victim.removeOldest()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheType represents a type of cache.
|
||||||
|
type CacheType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// The MainCache is the cache for items that this peer is the
|
||||||
|
// owner for.
|
||||||
|
MainCache CacheType = iota + 1
|
||||||
|
|
||||||
|
// The HotCache is the cache for items that seem popular
|
||||||
|
// enough to replicate to this node, even though it's not the
|
||||||
|
// owner.
|
||||||
|
HotCache
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheStats returns stats about the provided cache within the group.
|
||||||
|
func (g *Group) CacheStats(which CacheType) CacheStats {
|
||||||
|
switch which {
|
||||||
|
case MainCache:
|
||||||
|
return g.mainCache.stats()
|
||||||
|
case HotCache:
|
||||||
|
return g.hotCache.stats()
|
||||||
|
default:
|
||||||
|
return CacheStats{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cache is a wrapper around an *lru.Cache that adds synchronization,
|
||||||
|
// makes values always be ByteView, and counts the size of all keys and
|
||||||
|
// values.
|
||||||
|
type cache struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
nbytes int64 // of all keys and values
|
||||||
|
lru *lru.Cache
|
||||||
|
nhit, nget int64
|
||||||
|
nevict int64 // number of evictions
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) stats() CacheStats {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
return CacheStats{
|
||||||
|
Bytes: c.nbytes,
|
||||||
|
Items: c.itemsLocked(),
|
||||||
|
Gets: c.nget,
|
||||||
|
Hits: c.nhit,
|
||||||
|
Evictions: c.nevict,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) add(key string, value ByteView) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.lru == nil {
|
||||||
|
c.lru = &lru.Cache{
|
||||||
|
OnEvicted: func(key lru.Key, value interface{}) {
|
||||||
|
val := value.(ByteView)
|
||||||
|
c.nbytes -= int64(len(key.(string))) + int64(val.Len())
|
||||||
|
c.nevict++
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.lru.Add(key, value)
|
||||||
|
c.nbytes += int64(len(key)) + int64(value.Len())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) get(key string) (value ByteView, ok bool) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.nget++
|
||||||
|
if c.lru == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
vi, ok := c.lru.Get(key)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.nhit++
|
||||||
|
return vi.(ByteView), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) removeOldest() {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.lru != nil {
|
||||||
|
c.lru.RemoveOldest()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) bytes() int64 {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
return c.nbytes
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) items() int64 {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
return c.itemsLocked()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cache) itemsLocked() int64 {
|
||||||
|
if c.lru == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int64(c.lru.Len())
|
||||||
|
}
|
||||||
|
|
||||||
|
// An AtomicInt is an int64 to be accessed atomically.
|
||||||
|
type AtomicInt int64
|
||||||
|
|
||||||
|
// Add atomically adds n to i.
|
||||||
|
func (i *AtomicInt) Add(n int64) {
|
||||||
|
atomic.AddInt64((*int64)(i), n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get atomically gets the value of i.
|
||||||
|
func (i *AtomicInt) Get() int64 {
|
||||||
|
return atomic.LoadInt64((*int64)(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *AtomicInt) String() string {
|
||||||
|
return strconv.FormatInt(i.Get(), 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheStats are returned by stats accessors on Group.
|
||||||
|
type CacheStats struct {
|
||||||
|
Bytes int64
|
||||||
|
Items int64
|
||||||
|
Gets int64
|
||||||
|
Hits int64
|
||||||
|
Evictions int64
|
||||||
|
}
|
65
groupcache.pb.go
Normal file
65
groupcache.pb.go
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
// Code generated by protoc-gen-go.
|
||||||
|
// source: groupcache.proto
|
||||||
|
// DO NOT EDIT!
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import proto "code.google.com/p/goprotobuf/proto"
|
||||||
|
import json "encoding/json"
|
||||||
|
import math "math"
|
||||||
|
|
||||||
|
// Reference proto, json, and math imports to suppress error if they are not otherwise used.
|
||||||
|
var _ = proto.Marshal
|
||||||
|
var _ = &json.SyntaxError{}
|
||||||
|
var _ = math.Inf
|
||||||
|
|
||||||
|
type GetRequest struct {
|
||||||
|
Group *string `protobuf:"bytes,1,req,name=group" json:"group,omitempty"`
|
||||||
|
Key *string `protobuf:"bytes,2,req,name=key" json:"key,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetRequest) Reset() { *m = GetRequest{} }
|
||||||
|
func (m *GetRequest) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*GetRequest) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *GetRequest) GetGroup() string {
|
||||||
|
if m != nil && m.Group != nil {
|
||||||
|
return *m.Group
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetRequest) GetKey() string {
|
||||||
|
if m != nil && m.Key != nil {
|
||||||
|
return *m.Key
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type GetResponse struct {
|
||||||
|
Value []byte `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"`
|
||||||
|
MinuteQps *float64 `protobuf:"fixed64,2,opt,name=minute_qps" json:"minute_qps,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetResponse) Reset() { *m = GetResponse{} }
|
||||||
|
func (m *GetResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*GetResponse) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *GetResponse) GetValue() []byte {
|
||||||
|
if m != nil {
|
||||||
|
return m.Value
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetResponse) GetMinuteQps() float64 {
|
||||||
|
if m != nil && m.MinuteQps != nil {
|
||||||
|
return *m.MinuteQps
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
}
|
367
groupcache_test.go
Normal file
367
groupcache_test.go
Normal file
@ -0,0 +1,367 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Tests for groupcache.
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"hash/crc32"
|
||||||
|
"math/rand"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.google.com/p/goprotobuf/proto"
|
||||||
|
|
||||||
|
pb "github.com/golang/groupcache/groupcachepb"
|
||||||
|
testpb "github.com/golang/groupcache/testpb"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
once sync.Once
|
||||||
|
stringGroup, protoGroup Getter
|
||||||
|
|
||||||
|
stringc = make(chan string)
|
||||||
|
|
||||||
|
dummyCtx Context
|
||||||
|
|
||||||
|
// cacheFills is the number of times stringGroup or
|
||||||
|
// protoGroup's Getter have been called. Read using the
|
||||||
|
// cacheFills function.
|
||||||
|
cacheFills AtomicInt
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
stringGroupName = "string-group"
|
||||||
|
protoGroupName = "proto-group"
|
||||||
|
testMessageType = "google3/net/groupcache/go/test_proto.TestMessage"
|
||||||
|
fromChan = "from-chan"
|
||||||
|
cacheSize = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
func testSetup() {
|
||||||
|
stringGroup = NewGroup(stringGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
|
||||||
|
if key == fromChan {
|
||||||
|
key = <-stringc
|
||||||
|
}
|
||||||
|
cacheFills.Add(1)
|
||||||
|
return dest.SetString("ECHO:" + key)
|
||||||
|
}))
|
||||||
|
|
||||||
|
protoGroup = NewGroup(protoGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
|
||||||
|
if key == fromChan {
|
||||||
|
key = <-stringc
|
||||||
|
}
|
||||||
|
cacheFills.Add(1)
|
||||||
|
return dest.SetProto(&testpb.TestMessage{
|
||||||
|
Name: proto.String("ECHO:" + key),
|
||||||
|
City: proto.String("SOME-CITY"),
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// tests that a Getter's Get method is only called once with two
|
||||||
|
// outstanding callers. This is the string variant.
|
||||||
|
func TestGetDupSuppressString(t *testing.T) {
|
||||||
|
once.Do(testSetup)
|
||||||
|
// Start two getters. The first should block (waiting reading
|
||||||
|
// from stringc) and the second should latch on to the first
|
||||||
|
// one.
|
||||||
|
resc := make(chan string, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go func() {
|
||||||
|
var s string
|
||||||
|
if err := stringGroup.Get(dummyCtx, fromChan, StringSink(&s)); err != nil {
|
||||||
|
resc <- "ERROR:" + err.Error()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resc <- s
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait a bit so both goroutines get merged together via
|
||||||
|
// singleflight.
|
||||||
|
// TODO(bradfitz): decide whether there are any non-offensive
|
||||||
|
// debug/test hooks that could be added to singleflight to
|
||||||
|
// make a sleep here unnecessary.
|
||||||
|
time.Sleep(250 * time.Millisecond)
|
||||||
|
|
||||||
|
// Unblock the first getter, which should unblock the second
|
||||||
|
// as well.
|
||||||
|
stringc <- "foo"
|
||||||
|
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
select {
|
||||||
|
case v := <-resc:
|
||||||
|
if v != "ECHO:foo" {
|
||||||
|
t.Errorf("got %q; want %q", v, "ECHO:foo")
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Errorf("timeout waiting on getter #%d of 2", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tests that a Getter's Get method is only called once with two
|
||||||
|
// outstanding callers. This is the proto variant.
|
||||||
|
func TestGetDupSuppressProto(t *testing.T) {
|
||||||
|
once.Do(testSetup)
|
||||||
|
// Start two getters. The first should block (waiting reading
|
||||||
|
// from stringc) and the second should latch on to the first
|
||||||
|
// one.
|
||||||
|
resc := make(chan *testpb.TestMessage, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go func() {
|
||||||
|
tm := new(testpb.TestMessage)
|
||||||
|
if err := protoGroup.Get(dummyCtx, fromChan, ProtoSink(tm)); err != nil {
|
||||||
|
tm.Name = proto.String("ERROR:" + err.Error())
|
||||||
|
}
|
||||||
|
resc <- tm
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait a bit so both goroutines get merged together via
|
||||||
|
// singleflight.
|
||||||
|
// TODO(bradfitz): decide whether there are any non-offensive
|
||||||
|
// debug/test hooks that could be added to singleflight to
|
||||||
|
// make a sleep here unnecessary.
|
||||||
|
time.Sleep(250 * time.Millisecond)
|
||||||
|
|
||||||
|
// Unblock the first getter, which should unblock the second
|
||||||
|
// as well.
|
||||||
|
stringc <- "Fluffy"
|
||||||
|
want := &testpb.TestMessage{
|
||||||
|
Name: proto.String("ECHO:Fluffy"),
|
||||||
|
City: proto.String("SOME-CITY"),
|
||||||
|
}
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
select {
|
||||||
|
case v := <-resc:
|
||||||
|
if !reflect.DeepEqual(v, want) {
|
||||||
|
t.Errorf(" Got: %v\nWant: %v", proto.CompactTextString(v), proto.CompactTextString(want))
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Errorf("timeout waiting on getter #%d of 2", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func countFills(f func()) int64 {
|
||||||
|
fills0 := cacheFills.Get()
|
||||||
|
f()
|
||||||
|
return cacheFills.Get() - fills0
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCaching(t *testing.T) {
|
||||||
|
once.Do(testSetup)
|
||||||
|
fills := countFills(func() {
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
var s string
|
||||||
|
if err := stringGroup.Get(dummyCtx, "TestCaching-key", StringSink(&s)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if fills != 1 {
|
||||||
|
t.Errorf("expected 1 cache fill; got %d", fills)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCacheEviction(t *testing.T) {
|
||||||
|
once.Do(testSetup)
|
||||||
|
testKey := "TestCacheEviction-key"
|
||||||
|
getTestKey := func() {
|
||||||
|
var res string
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
if err := stringGroup.Get(dummyCtx, testKey, StringSink(&res)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fills := countFills(getTestKey)
|
||||||
|
if fills != 1 {
|
||||||
|
t.Fatalf("expected 1 cache fill; got %d", fills)
|
||||||
|
}
|
||||||
|
|
||||||
|
g := stringGroup.(*Group)
|
||||||
|
evict0 := g.mainCache.nevict
|
||||||
|
|
||||||
|
// Trash the cache with other keys.
|
||||||
|
var bytesFlooded int64
|
||||||
|
// cacheSize/len(testKey) is approximate
|
||||||
|
for bytesFlooded < cacheSize+1024 {
|
||||||
|
var res string
|
||||||
|
key := fmt.Sprintf("dummy-key-%d", bytesFlooded)
|
||||||
|
stringGroup.Get(dummyCtx, key, StringSink(&res))
|
||||||
|
bytesFlooded += int64(len(key) + len(res))
|
||||||
|
}
|
||||||
|
evicts := g.mainCache.nevict - evict0
|
||||||
|
if evicts <= 0 {
|
||||||
|
t.Errorf("evicts = %v; want more than 0", evicts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test that the key is gone.
|
||||||
|
fills = countFills(getTestKey)
|
||||||
|
if fills != 1 {
|
||||||
|
t.Fatalf("expected 1 cache fill after cache trashing; got %d", fills)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakePeer struct {
|
||||||
|
hits int
|
||||||
|
fail bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *fakePeer) Get(_ Context, in *pb.GetRequest, out *pb.GetResponse) error {
|
||||||
|
p.hits++
|
||||||
|
if p.fail {
|
||||||
|
return errors.New("simulated error from peer")
|
||||||
|
}
|
||||||
|
out.Value = []byte("got:" + in.GetKey())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakePeers []ProtoGetter
|
||||||
|
|
||||||
|
func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
|
||||||
|
if len(p) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
|
||||||
|
return p[n], p[n] != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tests that peers (virtual, in-process) are hit, and how much.
|
||||||
|
func TestPeers(t *testing.T) {
|
||||||
|
once.Do(testSetup)
|
||||||
|
rand.Seed(123)
|
||||||
|
peer0 := &fakePeer{}
|
||||||
|
peer1 := &fakePeer{}
|
||||||
|
peer2 := &fakePeer{}
|
||||||
|
peerList := fakePeers([]ProtoGetter{peer0, peer1, peer2, nil})
|
||||||
|
const cacheSize = 0 // disabled
|
||||||
|
localHits := 0
|
||||||
|
getter := func(_ Context, key string, dest Sink) error {
|
||||||
|
localHits++
|
||||||
|
return dest.SetString("got:" + key)
|
||||||
|
}
|
||||||
|
testGroup := newGroup("TestPeers-group", cacheSize, GetterFunc(getter), peerList)
|
||||||
|
run := func(name string, n int, wantSummary string) {
|
||||||
|
// Reset counters
|
||||||
|
localHits = 0
|
||||||
|
for _, p := range []*fakePeer{peer0, peer1, peer2} {
|
||||||
|
p.hits = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
key := fmt.Sprintf("key-%d", i)
|
||||||
|
want := "got:" + key
|
||||||
|
var got string
|
||||||
|
err := testGroup.Get(dummyCtx, key, StringSink(&got))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: error on key %q: %v", name, key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("%s: for key %q, got %q; want %q", name, key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
summary := func() string {
|
||||||
|
return fmt.Sprintf("localHits = %d, peers = %d %d %d", localHits, peer0.hits, peer1.hits, peer2.hits)
|
||||||
|
}
|
||||||
|
if got := summary(); got != wantSummary {
|
||||||
|
t.Errorf("%s: got %q; want %q", name, got, wantSummary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resetCacheSize := func(maxBytes int64) {
|
||||||
|
g := testGroup
|
||||||
|
g.cacheBytes = maxBytes
|
||||||
|
g.mainCache = cache{}
|
||||||
|
g.hotCache = cache{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Base case; peers all up, with no problems.
|
||||||
|
resetCacheSize(1 << 20)
|
||||||
|
run("base", 200, "localHits = 49, peers = 51 49 51")
|
||||||
|
|
||||||
|
// Verify cache was hit. All localHits are gone, and some of
|
||||||
|
// the peer hits (the ones randomly selected to be maybe hot)
|
||||||
|
run("cached_base", 200, "localHits = 0, peers = 49 47 48")
|
||||||
|
resetCacheSize(0)
|
||||||
|
|
||||||
|
// With one of the peers being down.
|
||||||
|
// TODO(bradfitz): on a peer number being unavailable, the
|
||||||
|
// consistent hashing should maybe keep trying others to
|
||||||
|
// spread the load out. Currently it fails back to local
|
||||||
|
// execution if the first consistent-hash slot is unavailable.
|
||||||
|
peerList[0] = nil
|
||||||
|
run("one_peer_down", 200, "localHits = 100, peers = 0 49 51")
|
||||||
|
|
||||||
|
// Failing peer
|
||||||
|
peerList[0] = peer0
|
||||||
|
peer0.fail = true
|
||||||
|
run("peer0_failing", 200, "localHits = 100, peers = 51 49 51")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTruncatingByteSliceTarget(t *testing.T) {
|
||||||
|
var buf [100]byte
|
||||||
|
s := buf[:]
|
||||||
|
if err := stringGroup.Get(dummyCtx, "short", TruncatingByteSliceSink(&s)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := "ECHO:short"; string(s) != want {
|
||||||
|
t.Errorf("short key got %q; want %q", s, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
s = buf[:6]
|
||||||
|
if err := stringGroup.Get(dummyCtx, "truncated", TruncatingByteSliceSink(&s)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := "ECHO:t"; string(s) != want {
|
||||||
|
t.Errorf("truncated key got %q; want %q", s, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllocatingByteSliceTarget(t *testing.T) {
|
||||||
|
var dst []byte
|
||||||
|
sink := AllocatingByteSliceSink(&dst)
|
||||||
|
|
||||||
|
inBytes := []byte("some bytes")
|
||||||
|
sink.SetBytes(inBytes)
|
||||||
|
if want := "some bytes"; string(dst) != want {
|
||||||
|
t.Errorf("SetBytes resulted in %q; want %q", dst, want)
|
||||||
|
}
|
||||||
|
v, err := sink.view()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("view after SetBytes failed: %v", err)
|
||||||
|
}
|
||||||
|
if &inBytes[0] == &dst[0] {
|
||||||
|
t.Error("inBytes and dst share memory")
|
||||||
|
}
|
||||||
|
if &inBytes[0] == &v.b[0] {
|
||||||
|
t.Error("inBytes and view share memory")
|
||||||
|
}
|
||||||
|
if &dst[0] == &v.b[0] {
|
||||||
|
t.Error("dst and view share memory")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO(bradfitz): port the Google-internal full integration test into here,
|
||||||
|
// using HTTP requests instead of our RPC system.
|
65
groupcachepb/groupcache.pb.go
Normal file
65
groupcachepb/groupcache.pb.go
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
// Code generated by protoc-gen-go.
|
||||||
|
// source: groupcache.proto
|
||||||
|
// DO NOT EDIT!
|
||||||
|
|
||||||
|
package groupcachepb
|
||||||
|
|
||||||
|
import proto "code.google.com/p/goprotobuf/proto"
|
||||||
|
import json "encoding/json"
|
||||||
|
import math "math"
|
||||||
|
|
||||||
|
// Reference proto, json, and math imports to suppress error if they are not otherwise used.
|
||||||
|
var _ = proto.Marshal
|
||||||
|
var _ = &json.SyntaxError{}
|
||||||
|
var _ = math.Inf
|
||||||
|
|
||||||
|
type GetRequest struct {
|
||||||
|
Group *string `protobuf:"bytes,1,req,name=group" json:"group,omitempty"`
|
||||||
|
Key *string `protobuf:"bytes,2,req,name=key" json:"key,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetRequest) Reset() { *m = GetRequest{} }
|
||||||
|
func (m *GetRequest) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*GetRequest) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *GetRequest) GetGroup() string {
|
||||||
|
if m != nil && m.Group != nil {
|
||||||
|
return *m.Group
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetRequest) GetKey() string {
|
||||||
|
if m != nil && m.Key != nil {
|
||||||
|
return *m.Key
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type GetResponse struct {
|
||||||
|
Value []byte `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"`
|
||||||
|
MinuteQps *float64 `protobuf:"fixed64,2,opt,name=minute_qps" json:"minute_qps,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetResponse) Reset() { *m = GetResponse{} }
|
||||||
|
func (m *GetResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*GetResponse) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *GetResponse) GetValue() []byte {
|
||||||
|
if m != nil {
|
||||||
|
return m.Value
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *GetResponse) GetMinuteQps() float64 {
|
||||||
|
if m != nil && m.MinuteQps != nil {
|
||||||
|
return *m.MinuteQps
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
}
|
32
groupcachepb/groupcache.proto
Normal file
32
groupcachepb/groupcache.proto
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package groupcachepb;
|
||||||
|
|
||||||
|
message GetRequest {
|
||||||
|
required string group = 1;
|
||||||
|
required string key = 2; // not actually required/guaranteed to be UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetResponse {
|
||||||
|
optional bytes value = 1;
|
||||||
|
optional double minute_qps = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
service GroupCache {
|
||||||
|
rpc Get(GetRequest) returns (GetResponse) {
|
||||||
|
};
|
||||||
|
}
|
121
lru/lru.go
Normal file
121
lru/lru.go
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2013 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package lru implements an LRU cache.
|
||||||
|
package lru
|
||||||
|
|
||||||
|
import "container/list"
|
||||||
|
|
||||||
|
// Cache is an LRU cache. It is not safe for concurrent access.
|
||||||
|
type Cache struct {
|
||||||
|
// MaxEntries is the maximum number of cache entries before
|
||||||
|
// an item is evicted. Zero means no limit.
|
||||||
|
MaxEntries int
|
||||||
|
|
||||||
|
// OnEvicted optionally specificies a callback function to be
|
||||||
|
// executed when an entry is purged from the cache.
|
||||||
|
OnEvicted func(key Key, value interface{})
|
||||||
|
|
||||||
|
ll *list.List
|
||||||
|
cache map[interface{}]*list.Element
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
|
||||||
|
type Key interface{}
|
||||||
|
|
||||||
|
type entry struct {
|
||||||
|
key Key
|
||||||
|
value interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a new Cache.
|
||||||
|
// If maxEntries is zero, the cache has no limit and it's assumed
|
||||||
|
// that eviction is done by the caller.
|
||||||
|
func New(maxEntries int) *Cache {
|
||||||
|
return &Cache{
|
||||||
|
MaxEntries: maxEntries,
|
||||||
|
ll: list.New(),
|
||||||
|
cache: make(map[interface{}]*list.Element),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds a value to the cache.
|
||||||
|
func (c *Cache) Add(key Key, value interface{}) {
|
||||||
|
if c.cache == nil {
|
||||||
|
c.cache = make(map[interface{}]*list.Element)
|
||||||
|
c.ll = list.New()
|
||||||
|
}
|
||||||
|
if ee, ok := c.cache[key]; ok {
|
||||||
|
c.ll.MoveToFront(ee)
|
||||||
|
ee.Value.(*entry).value = value
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ele := c.ll.PushFront(&entry{key, value})
|
||||||
|
c.cache[key] = ele
|
||||||
|
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
|
||||||
|
c.RemoveOldest()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get looks up a key's value from the cache.
|
||||||
|
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
|
||||||
|
if c.cache == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ele, hit := c.cache[key]; hit {
|
||||||
|
c.ll.MoveToFront(ele)
|
||||||
|
return ele.Value.(*entry).value, true
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove removes the provided key from the cache.
|
||||||
|
func (c *Cache) Remove(key Key) {
|
||||||
|
if c.cache == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ele, hit := c.cache[key]; hit {
|
||||||
|
c.removeElement(ele)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveOldest removes the oldest item from the cache.
|
||||||
|
func (c *Cache) RemoveOldest() {
|
||||||
|
if c.cache == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ele := c.ll.Back()
|
||||||
|
if ele != nil {
|
||||||
|
c.removeElement(ele)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) removeElement(e *list.Element) {
|
||||||
|
c.ll.Remove(e)
|
||||||
|
kv := e.Value.(*entry)
|
||||||
|
delete(c.cache, kv.key)
|
||||||
|
if c.OnEvicted != nil {
|
||||||
|
c.OnEvicted(kv.key, kv.value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Len returns the number of items in the cache.
|
||||||
|
func (c *Cache) Len() int {
|
||||||
|
if c.cache == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return c.ll.Len()
|
||||||
|
}
|
73
lru/lru_test.go
Normal file
73
lru/lru_test.go
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2013 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package lru
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type simpleStruct struct {
|
||||||
|
int
|
||||||
|
string
|
||||||
|
}
|
||||||
|
|
||||||
|
type complexStruct struct {
|
||||||
|
int
|
||||||
|
simpleStruct
|
||||||
|
}
|
||||||
|
|
||||||
|
var getTests = []struct {
|
||||||
|
name string
|
||||||
|
keyToAdd interface{}
|
||||||
|
keyToGet interface{}
|
||||||
|
expectedOk bool
|
||||||
|
}{
|
||||||
|
{"string_hit", "myKey", "myKey", true},
|
||||||
|
{"string_miss", "myKey", "nonsense", false},
|
||||||
|
{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
|
||||||
|
{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
|
||||||
|
{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
|
||||||
|
complexStruct{1, simpleStruct{2, "three"}}, true},
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGet(t *testing.T) {
|
||||||
|
for _, tt := range getTests {
|
||||||
|
lru := New(0)
|
||||||
|
lru.Add(tt.keyToAdd, 1234)
|
||||||
|
val, ok := lru.Get(tt.keyToGet)
|
||||||
|
if ok != tt.expectedOk {
|
||||||
|
t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
|
||||||
|
} else if ok && val != 1234 {
|
||||||
|
t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemove(t *testing.T) {
|
||||||
|
lru := New(0)
|
||||||
|
lru.Add("myKey", 1234)
|
||||||
|
if val, ok := lru.Get("myKey"); !ok {
|
||||||
|
t.Fatal("TestRemove returned no match")
|
||||||
|
} else if val != 1234 {
|
||||||
|
t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val)
|
||||||
|
}
|
||||||
|
|
||||||
|
lru.Remove("myKey")
|
||||||
|
if _, ok := lru.Get("myKey"); ok {
|
||||||
|
t.Fatal("TestRemove returned a removed entry")
|
||||||
|
}
|
||||||
|
}
|
68
peers.go
Normal file
68
peers.go
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// peers.go defines how processes find and communicate with their peers.
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
pb "github.com/golang/groupcache/groupcachepb"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Context is an opaque value passed through calls to the
|
||||||
|
// ProtoGetter. It may be nil if your ProtoGetter implementation does
|
||||||
|
// not require a context.
|
||||||
|
type Context interface{}
|
||||||
|
|
||||||
|
// ProtoGetter is the interface that must be implemented by a peer.
|
||||||
|
type ProtoGetter interface {
|
||||||
|
Get(context Context, in *pb.GetRequest, out *pb.GetResponse) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// PeerPicker is the interface that must be implemented to locate
|
||||||
|
// the peer that owns a specific key.
|
||||||
|
type PeerPicker interface {
|
||||||
|
PickPeer(key string) (peer ProtoGetter, ok bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoPeers is an implementation of PeerPicker that never finds a peer.
|
||||||
|
type NoPeers struct{}
|
||||||
|
|
||||||
|
func (NoPeers) PickPeer(key string) (peer ProtoGetter, ok bool) { return }
|
||||||
|
|
||||||
|
var (
|
||||||
|
portPicker func() PeerPicker
|
||||||
|
)
|
||||||
|
|
||||||
|
// RegisterInitPeers registers the peer initialization function.
|
||||||
|
// It is called once, when the first group is created.
|
||||||
|
func RegisterPeerPicker(fn func() PeerPicker) {
|
||||||
|
if portPicker != nil {
|
||||||
|
panic("RegisterInitPeers called more than once")
|
||||||
|
}
|
||||||
|
portPicker = fn
|
||||||
|
}
|
||||||
|
|
||||||
|
func getPeers() PeerPicker {
|
||||||
|
if portPicker == nil {
|
||||||
|
return NoPeers{}
|
||||||
|
}
|
||||||
|
pk := portPicker()
|
||||||
|
if pk == nil {
|
||||||
|
pk = NoPeers{}
|
||||||
|
}
|
||||||
|
return pk
|
||||||
|
}
|
64
singleflight/singleflight.go
Normal file
64
singleflight/singleflight.go
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package singleflight provides a duplicate function call suppression
|
||||||
|
// mechanism.
|
||||||
|
package singleflight
|
||||||
|
|
||||||
|
import "sync"
|
||||||
|
|
||||||
|
// call is an in-flight or completed Do call
|
||||||
|
type call struct {
|
||||||
|
wg sync.WaitGroup
|
||||||
|
val interface{}
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group represents a class of work and forms a namespace in which
|
||||||
|
// units of work can be executed with duplicate suppression.
|
||||||
|
type Group struct {
|
||||||
|
mu sync.Mutex // protects m
|
||||||
|
m map[string]*call // lazily initialized
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do executes and returns the results of the given function, making
|
||||||
|
// sure that only one execution is in-flight for a given key at a
|
||||||
|
// time. If a duplicate comes in, the duplicate caller waits for the
|
||||||
|
// original to complete and receives the same results.
|
||||||
|
func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) {
|
||||||
|
g.mu.Lock()
|
||||||
|
if g.m == nil {
|
||||||
|
g.m = make(map[string]*call)
|
||||||
|
}
|
||||||
|
if c, ok := g.m[key]; ok {
|
||||||
|
g.mu.Unlock()
|
||||||
|
c.wg.Wait()
|
||||||
|
return c.val, c.err
|
||||||
|
}
|
||||||
|
c := new(call)
|
||||||
|
c.wg.Add(1)
|
||||||
|
g.m[key] = c
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
c.val, c.err = fn()
|
||||||
|
c.wg.Done()
|
||||||
|
|
||||||
|
g.mu.Lock()
|
||||||
|
delete(g.m, key)
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
return c.val, c.err
|
||||||
|
}
|
85
singleflight/singleflight_test.go
Normal file
85
singleflight/singleflight_test.go
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package singleflight
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDo(t *testing.T) {
|
||||||
|
var g Group
|
||||||
|
v, err := g.Do("key", func() (interface{}, error) {
|
||||||
|
return "bar", nil
|
||||||
|
})
|
||||||
|
if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
|
||||||
|
t.Errorf("Do = %v; want %v", got, want)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Do error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoErr(t *testing.T) {
|
||||||
|
var g Group
|
||||||
|
someErr := errors.New("Some error")
|
||||||
|
v, err := g.Do("key", func() (interface{}, error) {
|
||||||
|
return nil, someErr
|
||||||
|
})
|
||||||
|
if err != someErr {
|
||||||
|
t.Errorf("Do error = %v; want someErr", err, someErr)
|
||||||
|
}
|
||||||
|
if v != nil {
|
||||||
|
t.Errorf("unexpected non-nil value %#v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoDupSuppress(t *testing.T) {
|
||||||
|
var g Group
|
||||||
|
c := make(chan string)
|
||||||
|
var calls int32
|
||||||
|
fn := func() (interface{}, error) {
|
||||||
|
atomic.AddInt32(&calls, 1)
|
||||||
|
return <-c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const n = 10
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
v, err := g.Do("key", fn)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Do error: %v", err)
|
||||||
|
}
|
||||||
|
if v.(string) != "bar" {
|
||||||
|
t.Errorf("got %q; want %q", v, "bar")
|
||||||
|
}
|
||||||
|
wg.Done()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond) // let goroutines above block
|
||||||
|
c <- "bar"
|
||||||
|
wg.Wait()
|
||||||
|
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||||
|
t.Errorf("number of calls = %d; want 1", got)
|
||||||
|
}
|
||||||
|
}
|
322
sinks.go
Normal file
322
sinks.go
Normal file
@ -0,0 +1,322 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package groupcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"code.google.com/p/goprotobuf/proto"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Sink receives data from a Get call.
|
||||||
|
//
|
||||||
|
// Implementation of Getter must call exactly one of the Set methods
|
||||||
|
// on success.
|
||||||
|
type Sink interface {
|
||||||
|
// SetString sets the value to s.
|
||||||
|
SetString(s string) error
|
||||||
|
|
||||||
|
// SetBytes sets the value to the contents of v.
|
||||||
|
// The caller retains ownership of v.
|
||||||
|
SetBytes(v []byte) error
|
||||||
|
|
||||||
|
// SetProto sets the value to the encoded version of m.
|
||||||
|
// The caller retains ownership of m.
|
||||||
|
SetProto(m proto.Message) error
|
||||||
|
|
||||||
|
// view returns a frozen view of the bytes for caching.
|
||||||
|
view() (ByteView, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneBytes(b []byte) []byte {
|
||||||
|
c := make([]byte, len(b))
|
||||||
|
copy(c, b)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func setSinkView(s Sink, v ByteView) error {
|
||||||
|
// A viewSetter is a Sink that can also receive its value from
|
||||||
|
// a ByteView. This is a fast path to minimize copies when the
|
||||||
|
// item was already cached locally in memory (where it's
|
||||||
|
// cached as a ByteView)
|
||||||
|
type viewSetter interface {
|
||||||
|
setView(v ByteView) error
|
||||||
|
}
|
||||||
|
if vs, ok := s.(viewSetter); ok {
|
||||||
|
return vs.setView(v)
|
||||||
|
}
|
||||||
|
if v.b != nil {
|
||||||
|
return s.SetBytes(v.b)
|
||||||
|
}
|
||||||
|
return s.SetString(v.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// StringSink returns a Sink that populates the provided string pointer.
|
||||||
|
func StringSink(sp *string) Sink {
|
||||||
|
return &stringSink{sp: sp}
|
||||||
|
}
|
||||||
|
|
||||||
|
type stringSink struct {
|
||||||
|
sp *string
|
||||||
|
v ByteView
|
||||||
|
// TODO(bradfitz): track whether any Sets were called.
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stringSink) view() (ByteView, error) {
|
||||||
|
// TODO(bradfitz): return an error if no Set was called
|
||||||
|
return s.v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stringSink) SetString(v string) error {
|
||||||
|
s.v.b = nil
|
||||||
|
s.v.s = v
|
||||||
|
*s.sp = v
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stringSink) SetBytes(v []byte) error {
|
||||||
|
return s.SetString(string(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stringSink) SetProto(m proto.Message) error {
|
||||||
|
b, err := proto.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.v.b = b
|
||||||
|
*s.sp = string(b)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ByteViewSink returns a Sink that populates a ByteView.
|
||||||
|
func ByteViewSink(dst *ByteView) Sink {
|
||||||
|
if dst == nil {
|
||||||
|
panic("nil dst")
|
||||||
|
}
|
||||||
|
return &byteViewSink{dst: dst}
|
||||||
|
}
|
||||||
|
|
||||||
|
type byteViewSink struct {
|
||||||
|
dst *ByteView
|
||||||
|
|
||||||
|
// if this code ever ends up tracking that at least one set*
|
||||||
|
// method was called, don't make it an error to call set
|
||||||
|
// methods multiple times. Lorry's payload.go does that, and
|
||||||
|
// it makes sense. The comment at the top of this file about
|
||||||
|
// "exactly one of the Set methods" is overly strict. We
|
||||||
|
// really care about at least once (in a handler), but if
|
||||||
|
// multiple handlers fail (or multiple functions in a program
|
||||||
|
// using a Sink), it's okay to re-use the same one.
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *byteViewSink) setView(v ByteView) error {
|
||||||
|
*s.dst = v
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *byteViewSink) view() (ByteView, error) {
|
||||||
|
return *s.dst, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *byteViewSink) SetProto(m proto.Message) error {
|
||||||
|
b, err := proto.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*s.dst = ByteView{b: b}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *byteViewSink) SetBytes(b []byte) error {
|
||||||
|
*s.dst = ByteView{b: cloneBytes(b)}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *byteViewSink) SetString(v string) error {
|
||||||
|
*s.dst = ByteView{s: v}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProtoSink returns a sink that unmarshals binary proto values into m.
|
||||||
|
func ProtoSink(m proto.Message) Sink {
|
||||||
|
return &protoSink{
|
||||||
|
dst: m,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type protoSink struct {
|
||||||
|
dst proto.Message // authorative value
|
||||||
|
typ string
|
||||||
|
|
||||||
|
v ByteView // encoded
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *protoSink) view() (ByteView, error) {
|
||||||
|
return s.v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *protoSink) SetBytes(b []byte) error {
|
||||||
|
err := proto.Unmarshal(b, s.dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.v.b = cloneBytes(b)
|
||||||
|
s.v.s = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *protoSink) SetString(v string) error {
|
||||||
|
b := []byte(v)
|
||||||
|
err := proto.Unmarshal(b, s.dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.v.b = b
|
||||||
|
s.v.s = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *protoSink) SetProto(m proto.Message) error {
|
||||||
|
b, err := proto.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// TODO(bradfitz): optimize for same-task case more and write
|
||||||
|
// right through? would need to document ownership rules at
|
||||||
|
// the same time. but then we could just assign *dst = *m
|
||||||
|
// here. This works for now:
|
||||||
|
err = proto.Unmarshal(b, s.dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.v.b = b
|
||||||
|
s.v.s = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllocatingByteSliceSink returns a Sink that allocates
|
||||||
|
// a byte slice to hold the received value and assigns
|
||||||
|
// it to *dst. The memory is not retained by groupcache.
|
||||||
|
func AllocatingByteSliceSink(dst *[]byte) Sink {
|
||||||
|
return &allocBytesSink{dst: dst}
|
||||||
|
}
|
||||||
|
|
||||||
|
type allocBytesSink struct {
|
||||||
|
dst *[]byte
|
||||||
|
v ByteView
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) view() (ByteView, error) {
|
||||||
|
return s.v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) setView(v ByteView) error {
|
||||||
|
if v.b != nil {
|
||||||
|
*s.dst = cloneBytes(v.b)
|
||||||
|
} else {
|
||||||
|
*s.dst = []byte(v.s)
|
||||||
|
}
|
||||||
|
s.v = v
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) SetProto(m proto.Message) error {
|
||||||
|
b, err := proto.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s.setBytesOwned(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) SetBytes(b []byte) error {
|
||||||
|
return s.setBytesOwned(cloneBytes(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) setBytesOwned(b []byte) error {
|
||||||
|
if s.dst == nil {
|
||||||
|
return errors.New("nil AllocatingByteSliceSink *[]byte dst")
|
||||||
|
}
|
||||||
|
*s.dst = cloneBytes(b) // another copy, protecting the read-only s.v.b view
|
||||||
|
s.v.b = b
|
||||||
|
s.v.s = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *allocBytesSink) SetString(v string) error {
|
||||||
|
if s.dst == nil {
|
||||||
|
return errors.New("nil AllocatingByteSliceSink *[]byte dst")
|
||||||
|
}
|
||||||
|
*s.dst = []byte(v)
|
||||||
|
s.v.b = nil
|
||||||
|
s.v.s = v
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TruncatingByteSliceSink returns a Sink that writes up to len(*dst)
|
||||||
|
// bytes to *dst. If more bytes are available, they're silently
|
||||||
|
// truncated. If fewer bytes are available than len(*dst), *dst
|
||||||
|
// is shrunk to fit the number of bytes available.
|
||||||
|
func TruncatingByteSliceSink(dst *[]byte) Sink {
|
||||||
|
return &truncBytesSink{dst: dst}
|
||||||
|
}
|
||||||
|
|
||||||
|
type truncBytesSink struct {
|
||||||
|
dst *[]byte
|
||||||
|
v ByteView
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *truncBytesSink) view() (ByteView, error) {
|
||||||
|
return s.v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *truncBytesSink) SetProto(m proto.Message) error {
|
||||||
|
b, err := proto.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s.setBytesOwned(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *truncBytesSink) SetBytes(b []byte) error {
|
||||||
|
return s.setBytesOwned(cloneBytes(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *truncBytesSink) setBytesOwned(b []byte) error {
|
||||||
|
if s.dst == nil {
|
||||||
|
return errors.New("nil TruncatingByteSliceSink *[]byte dst")
|
||||||
|
}
|
||||||
|
n := copy(*s.dst, b)
|
||||||
|
if n < len(*s.dst) {
|
||||||
|
*s.dst = (*s.dst)[:n]
|
||||||
|
}
|
||||||
|
s.v.b = b
|
||||||
|
s.v.s = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *truncBytesSink) SetString(v string) error {
|
||||||
|
if s.dst == nil {
|
||||||
|
return errors.New("nil TruncatingByteSliceSink *[]byte dst")
|
||||||
|
}
|
||||||
|
n := copy(*s.dst, v)
|
||||||
|
if n < len(*s.dst) {
|
||||||
|
*s.dst = (*s.dst)[:n]
|
||||||
|
}
|
||||||
|
s.v.b = nil
|
||||||
|
s.v.s = v
|
||||||
|
return nil
|
||||||
|
}
|
235
testpb/test.pb.go
Normal file
235
testpb/test.pb.go
Normal file
@ -0,0 +1,235 @@
|
|||||||
|
// Code generated by protoc-gen-go.
|
||||||
|
// source: test.proto
|
||||||
|
// DO NOT EDIT!
|
||||||
|
|
||||||
|
package testpb
|
||||||
|
|
||||||
|
import proto "code.google.com/p/goprotobuf/proto"
|
||||||
|
import json "encoding/json"
|
||||||
|
import math "math"
|
||||||
|
|
||||||
|
// Reference proto, json, and math imports to suppress error if they are not otherwise used.
|
||||||
|
var _ = proto.Marshal
|
||||||
|
var _ = &json.SyntaxError{}
|
||||||
|
var _ = math.Inf
|
||||||
|
|
||||||
|
type TestMessage struct {
|
||||||
|
Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
|
||||||
|
City *string `protobuf:"bytes,2,opt,name=city" json:"city,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TestMessage) Reset() { *m = TestMessage{} }
|
||||||
|
func (m *TestMessage) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*TestMessage) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *TestMessage) GetName() string {
|
||||||
|
if m != nil && m.Name != nil {
|
||||||
|
return *m.Name
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TestMessage) GetCity() string {
|
||||||
|
if m != nil && m.City != nil {
|
||||||
|
return *m.City
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type TestRequest struct {
|
||||||
|
Lower *string `protobuf:"bytes,1,req,name=lower" json:"lower,omitempty"`
|
||||||
|
RepeatCount *int32 `protobuf:"varint,2,opt,name=repeat_count,def=1" json:"repeat_count,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TestRequest) Reset() { *m = TestRequest{} }
|
||||||
|
func (m *TestRequest) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*TestRequest) ProtoMessage() {}
|
||||||
|
|
||||||
|
const Default_TestRequest_RepeatCount int32 = 1
|
||||||
|
|
||||||
|
func (m *TestRequest) GetLower() string {
|
||||||
|
if m != nil && m.Lower != nil {
|
||||||
|
return *m.Lower
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TestRequest) GetRepeatCount() int32 {
|
||||||
|
if m != nil && m.RepeatCount != nil {
|
||||||
|
return *m.RepeatCount
|
||||||
|
}
|
||||||
|
return Default_TestRequest_RepeatCount
|
||||||
|
}
|
||||||
|
|
||||||
|
type TestResponse struct {
|
||||||
|
Value *string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TestResponse) Reset() { *m = TestResponse{} }
|
||||||
|
func (m *TestResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*TestResponse) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *TestResponse) GetValue() string {
|
||||||
|
if m != nil && m.Value != nil {
|
||||||
|
return *m.Value
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type CacheStats struct {
|
||||||
|
Items *int64 `protobuf:"varint,1,opt,name=items" json:"items,omitempty"`
|
||||||
|
Bytes *int64 `protobuf:"varint,2,opt,name=bytes" json:"bytes,omitempty"`
|
||||||
|
Gets *int64 `protobuf:"varint,3,opt,name=gets" json:"gets,omitempty"`
|
||||||
|
Hits *int64 `protobuf:"varint,4,opt,name=hits" json:"hits,omitempty"`
|
||||||
|
Evicts *int64 `protobuf:"varint,5,opt,name=evicts" json:"evicts,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *CacheStats) Reset() { *m = CacheStats{} }
|
||||||
|
func (m *CacheStats) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*CacheStats) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *CacheStats) GetItems() int64 {
|
||||||
|
if m != nil && m.Items != nil {
|
||||||
|
return *m.Items
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *CacheStats) GetBytes() int64 {
|
||||||
|
if m != nil && m.Bytes != nil {
|
||||||
|
return *m.Bytes
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *CacheStats) GetGets() int64 {
|
||||||
|
if m != nil && m.Gets != nil {
|
||||||
|
return *m.Gets
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *CacheStats) GetHits() int64 {
|
||||||
|
if m != nil && m.Hits != nil {
|
||||||
|
return *m.Hits
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *CacheStats) GetEvicts() int64 {
|
||||||
|
if m != nil && m.Evicts != nil {
|
||||||
|
return *m.Evicts
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsResponse struct {
|
||||||
|
Gets *int64 `protobuf:"varint,1,opt,name=gets" json:"gets,omitempty"`
|
||||||
|
CacheHits *int64 `protobuf:"varint,12,opt,name=cache_hits" json:"cache_hits,omitempty"`
|
||||||
|
Fills *int64 `protobuf:"varint,2,opt,name=fills" json:"fills,omitempty"`
|
||||||
|
TotalAlloc *uint64 `protobuf:"varint,3,opt,name=total_alloc" json:"total_alloc,omitempty"`
|
||||||
|
MainCache *CacheStats `protobuf:"bytes,4,opt,name=main_cache" json:"main_cache,omitempty"`
|
||||||
|
HotCache *CacheStats `protobuf:"bytes,5,opt,name=hot_cache" json:"hot_cache,omitempty"`
|
||||||
|
ServerIn *int64 `protobuf:"varint,6,opt,name=server_in" json:"server_in,omitempty"`
|
||||||
|
Loads *int64 `protobuf:"varint,8,opt,name=loads" json:"loads,omitempty"`
|
||||||
|
PeerLoads *int64 `protobuf:"varint,9,opt,name=peer_loads" json:"peer_loads,omitempty"`
|
||||||
|
PeerErrors *int64 `protobuf:"varint,10,opt,name=peer_errors" json:"peer_errors,omitempty"`
|
||||||
|
LocalLoads *int64 `protobuf:"varint,11,opt,name=local_loads" json:"local_loads,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) Reset() { *m = StatsResponse{} }
|
||||||
|
func (m *StatsResponse) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*StatsResponse) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetGets() int64 {
|
||||||
|
if m != nil && m.Gets != nil {
|
||||||
|
return *m.Gets
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetCacheHits() int64 {
|
||||||
|
if m != nil && m.CacheHits != nil {
|
||||||
|
return *m.CacheHits
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetFills() int64 {
|
||||||
|
if m != nil && m.Fills != nil {
|
||||||
|
return *m.Fills
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetTotalAlloc() uint64 {
|
||||||
|
if m != nil && m.TotalAlloc != nil {
|
||||||
|
return *m.TotalAlloc
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetMainCache() *CacheStats {
|
||||||
|
if m != nil {
|
||||||
|
return m.MainCache
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetHotCache() *CacheStats {
|
||||||
|
if m != nil {
|
||||||
|
return m.HotCache
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetServerIn() int64 {
|
||||||
|
if m != nil && m.ServerIn != nil {
|
||||||
|
return *m.ServerIn
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetLoads() int64 {
|
||||||
|
if m != nil && m.Loads != nil {
|
||||||
|
return *m.Loads
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetPeerLoads() int64 {
|
||||||
|
if m != nil && m.PeerLoads != nil {
|
||||||
|
return *m.PeerLoads
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetPeerErrors() int64 {
|
||||||
|
if m != nil && m.PeerErrors != nil {
|
||||||
|
return *m.PeerErrors
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StatsResponse) GetLocalLoads() int64 {
|
||||||
|
if m != nil && m.LocalLoads != nil {
|
||||||
|
return *m.LocalLoads
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
type Empty struct {
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Empty) Reset() { *m = Empty{} }
|
||||||
|
func (m *Empty) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*Empty) ProtoMessage() {}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
}
|
61
testpb/test.proto
Normal file
61
testpb/test.proto
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2012 Google Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package testpb;
|
||||||
|
|
||||||
|
message TestMessage {
|
||||||
|
optional string name = 1;
|
||||||
|
optional string city = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message TestRequest {
|
||||||
|
required string lower = 1; // to be returned upper case
|
||||||
|
optional int32 repeat_count = 2 [default = 1]; // .. this many times
|
||||||
|
}
|
||||||
|
|
||||||
|
message TestResponse {
|
||||||
|
optional string value = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message CacheStats {
|
||||||
|
optional int64 items = 1;
|
||||||
|
optional int64 bytes = 2;
|
||||||
|
optional int64 gets = 3;
|
||||||
|
optional int64 hits = 4;
|
||||||
|
optional int64 evicts = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message StatsResponse {
|
||||||
|
optional int64 gets = 1;
|
||||||
|
optional int64 cache_hits = 12;
|
||||||
|
optional int64 fills = 2;
|
||||||
|
optional uint64 total_alloc = 3;
|
||||||
|
optional CacheStats main_cache = 4;
|
||||||
|
optional CacheStats hot_cache = 5;
|
||||||
|
optional int64 server_in = 6;
|
||||||
|
optional int64 loads = 8;
|
||||||
|
optional int64 peer_loads = 9;
|
||||||
|
optional int64 peer_errors = 10;
|
||||||
|
optional int64 local_loads = 11;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Empty {}
|
||||||
|
|
||||||
|
service GroupCacheTest {
|
||||||
|
rpc InitPeers(Empty) returns (Empty) {};
|
||||||
|
rpc Get(TestRequest) returns (TestResponse) {};
|
||||||
|
rpc GetStats(Empty) returns (StatsResponse) {};
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user