From fb5bf615e7e928aa225298134b7c5c49b7244295 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 03:52:44 +0000 Subject: [PATCH] Bump go.etcd.io/bbolt from 1.4.3 to 1.5.0 Bumps [go.etcd.io/bbolt](https://github.com/etcd-io/bbolt) from 1.4.3 to 1.5.0. - [Release notes](https://github.com/etcd-io/bbolt/releases) - [Commits](https://github.com/etcd-io/bbolt/compare/v1.4.3...v1.5.0) --- updated-dependencies: - dependency-name: go.etcd.io/bbolt dependency-version: 1.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- vendor/go.etcd.io/bbolt/.gitattributes | 4 + vendor/go.etcd.io/bbolt/.go-version | 2 +- vendor/go.etcd.io/bbolt/.golangci.yaml | 40 +++++ vendor/go.etcd.io/bbolt/Makefile | 26 ++- vendor/go.etcd.io/bbolt/OWNERS | 1 + vendor/go.etcd.io/bbolt/README.md | 18 +- vendor/go.etcd.io/bbolt/bucket.go | 24 ++- vendor/go.etcd.io/bbolt/code-of-conduct.md | 3 + vendor/go.etcd.io/bbolt/db.go | 159 ++++++++++++------ vendor/go.etcd.io/bbolt/errors/errors.go | 3 + .../go.etcd.io/bbolt/internal/common/page.go | 38 +++-- .../bbolt/internal/freelist/hashmap.go | 46 +++-- .../bbolt/internal/freelist/shared.go | 6 +- vendor/go.etcd.io/bbolt/tx.go | 18 +- vendor/go.etcd.io/bbolt/tx_check.go | 9 +- vendor/modules.txt | 4 +- 18 files changed, 281 insertions(+), 126 deletions(-) create mode 100644 vendor/go.etcd.io/bbolt/.gitattributes create mode 100644 vendor/go.etcd.io/bbolt/.golangci.yaml create mode 100644 vendor/go.etcd.io/bbolt/code-of-conduct.md diff --git a/go.mod b/go.mod index 4907e0d3f5..6bf5233291 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 - go.etcd.io/bbolt v1.4.3 + go.etcd.io/bbolt v1.5.0 go.opencensus.io v0.24.0 go.uber.org/mock v0.6.0 golang.org/x/net v0.56.0 diff --git a/go.sum b/go.sum index f1392d64e0..9cbcec8d4e 100644 --- a/go.sum +++ b/go.sum @@ -979,8 +979,8 @@ github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg= -go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= -go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/vendor/go.etcd.io/bbolt/.gitattributes b/vendor/go.etcd.io/bbolt/.gitattributes new file mode 100644 index 0000000000..a681ce365f --- /dev/null +++ b/vendor/go.etcd.io/bbolt/.gitattributes @@ -0,0 +1,4 @@ +# ensure that line endings for Windows builds are properly formatted +# see https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#how-to-use +# at "Multiple OS Example" section +*.go text eol=lf diff --git a/vendor/go.etcd.io/bbolt/.go-version b/vendor/go.etcd.io/bbolt/.go-version index 7bdcec52d0..4fd1625308 100644 --- a/vendor/go.etcd.io/bbolt/.go-version +++ b/vendor/go.etcd.io/bbolt/.go-version @@ -1 +1 @@ -1.23.12 +1.25.11 diff --git a/vendor/go.etcd.io/bbolt/.golangci.yaml b/vendor/go.etcd.io/bbolt/.golangci.yaml new file mode 100644 index 0000000000..bef3f6de7a --- /dev/null +++ b/vendor/go.etcd.io/bbolt/.golangci.yaml @@ -0,0 +1,40 @@ +formatters: + enable: + - gci + - gofmt + - goimports + settings: # please keep this alphabetized + gci: + sections: + - standard + - default + - prefix(go.etcd.io) + goimports: + local-prefixes: + - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. +issues: + max-same-issues: 0 +linters: + default: none + enable: # please keep this alphabetized + - errcheck + - govet + - ineffassign + - staticcheck + - unused + exclusions: + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + settings: # please keep this alphabetized + staticcheck: + checks: + - all + - -QF1003 # Convert if/else-if chain to tagged switch + - -QF1010 # Convert slice of bytes to string when printing it + - -ST1003 # Poorly chosen identifier + - -ST1005 # Incorrectly formatted error string + - -ST1012 # Poorly chosen name for error variable +version: "2" diff --git a/vendor/go.etcd.io/bbolt/Makefile b/vendor/go.etcd.io/bbolt/Makefile index f5a6703a0b..f68708cf5b 100644 --- a/vendor/go.etcd.io/bbolt/Makefile +++ b/vendor/go.etcd.io/bbolt/Makefile @@ -2,6 +2,8 @@ BRANCH=`git rev-parse --abbrev-ref HEAD` COMMIT=`git rev-parse --short HEAD` GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" GOFILES = $(shell find . -name \*.go) +REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) +GOTOOLCHAIN ?= go$(shell cat $(REPOSITORY_ROOT)/.go-version) TESTFLAGS_RACE=-race=false ifdef ENABLE_RACE @@ -33,7 +35,7 @@ fmt: @!(gofmt -l -s -d ${GOFILES} | grep '[a-z]') @echo "Verifying goimports, failures can be fixed with ./scripts/fix.sh" - @!(go run golang.org/x/tools/cmd/goimports@latest -l -d ${GOFILES} | grep '[a-z]') + @!(go tool golang.org/x/tools/cmd/goimports -l -d ${GOFILES} | grep '[a-z]') .PHONY: lint lint: @@ -44,12 +46,12 @@ test: @echo "hashmap freelist test" BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/... - BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt + BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt/... @echo "array freelist test" BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/... - BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt + BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt/... .PHONY: coverage coverage: @@ -71,16 +73,12 @@ clean: # Clean binaries rm -f ./bin/${BOLT_CMD} .PHONY: gofail-enable -gofail-enable: install-gofail - gofail enable . +gofail-enable: + go tool go.etcd.io/gofail enable . .PHONY: gofail-disable -gofail-disable: install-gofail - gofail disable . - -.PHONY: install-gofail -install-gofail: - go install go.etcd.io/gofail +gofail-disable: + go tool go.etcd.io/gofail disable . .PHONY: test-failpoint test-failpoint: @@ -99,10 +97,6 @@ test-robustness: gofail-enable build .PHONY: test-benchmark-compare # Runs benchmark tests on the current git ref and the given REF, and compares # the two. -test-benchmark-compare: install-benchstat +test-benchmark-compare: @git fetch ./scripts/compare_benchmarks.sh $(REF) - -.PHONY: install-benchstat -install-benchstat: - go install golang.org/x/perf/cmd/benchstat@latest diff --git a/vendor/go.etcd.io/bbolt/OWNERS b/vendor/go.etcd.io/bbolt/OWNERS index 91f168a798..f8ab19db29 100644 --- a/vendor/go.etcd.io/bbolt/OWNERS +++ b/vendor/go.etcd.io/bbolt/OWNERS @@ -6,5 +6,6 @@ approvers: - ptabor # Piotr Tabor - spzala # Sahdev Zala reviewers: + - elbehery # Mustafa Elbehery - fuweid # Wei Fu - tjungblu # Thomas Jungblut diff --git a/vendor/go.etcd.io/bbolt/README.md b/vendor/go.etcd.io/bbolt/README.md index f365e51e3e..7f6468e73b 100644 --- a/vendor/go.etcd.io/bbolt/README.md +++ b/vendor/go.etcd.io/bbolt/README.md @@ -890,13 +890,19 @@ Here are a few things to note when evaluating and using Bolt: to grow. However, it's important to note that deleting large chunks of data will not allow you to reclaim that space on disk. + For more information on page allocation, [see this comment][page-allocation]. + * Removing key/values pairs in a bucket during iteration on the bucket using cursor may not work properly. Each time when removing a key/value pair, the cursor may automatically move to the next position if present. When users call `c.Next()` after removing a key, it may skip one key/value pair. Refer to https://github.com/etcd-io/bbolt/pull/611 for more detailed info. - For more information on page allocation, [see this comment][page-allocation]. +* Bolt db can be corrupted during the initialization phase due to abrupt power failure. + - Please note: This issue can only be reproduced during the very first initialization phase, when there is + no existing data in bolt database. + - In normal production environment, it is difficult to reproduce this. Once the database file has been initialized, it can no longer occur. + - Please refer to this issue for more details: https://github.com/etcd-io/etcd/issues/16596. [page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638 @@ -954,11 +960,16 @@ them via pull request. - bbolt might run into data corruption issue on Linux when the feature [ext4: fast commit](https://lwn.net/Articles/842385/), which was introduced in - linux kernel version v5.10, is enabled. The fixes to the issue were included in - linux kernel version v5.17, please refer to links below, + linux kernel version v5.10, is enabled. The fixes to the issue are included in + stable LTS patchlevels 5.10.94+ and 5.15.17+ (ftruncate tracking), plus + 5.15.27+ (ineligible-commit fallback). Linux 5.17 includes these fixes as + well, but 5.17 is not an LTS release. Please refer to links below, * [ext4: fast commit may miss tracking unwritten range during ftruncate](https://lore.kernel.org/linux-ext4/20211223032337.5198-3-yinxin.x@bytedance.com/) + * [5.10.94 stable backport](https://lore.kernel.org/stable/20220124184041.063143682@linuxfoundation.org/) + * [5.15.17 stable backport](https://lore.kernel.org/stable/20220124184125.887304707@linuxfoundation.org/) * [ext4: fast commit may not fallback for ineligible commit](https://lore.kernel.org/lkml/202201091544.W5HHEXAp-lkp@intel.com/T/#ma0768815e4b5f671e9e451d578256ef9a76fe30e) + * [5.15.27 stable backport](https://lore.kernel.org/stable/20220307091703.544901888@linuxfoundation.org/) * [ext4 updates for 5.17](https://lore.kernel.org/lkml/YdyxjTFaLWif6BCM@mit.edu/) Please also refer to the discussion in https://github.com/etcd-io/bbolt/issues/562. @@ -1000,6 +1011,7 @@ Below is a list of public, open source projects that use Bolt: * [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. * [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains * [gokv](https://github.com/philippgille/gokv) - Simple key-value store abstraction and implementations for Go (Redis, Consul, etcd, bbolt, BadgerDB, LevelDB, Memcached, DynamoDB, S3, PostgreSQL, MongoDB, CockroachDB and many more) +* [goraphdb](https://github.com/mstrYoda/goraphdb) - A graph database provides Cypher query, fluent builder and management UI. * [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. diff --git a/vendor/go.etcd.io/bbolt/bucket.go b/vendor/go.etcd.io/bbolt/bucket.go index 6371ace972..fec2d6c957 100644 --- a/vendor/go.etcd.io/bbolt/bucket.go +++ b/vendor/go.etcd.io/bbolt/bucket.go @@ -661,16 +661,20 @@ func (b *Bucket) Stats() BucketStats { } } else if p.IsBranchPage() { s.BranchPageN++ - lastElement := p.BranchPageElement(p.Count() - 1) - // used totals the used bytes for the page - // Add header and all element headers. - used := common.PageHeaderSize + (common.BranchPageElementSize * uintptr(p.Count()-1)) + used := common.PageHeaderSize + if p.Count() != 0 { + lastElement := p.BranchPageElement(p.Count() - 1) + + // Add all element headers. + used += common.BranchPageElementSize * uintptr(p.Count()-1) + + // Add size of all keys and values. + // Again, use the fact that last element's position equals to + // the total of key, value sizes of all previous elements. + used += uintptr(lastElement.Pos() + lastElement.Ksize()) + } - // Add size of all keys and values. - // Again, use the fact that last element's position equals to - // the total of key, value sizes of all previous elements. - used += uintptr(lastElement.Pos() + lastElement.Ksize()) s.BranchInuse += int(used) s.BranchOverflowN += int(p.Overflow()) } @@ -880,7 +884,9 @@ func (b *Bucket) node(pgId common.Pgid, parent *node) *node { // if p isn't nil, then it's an inline bucket. // The pgId must be 0 in this case. common.Verify(func() { - common.Assert(pgId == 0, "The page ID (%d) isn't 0 for an inline bucket", pgId) + if pgId != 0 { + panic(fmt.Sprintf("assertion failed: The page ID (%d) isn't 0 for an inline bucket", pgId)) + } }) } diff --git a/vendor/go.etcd.io/bbolt/code-of-conduct.md b/vendor/go.etcd.io/bbolt/code-of-conduct.md new file mode 100644 index 0000000000..f78dd84bc2 --- /dev/null +++ b/vendor/go.etcd.io/bbolt/code-of-conduct.md @@ -0,0 +1,3 @@ +# etcd Community Code of Conduct + +Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). diff --git a/vendor/go.etcd.io/bbolt/db.go b/vendor/go.etcd.io/bbolt/db.go index 622947d9cb..5babb6ab16 100644 --- a/vendor/go.etcd.io/bbolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -36,12 +36,6 @@ const ( // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. type DB struct { - // Put `stats` at the first field to ensure it's 64-bit aligned. Note that - // the first word in an allocated struct can be relied upon to be 64-bit - // aligned. Refer to https://pkg.go.dev/sync/atomic#pkg-note-BUG. Also - // refer to discussion in https://github.com/etcd-io/bbolt/issues/577. - stats Stats - // When enabled, the database will perform a Check() after every commit. // A panic is issued if the database is in an inconsistent state. This // flag has a large performance impact so it should only be used for @@ -110,6 +104,12 @@ type DB struct { // of truncate() and fsync() when growing the data file. AllocSize int + // MaxSize is the maximum size (in bytes) allowed for the data file. + // If a caller's attempt to add data results in the need to grow + // the data file, an error will be returned and the data file will not grow. + // <=0 means no limit. + MaxSize int + // Mlock locks database file in memory when set to true. // It prevents major page faults, however used memory can't be reclaimed. // @@ -132,7 +132,7 @@ type DB struct { pageSize int opened bool rwtx *Tx - txs []*Tx + stats *Stats freelist fl.Interface freelistLoad sync.Once @@ -191,12 +191,17 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) { db.PreLoadFreelist = options.PreLoadFreelist db.FreelistType = options.FreelistType db.Mlock = options.Mlock + db.MaxSize = options.MaxSize // Set default values for later DB operations. db.MaxBatchSize = common.DefaultMaxBatchSize db.MaxBatchDelay = common.DefaultMaxBatchDelay db.AllocSize = common.DefaultAllocSize + if !options.NoStatistics { + db.stats = new(Stats) + } + if options.Logger == nil { db.logger = getDiscardLogger() } else { @@ -424,7 +429,9 @@ func (db *DB) loadFreelist() { // Read free list from freelist page. db.freelist.Read(db.page(db.meta().Freelist())) } - db.stats.FreePageN = db.freelist.FreeCount() + if db.stats != nil { + db.stats.FreePageN = db.freelist.FreeCount() + } }) } @@ -469,6 +476,23 @@ func (db *DB) mmap(minsz int) (err error) { return err } + if db.MaxSize > 0 && size > db.MaxSize { + // On Windows, the data file is expanded to the full mapped size during mmap, + // so we must reject any mmap size larger than the configured max size. + // + // On other platforms, mmap itself does not grow the file immediately, so the + // mapped size may exceed the max size temporarily. The file size limit is + // enforced later when the file actually grows. + // + // In practice, this check mainly applies when opening a database with a large + // InitialMmapSize. In all other cases, file growth is already guarded during + // page allocation. + if size > fileSize && runtime.GOOS == "windows" { + db.Logger().Errorf("[GOOS: %s, GOARCH: %s] maximum db size reached, size: %d, db.MaxSize: %d", runtime.GOOS, runtime.GOARCH, size, db.MaxSize) + return berrors.ErrMaxSizeReached + } + } + if db.Mlock { // Unlock db memory if err := db.munlock(fileSize); err != nil { @@ -545,7 +569,7 @@ func (db *DB) munmap() error { // return errors.New(unmapError) if err := munmap(db); err != nil { db.Logger().Errorf("[GOOS: %s, GOARCH: %s] munmap failed, db.datasz: %d, error: %v", runtime.GOOS, runtime.GOARCH, db.datasz, err) - return fmt.Errorf("unmap error: %v", err.Error()) + return fmt.Errorf("unmap error: %w", err) } return nil @@ -593,7 +617,7 @@ func (db *DB) munlock(fileSize int) error { // return errors.New(munlockError) if err := munlock(db, fileSize); err != nil { db.Logger().Errorf("[GOOS: %s, GOARCH: %s] munlock failed, fileSize: %d, db.datasz: %d, error: %v", runtime.GOOS, runtime.GOARCH, fileSize, db.datasz, err) - return fmt.Errorf("munlock error: %v", err.Error()) + return fmt.Errorf("munlock error: %w", err) } return nil } @@ -603,7 +627,7 @@ func (db *DB) mlock(fileSize int) error { // return errors.New(mlockError) if err := mlock(db, fileSize); err != nil { db.Logger().Errorf("[GOOS: %s, GOARCH: %s] mlock failed, fileSize: %d, db.datasz: %d, error: %v", runtime.GOOS, runtime.GOARCH, fileSize, db.datasz, err) - return fmt.Errorf("mlock error: %v", err.Error()) + return fmt.Errorf("mlock error: %w", err) } return nil } @@ -794,9 +818,6 @@ func (db *DB) beginTx() (*Tx, error) { t := &Tx{} t.init(db) - // Keep track of transaction until it closes. - db.txs = append(db.txs, t) - n := len(db.txs) if db.freelist != nil { db.freelist.AddReadonlyTXID(t.meta.Txid()) } @@ -805,10 +826,12 @@ func (db *DB) beginTx() (*Tx, error) { db.metalock.Unlock() // Update the transaction stats. - db.statlock.Lock() - db.stats.TxN++ - db.stats.OpenTxN = n - db.statlock.Unlock() + if db.stats != nil { + db.statlock.Lock() + db.stats.TxN++ + db.stats.OpenTxN++ + db.statlock.Unlock() + } return t, nil } @@ -856,17 +879,6 @@ func (db *DB) removeTx(tx *Tx) { // Use the meta lock to restrict access to the DB object. db.metalock.Lock() - // Remove the transaction. - for i, t := range db.txs { - if t == tx { - last := len(db.txs) - 1 - db.txs[i] = db.txs[last] - db.txs[last] = nil - db.txs = db.txs[:last] - break - } - } - n := len(db.txs) if db.freelist != nil { db.freelist.RemoveReadonlyTXID(tx.meta.Txid()) } @@ -875,10 +887,12 @@ func (db *DB) removeTx(tx *Tx) { db.metalock.Unlock() // Merge statistics. - db.statlock.Lock() - db.stats.OpenTxN = n - db.stats.TxStats.add(&tx.stats) - db.statlock.Unlock() + if db.stats != nil { + db.statlock.Lock() + db.stats.OpenTxN-- + db.stats.TxStats.add(&tx.stats) + db.statlock.Unlock() + } } // Update executes a function within the context of a read-write managed transaction. @@ -1096,9 +1110,13 @@ func (db *DB) Sync() (err error) { // Stats retrieves ongoing performance stats for the database. // This is only updated when a transaction closes. func (db *DB) Stats() Stats { - db.statlock.RLock() - defer db.statlock.RUnlock() - return db.stats + var s Stats + if db.stats != nil { + db.statlock.RLock() + s = *db.stats + db.statlock.RUnlock() + } + return s } // This is for internal access to the raw data bytes from the C cursor, use @@ -1164,9 +1182,33 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) { // Resize mmap() if we're at the end. p.SetId(db.rwtx.meta.Pgid()) var minsz = int((p.Id()+common.Pgid(count))+1) * db.pageSize + if db.MaxSize > 0 { + nextAllocSize := minsz + nextMmapSize, err := db.mmapSize(minsz) + if err != nil { + return nil, fmt.Errorf("mmap size calculation error: %w", err) + } + if runtime.GOOS == "windows" { + // nextAllocSize may not exactly match nextMmapSize. + // On Windows, this mismatch may cause the file size to slightly exceed maxSize, + // while it is harmless on other platforms. + nextAllocSize = nextMmapSize + } else { + // On non-Windows platforms, the database file is only grown explicitly in grow calls. + nextAllocSize = db.growSize(nextMmapSize, nextAllocSize) + } + if nextAllocSize > db.MaxSize { + db.Logger().Errorf("[GOOS: %s, GOARCH: %s] maximum db size reached, minSize: %d (allocSize: %d), db.MaxSize: %d", runtime.GOOS, runtime.GOARCH, minsz, nextAllocSize, db.MaxSize) + return nil, berrors.ErrMaxSizeReached + } + } if minsz >= db.datasz { if err := db.mmap(minsz); err != nil { - return nil, fmt.Errorf("mmap allocate error: %s", err) + if err == berrors.ErrMaxSizeReached { + return nil, err + } else { + return nil, fmt.Errorf("mmap allocate error: %s", err) + } } } @@ -1190,13 +1232,7 @@ func (db *DB) grow(sz int) error { return nil } - // If the data is smaller than the alloc size then only allocate what's needed. - // Once it goes over the allocation size then allocate in chunks. - if db.datasz <= db.AllocSize { - sz = db.datasz - } else { - sz += db.AllocSize - } + sz = db.growSize(db.datasz, sz) // Truncate and fsync to ensure file size metadata is flushed. // https://github.com/boltdb/bolt/issues/284 @@ -1224,6 +1260,16 @@ func (db *DB) grow(sz int) error { return nil } +func (db *DB) growSize(mmapSize, growSize int) int { + // If the data is smaller than the alloc size then only allocate what's needed. + // Once it goes over the allocation size then allocate in chunks. + if mmapSize <= db.AllocSize { + return mmapSize + } else { + return growSize + db.AllocSize + } +} + func (db *DB) IsReadOnly() bool { return db.readOnly } @@ -1243,13 +1289,16 @@ func (db *DB) freepages() []common.Pgid { reachable := make(map[common.Pgid]*common.Page) nofreed := make(map[common.Pgid]bool) ech := make(chan error) + go func() { - for e := range ech { - panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e)) - } + defer close(ech) + tx.recursivelyCheckBucket(&tx.root, reachable, nofreed, HexKVStringer(), ech) }() - tx.recursivelyCheckBucket(&tx.root, reachable, nofreed, HexKVStringer(), ech) - close(ech) + // following for loop will exit once channel is closed in the above goroutine. + // we don't need to wait explictly with a waitgroup + for e := range ech { + panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e)) + } // TODO: If check bucket reported any corruptions (ech) we shouldn't proceed to freeing the pages. @@ -1320,6 +1369,9 @@ type Options struct { // PageSize overrides the default OS page size. PageSize int + // MaxSize sets the maximum size of the data file. A value <= 0 means no limit. + MaxSize int + // NoSync sets the initial value of DB.NoSync. Normally this can just be // set directly on the DB itself when returned from Open(), but this option // is useful in APIs which expose Options but not the underlying DB. @@ -1336,6 +1388,11 @@ type Options struct { // Logger is the logger used for bbolt. Logger Logger + + // NoStatistics turns off statistics collection, Stats method will + // return empty structure in this case. This can be beneficial for + // performance under high-concurrency read-only transactions. + NoStatistics bool } func (o *Options) String() string { @@ -1343,8 +1400,8 @@ func (o *Options) String() string { return "{}" } - return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p}", - o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger) + return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, MaxSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p, NoStatistics: %t}", + o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.MaxSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger, o.NoStatistics) } diff --git a/vendor/go.etcd.io/bbolt/errors/errors.go b/vendor/go.etcd.io/bbolt/errors/errors.go index c115289e56..dbebd6330e 100644 --- a/vendor/go.etcd.io/bbolt/errors/errors.go +++ b/vendor/go.etcd.io/bbolt/errors/errors.go @@ -69,6 +69,9 @@ var ( // ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize. ErrValueTooLarge = errors.New("value too large") + // ErrMaxSizeReached is returned when the configured maximum size of the data file is reached. + ErrMaxSizeReached = errors.New("database reached maximum size") + // ErrIncompatibleValue is returned when trying to create or delete a bucket // on an existing non-bucket key or when trying to create or delete a // non-bucket key on an existing bucket key. diff --git a/vendor/go.etcd.io/bbolt/internal/common/page.go b/vendor/go.etcd.io/bbolt/internal/common/page.go index ee808967c5..747588aee7 100644 --- a/vendor/go.etcd.io/bbolt/internal/common/page.go +++ b/vendor/go.etcd.io/bbolt/internal/common/page.go @@ -74,19 +74,27 @@ func (p *Page) IsFreelistPage() bool { return p.flags == FreelistPageFlag } +// IsValidPage checks Page flags correctness, only a single proper flag can be used. +func (p *Page) IsValidPage() bool { + return p.IsBranchPage() || + p.IsLeafPage() || + p.IsMetaPage() || + p.IsFreelistPage() +} + // Meta returns a pointer to the metadata section of the page. func (p *Page) Meta() *Meta { return (*Meta)(UnsafeAdd(unsafe.Pointer(p), unsafe.Sizeof(*p))) } func (p *Page) FastCheck(id Pgid) { - Assert(p.id == id, "Page expected to be: %v, but self identifies as %v", id, p.id) + if p.id != id { + panic(fmt.Sprintf("assertion failed: Page expected to be: %v, but self identifies as %v", id, p.id)) + } // Only one flag of page-type can be set. - Assert(p.IsBranchPage() || - p.IsLeafPage() || - p.IsMetaPage() || - p.IsFreelistPage(), - "page %v: has unexpected type/flags: %x", p.id, p.flags) + if !p.IsValidPage() { + panic(fmt.Sprintf("assertion failed: page %v: has unexpected type/flags: %x", p.id, p.flags)) + } } // LeafPageElement retrieves the leaf node by index @@ -122,7 +130,9 @@ func (p *Page) BranchPageElements() []branchPageElement { } func (p *Page) FreelistPageCount() (int, int) { - Assert(p.IsFreelistPage(), fmt.Sprintf("can't get freelist page count from a non-freelist page: %2x", p.flags)) + if !p.IsFreelistPage() { + panic(fmt.Sprintf("assertion failed: can't get freelist page count from a non-freelist page: %2x", p.flags)) + } // If the page.count is at the max uint16 value (64k) then it's considered // an overflow and the size of the freelist is stored as the first element. @@ -140,7 +150,9 @@ func (p *Page) FreelistPageCount() (int, int) { } func (p *Page) FreelistPageIds() []Pgid { - Assert(p.IsFreelistPage(), fmt.Sprintf("can't get freelist page IDs from a non-freelist page: %2x", p.flags)) + if !p.IsFreelistPage() { + panic(fmt.Sprintf("assertion failed: can't get freelist page IDs from a non-freelist page: %2x", p.flags)) + } idx, count := p.FreelistPageCount() @@ -335,16 +347,16 @@ func (s Pgids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s Pgids) Less(i, j int) bool { return s[i] < s[j] } // Merge returns the sorted union of a and b. -func (a Pgids) Merge(b Pgids) Pgids { +func (s Pgids) Merge(b Pgids) Pgids { // Return the opposite slice if one is nil. - if len(a) == 0 { + if len(s) == 0 { return b } if len(b) == 0 { - return a + return s } - merged := make(Pgids, len(a)+len(b)) - Mergepgids(merged, a, b) + merged := make(Pgids, len(s)+len(b)) + Mergepgids(merged, s, b) return merged } diff --git a/vendor/go.etcd.io/bbolt/internal/freelist/hashmap.go b/vendor/go.etcd.io/bbolt/internal/freelist/hashmap.go index 8d471f4b5b..720496be81 100644 --- a/vendor/go.etcd.io/bbolt/internal/freelist/hashmap.go +++ b/vendor/go.etcd.io/bbolt/internal/freelist/hashmap.go @@ -108,8 +108,10 @@ func (f *hashMap) Allocate(txid common.Txid, n int) common.Pgid { func (f *hashMap) FreeCount() int { common.Verify(func() { expectedFreePageCount := f.hashmapFreeCountSlow() - common.Assert(int(f.freePagesCount) == expectedFreePageCount, - "freePagesCount (%d) is out of sync with free pages map (%d)", f.freePagesCount, expectedFreePageCount) + if int(f.freePagesCount) != expectedFreePageCount { + panic(fmt.Sprintf("assertion failed: freePagesCount (%d) is out of sync with free pages map (%d)", + f.freePagesCount, expectedFreePageCount)) + } }) return int(f.freePagesCount) } @@ -169,6 +171,11 @@ func (f *hashMap) delSpan(start common.Pgid, size uint64) { } func (f *hashMap) mergeSpans(ids common.Pgids) { + if len(ids) == 0 { + return + } + sort.Sort(ids) + common.Verify(func() { ids1Freemap := f.idsFromFreemaps() ids2Forward := f.idsFromForwardMap() @@ -181,7 +188,6 @@ func (f *hashMap) mergeSpans(ids common.Pgids) { panic(fmt.Sprintf("Detected mismatch, f.freemaps: %v, f.backwardMap: %v", f.freemaps, f.backwardMap)) } - sort.Sort(ids) prev := common.Pgid(0) for _, id := range ids { // The ids shouldn't have duplicated free ID. @@ -196,26 +202,36 @@ func (f *hashMap) mergeSpans(ids common.Pgids) { } } }) - for _, id := range ids { - // try to see if we can merge and update - f.mergeWithExistingSpan(id) + + start := ids[0] + end := ids[0] + for i := 1; i < len(ids); i++ { + id := ids[i] + if id == end+1 { + end = id + continue + } + + f.mergeWithExistingSpan(start, end) + start, end = id, id } + f.mergeWithExistingSpan(start, end) } -// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward -func (f *hashMap) mergeWithExistingSpan(pid common.Pgid) { - prev := pid - 1 - next := pid + 1 +// mergeWithExistingSpan merges free span [start, end] with adjacent existing free spans (both backward and forward). +func (f *hashMap) mergeWithExistingSpan(start, end common.Pgid) { + prev := start - 1 + next := end + 1 preSize, mergeWithPrev := f.backwardMap[prev] nextSize, mergeWithNext := f.forwardMap[next] - newStart := pid - newSize := uint64(1) + newStart := start + newSize := uint64(end - start + 1) if mergeWithPrev { - //merge with previous span - start := prev + 1 - common.Pgid(preSize) - f.delSpan(start, preSize) + // merge with previous span + prevStart := prev + 1 - common.Pgid(preSize) + f.delSpan(prevStart, preSize) newStart -= common.Pgid(preSize) newSize += preSize diff --git a/vendor/go.etcd.io/bbolt/internal/freelist/shared.go b/vendor/go.etcd.io/bbolt/internal/freelist/shared.go index f2d1130083..f30a69f107 100644 --- a/vendor/go.etcd.io/bbolt/internal/freelist/shared.go +++ b/vendor/go.etcd.io/bbolt/internal/freelist/shared.go @@ -220,10 +220,10 @@ func (t *shared) Reload(p *common.Page) { func (t *shared) NoSyncReload(pgIds common.Pgids) { // Build a cache of only pending pages. - pcache := make(map[common.Pgid]bool) + pcache := make(map[common.Pgid]struct{}) for _, txp := range t.pending { for _, pendingID := range txp.ids { - pcache[pendingID] = true + pcache[pendingID] = struct{}{} } } @@ -231,7 +231,7 @@ func (t *shared) NoSyncReload(pgIds common.Pgids) { // with any pages not in the pending lists. a := []common.Pgid{} for _, id := range pgIds { - if !pcache[id] { + if _, ok := pcache[id]; !ok { a = append(a, id) } } diff --git a/vendor/go.etcd.io/bbolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go index 1669fb16a2..aa0066bd35 100644 --- a/vendor/go.etcd.io/bbolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -357,13 +357,15 @@ func (tx *Tx) close() { tx.db.rwlock.Unlock() // Merge statistics. - tx.db.statlock.Lock() - tx.db.stats.FreePageN = freelistFreeN - tx.db.stats.PendingPageN = freelistPendingN - tx.db.stats.FreeAlloc = (freelistFreeN + freelistPendingN) * tx.db.pageSize - tx.db.stats.FreelistInuse = freelistAlloc - tx.db.stats.TxStats.add(&tx.stats) - tx.db.statlock.Unlock() + if tx.db.stats != nil { + tx.db.statlock.Lock() + tx.db.stats.FreePageN = freelistFreeN + tx.db.stats.PendingPageN = freelistPendingN + tx.db.stats.FreeAlloc = (freelistFreeN + freelistPendingN) * tx.db.pageSize + tx.db.stats.FreelistInuse = freelistAlloc + tx.db.stats.TxStats.add(&tx.stats) + tx.db.statlock.Unlock() + } } else { tx.db.removeTx(tx) } @@ -396,7 +398,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // the transaction was based on. // // To overcome this, we reuse the already opened file handle when - // WritFlag not set. When the WriteFlag is set, we reopen the file + // WriteFlag not set. When the WriteFlag is set, we reopen the file // but verify that it still refers to the same underlying file // (by device and inode). If it does not, we fall back to // reusing the existing already opened file handle. diff --git a/vendor/go.etcd.io/bbolt/tx_check.go b/vendor/go.etcd.io/bbolt/tx_check.go index c3ecbb9750..a1e9b33cf0 100644 --- a/vendor/go.etcd.io/bbolt/tx_check.go +++ b/vendor/go.etcd.io/bbolt/tx_check.go @@ -36,6 +36,11 @@ func (tx *Tx) Check(options ...CheckOption) <-chan error { } func (tx *Tx) check(cfg checkConfig, ch chan error) { + defer func() { + if r := recover(); r != nil { + ch <- panicked{r} + } + }() // Force loading free list if opened in ReadOnly mode. tx.db.loadFreelist() @@ -281,10 +286,10 @@ func HexKVStringer() KVStringer { type hexKvStringer struct{} -func (_ hexKvStringer) KeyToString(key []byte) string { +func (hexKvStringer) KeyToString(key []byte) string { return hex.EncodeToString(key) } -func (_ hexKvStringer) ValueToString(value []byte) string { +func (hexKvStringer) ValueToString(value []byte) string { return hex.EncodeToString(value) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 9cea083fde..bab0c98e58 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -542,8 +542,8 @@ github.com/xrash/smetrics # github.com/yashtewari/glob-intersection v0.2.0 ## explicit; go 1.17 github.com/yashtewari/glob-intersection -# go.etcd.io/bbolt v1.4.3 -## explicit; go 1.23 +# go.etcd.io/bbolt v1.5.0 +## explicit; go 1.25.0 go.etcd.io/bbolt go.etcd.io/bbolt/errors go.etcd.io/bbolt/internal/common