mirror
/
gotosocial
mirror of https://github.com/superseriousbusiness/gotosocial.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

[bugfix] fix possible infinite loops in media / emoji cleanup (#2590) 

* update media / emoji cleaner funcs to use new paging package, check for same returned maxID

* fix other calls of getattachments and getmojis not using paging

* use alternative order-by function
pull/2591/head
kim 2 years ago committed by GitHub
parent
0f7a2024c3
commit
c9452f32f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 156 additions and 117 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 66
      cmd/gotosocial/action/admin/media/list.go
  2. 63
      internal/cleaner/emoji.go
  3. 37
      internal/cleaner/media.go
  4. 15
      internal/db/bundb/emoji.go
  5. 11
      internal/db/bundb/media.go
  6. 5
      internal/db/emoji.go
  7. 5
      internal/db/media.go
  8. 71
      internal/util/slices.go

66
cmd/gotosocial/action/admin/media/list.go
Unescape View File

@ -31,14 +31,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/state"
)
type list struct {
dbService db.DB
state *state.State
maxID string
limit int
page paging.Page
localOnly bool
remoteOnly bool
out *bufio.Writer
@ -47,31 +47,32 @@ type list struct {
// Get a list of attachment using a custom filter
func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
res := make([]string, 0, 100)
for {
attachments, err := l.dbService.GetAttachments(ctx, l.maxID, l.limit)
if err != nil {
// Get the next page of media attachments up to max ID.
attachments, err := l.dbService.GetAttachments(ctx, &l.page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
}
// Get current max ID.
maxID := l.page.Max.Value
// If no attachments or the same group is returned, we reached the end.
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
break
}
// Use last ID as the next 'maxID' value.
maxID = attachments[len(attachments)-1].ID
l.page.Max = paging.MaxID(maxID)
for _, a := range attachments {
v := filter(a)
if v != "" {
res = append(res, v)
}
}
// If we got less results than our limit, we've reached the
// last page to retrieve and we can break the loop. If the
// last batch happens to contain exactly the same amount of
// items as the limit we'll end up doing one extra query.
if len(attachments) < l.limit {
break
}
// Grab the last ID from the batch and set it as the maxID
// that'll be used in the next iteration so we don't get items
// we've already seen.
l.maxID = attachments[len(attachments)-1].ID
}
return res, nil
}
@ -80,30 +81,30 @@ func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.
func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) {
res := make([]string, 0, 100)
for {
attachments, err := l.dbService.GetEmojis(ctx, l.maxID, l.limit)
// Get the next page of emoji media up to max ID.
attachments, err := l.dbService.GetEmojis(ctx, &l.page)
if err != nil {
return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
}
// Get current max ID.
maxID := l.page.Max.Value
// If no attachments or the same group is returned, we reached the end.
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
break
}
// Use last ID as the next 'maxID' value.
maxID = attachments[len(attachments)-1].ID
l.page.Max = paging.MaxID(maxID)
for _, a := range attachments {
v := filter(a)
if v != "" {
res = append(res, v)
}
}
// If we got less results than our limit, we've reached the
// last page to retrieve and we can break the loop. If the
// last batch happens to contain exactly the same amount of
// items as the limit we'll end up doing one extra query.
if len(attachments) < l.limit {
break
}
// Grab the last ID from the batch and set it as the maxID
// that'll be used in the next iteration so we don't get items
// we've already seen.
l.maxID = attachments[len(attachments)-1].ID
}
return res, nil
}
@ -137,8 +138,7 @@ func setupList(ctx context.Context) (*list, error) {
return &list{
dbService: dbService,
state: &state,
limit: 200,
maxID: "",
page: paging.Page{Limit: 200},
localOnly: localOnly,
remoteOnly: remoteOnly,
out: bufio.NewWriter(os.Stdout),

63
internal/cleaner/emoji.go
Unescape View File

@ -27,6 +27,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging"
)
// Emoji encompasses a set of
@ -105,8 +106,9 @@ func (e *Emoji) UncacheRemote(ctx context.Context, olderThan time.Time) (int, er
return total, gtserror.Newf("error getting remote emoji: %w", err)
}
if len(emojis) == 0 {
// reached end.
// If no emojis / same group is returned, we reached the end.
if len(emojis) == 0 ||
olderThan.Equal(emojis[len(emojis)-1].CreatedAt) {
break
}
@ -140,23 +142,30 @@ func (e *Emoji) UncacheRemote(ctx context.Context, olderThan time.Time) (int, er
func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
var (
total int
maxID string
page paging.Page
)
// Set page select limit.
page.Limit = selectLimit
for {
// Fetch the next batch of emoji media up to next ID.
emojis, err := e.state.DB.GetEmojis(ctx, maxID, selectLimit)
// Fetch the next batch of emoji to next max ID.
emojis, err := e.state.DB.GetEmojis(ctx, &page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return total, gtserror.Newf("error getting emojis: %w", err)
}
if len(emojis) == 0 {
// reached end.
// Get current max ID.
maxID := page.Max.Value
// If no emoji or the same group is returned, we reached end.
if len(emojis) == 0 || maxID == emojis[len(emojis)-1].ID {
break
}
// Use last as the next 'maxID' value.
// Use last ID as the next 'maxID'.
maxID = emojis[len(emojis)-1].ID
page.Max = paging.MaxID(maxID)
for _, emoji := range emojis {
// Check / fix missing broken emoji.
@ -182,23 +191,30 @@ func (e *Emoji) FixBroken(ctx context.Context) (int, error) {
func (e *Emoji) PruneUnused(ctx context.Context) (int, error) {
var (
total int
maxID string
page paging.Page
)
// Set page select limit.
page.Limit = selectLimit
for {
// Fetch the next batch of emoji media up to next ID.
emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit)
// Fetch the next batch of emoji to next max ID.
emojis, err := e.state.DB.GetRemoteEmojis(ctx, &page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return total, gtserror.Newf("error getting remote emojis: %w", err)
}
if len(emojis) == 0 {
// reached end.
// Get current max ID.
maxID := page.Max.Value
// If no emoji or the same group is returned, we reached end.
if len(emojis) == 0 || maxID == emojis[len(emojis)-1].ID {
break
}
// Use last as the next 'maxID' value.
// Use last ID as the next 'maxID'.
maxID = emojis[len(emojis)-1].ID
page.Max = paging.MaxID(maxID)
for _, emoji := range emojis {
// Check / prune unused emoji media.
@ -224,23 +240,30 @@ func (e *Emoji) PruneUnused(ctx context.Context) (int, error) {
func (e *Emoji) FixCacheStates(ctx context.Context) (int, error) {
var (
total int
maxID string
page paging.Page
)
// Set page select limit.
page.Limit = selectLimit
for {
// Fetch the next batch of emoji media up to next ID.
emojis, err := e.state.DB.GetRemoteEmojis(ctx, maxID, selectLimit)
// Fetch the next batch of emoji to next max ID.
emojis, err := e.state.DB.GetRemoteEmojis(ctx, &page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return total, gtserror.Newf("error getting remote emojis: %w", err)
}
if len(emojis) == 0 {
// reached end.
// Get current max ID.
maxID := page.Max.Value
// If no emoji or the same group is returned, we reached end.
if len(emojis) == 0 || maxID == emojis[len(emojis)-1].ID {
break
}
// Use last as the next 'maxID' value.
// Use last ID as the next 'maxID'.
maxID = emojis[len(emojis)-1].ID
page.Max = paging.MaxID(maxID)
for _, emoji := range emojis {
// Check / fix required emoji cache states.

37
internal/cleaner/media.go
Unescape View File

@ -28,6 +28,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@ -128,23 +129,30 @@ func (m *Media) PruneOrphaned(ctx context.Context) (int, error) {
func (m *Media) PruneUnused(ctx context.Context) (int, error) {
var (
total int
maxID string
page paging.Page
)
// Set page select limit.
page.Limit = selectLimit
for {
// Fetch the next batch of media attachments up to next max ID.
attachments, err := m.state.DB.GetAttachments(ctx, maxID, selectLimit)
// Fetch the next batch of media attachments to next maxID.
attachments, err := m.state.DB.GetAttachments(ctx, &page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return total, gtserror.Newf("error getting attachments: %w", err)
}
if len(attachments) == 0 {
// reached end.
// Get current max ID.
maxID := page.Max.Value
// If no attachments or the same group is returned, we reached the end.
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
break
}
// Use last ID as the next 'maxID' value.
maxID = attachments[len(attachments)-1].ID
page.Max = paging.MaxID(maxID)
for _, media := range attachments {
// Check / prune unused media attachment.
@ -183,8 +191,9 @@ func (m *Media) UncacheRemote(ctx context.Context, olderThan time.Time) (int, er
return total, gtserror.Newf("error getting remote attachments: %w", err)
}
if len(attachments) == 0 {
// reached end.
// If no attachments / same group is returned, we reached the end.
if len(attachments) == 0 ||
olderThan.Equal(attachments[len(attachments)-1].CreatedAt) {
break
}
@ -215,23 +224,29 @@ func (m *Media) UncacheRemote(ctx context.Context, olderThan time.Time) (int, er
func (m *Media) FixCacheStates(ctx context.Context) (int, error) {
var (
total int
maxID string
page paging.Page
)
// Set page select limit.
page.Limit = selectLimit
for {
// Fetch the next batch of media attachments up to next max ID.
attachments, err := m.state.DB.GetRemoteAttachments(ctx, maxID, selectLimit)
attachments, err := m.state.DB.GetRemoteAttachments(ctx, &page)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return total, gtserror.Newf("error getting remote attachments: %w", err)
}
// Get current max ID.
maxID := page.Max.Value
if len(attachments) == 0 {
// reached end.
// If no attachments or the same group is returned, we reached the end.
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
break
}
// Use last ID as the next 'maxID' value.
maxID = attachments[len(attachments)-1].ID
page.Max = paging.MaxID(maxID)
for _, media := range attachments {
// Check / fix required media cache states.

15
internal/db/bundb/emoji.go
Unescape View File

@ -30,6 +30,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/util"
"github.com/uptrace/bun"
@ -326,8 +327,11 @@ func (e *emojiDB) GetEmojisBy(ctx context.Context, domain string, includeDisable
return e.GetEmojisByIDs(ctx, emojiIDs)
}
func (e *emojiDB) GetEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error) {
var emojiIDs []string
func (e *emojiDB) GetEmojis(ctx context.Context, page *paging.Page) ([]*gtsmodel.Emoji, error) {
maxID := page.GetMax()
limit := page.GetLimit()
emojiIDs := make([]string, 0, limit)
q := e.db.NewSelect().
Table("emojis").
@ -349,8 +353,11 @@ func (e *emojiDB) GetEmojis(ctx context.Context, maxID string, limit int) ([]*gt
return e.GetEmojisByIDs(ctx, emojiIDs)
}
func (e *emojiDB) GetRemoteEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error) {
var emojiIDs []string
func (e *emojiDB) GetRemoteEmojis(ctx context.Context, page *paging.Page) ([]*gtsmodel.Emoji, error) {
maxID := page.GetMax()
limit := page.GetLimit()
emojiIDs := make([]string, 0, limit)
q := e.db.NewSelect().
Table("emojis").

11
internal/db/bundb/media.go
Unescape View File

@ -27,6 +27,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/util"
"github.com/uptrace/bun"
@ -232,7 +233,10 @@ func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
return err
}
func (m *mediaDB) GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) {
func (m *mediaDB) GetAttachments(ctx context.Context, page *paging.Page) ([]*gtsmodel.MediaAttachment, error) {
maxID := page.GetMax()
limit := page.GetLimit()
attachmentIDs := make([]string, 0, limit)
q := m.db.NewSelect().
@ -255,7 +259,10 @@ func (m *mediaDB) GetAttachments(ctx context.Context, maxID string, limit int) (
return m.GetAttachmentsByIDs(ctx, attachmentIDs)
}
func (m *mediaDB) GetRemoteAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) {
func (m *mediaDB) GetRemoteAttachments(ctx context.Context, page *paging.Page) ([]*gtsmodel.MediaAttachment, error) {
maxID := page.GetMax()
limit := page.GetLimit()
attachmentIDs := make([]string, 0, limit)
q := m.db.NewSelect().

5
internal/db/emoji.go
Unescape View File

@ -22,6 +22,7 @@ import (
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/paging"
)
// EmojiAllDomains can be used as the `domain` value in a GetEmojis
@ -47,10 +48,10 @@ type Emoji interface {
GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, error)
// GetEmojis fetches all emojis with IDs less than 'maxID', up to a maximum of 'limit' emojis.
GetEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error)
GetEmojis(ctx context.Context, page *paging.Page) ([]*gtsmodel.Emoji, error)
// GetRemoteEmojis fetches all remote emojis with IDs less than 'maxID', up to a maximum of 'limit' emojis.
GetRemoteEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error)
GetRemoteEmojis(ctx context.Context, page *paging.Page) ([]*gtsmodel.Emoji, error)
// GetCachedEmojisOlderThan fetches all cached remote emojis with 'updated_at' greater than 'olderThan', up to a maximum of 'limit' emojis.
GetCachedEmojisOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.Emoji, error)

5
internal/db/media.go
Unescape View File

@ -22,6 +22,7 @@ import (
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/paging"
)
// Media contains functions related to creating/getting/removing media attachments.
@ -42,10 +43,10 @@ type Media interface {
DeleteAttachment(ctx context.Context, id string) error
// GetAttachments fetches media attachments up to a given max ID, and at most limit.
GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error)
GetAttachments(ctx context.Context, page *paging.Page) ([]*gtsmodel.MediaAttachment, error)
// GetRemoteAttachments fetches media attachments with a non-empty domain, up to a given max ID, and at most limit.
GetRemoteAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error)
GetRemoteAttachments(ctx context.Context, page *paging.Page) ([]*gtsmodel.MediaAttachment, error)
// GetCachedAttachmentsOlderThan gets limit n remote attachments (including avatars and headers) older than
// the given time. These will be returned in order of attachment.created_at descending (i.e. newest to oldest).

71
internal/util/slices.go
Unescape View File

@ -17,6 +17,8 @@
package util
import "slices"
// Deduplicate deduplicates entries in the given slice.
func Deduplicate[T comparable](in []T) []T {
var (
@ -47,6 +49,10 @@ func DeduplicateFunc[T any, C comparable](in []T, key func(v T) C) []T {
deduped = make([]T, 0, inL)
)
if key == nil {
panic("nil func")
}
for _, v := range in {
k := key(v)
@ -66,6 +72,10 @@ func DeduplicateFunc[T any, C comparable](in []T, key func(v T) C) []T {
// passing each item to 'get' and deduplicating the end result.
// Compared to Deduplicate() this returns []K, NOT input type []T.
func Collate[T any, K comparable](in []T, get func(T) K) []K {
if get == nil {
panic("nil func")
}
ks := make([]K, 0, len(in))
km := make(map[K]struct{}, len(in))
@ -86,50 +96,25 @@ func Collate[T any, K comparable](in []T, get func(T) K) []K {
// OrderBy orders a slice of given type by the provided alternative slice of comparable type.
func OrderBy[T any, K comparable](in []T, keys []K, key func(T) K) {
var (
start int
offset int
)
for i := 0; i < len(keys); i++ {
var (
// key at index.
k = keys[i]
// sentinel
// idx value.
idx = -1
)
// Look for model with key in slice.
for j := start; j < len(in); j++ {
if key(in[j]) == k {
idx = j
break
}
}
if idx == -1 {
// model with key
// was not found.
offset++
continue
}
// Update
// start
start++
if key == nil {
panic("nil func")
}
// Expected ID index.
exp := i - offset
// Create lookup of keys->idx.
m := make(map[K]int, len(in))
for i, k := range keys {
m[k] = i
}
if idx == exp {
// Model is in expected
// location, keep going.
continue
// Sort according to the reverse lookup.
slices.SortFunc(in, func(a, b T) int {
ai := m[key(a)]
bi := m[key(b)]
if ai < bi {
return -1
} else if bi < ai {
return +1
}
// Swap models at current and expected.
in[idx], in[exp] = in[exp], in[idx]
}
return 0
})
}

Write Preview
Loading…
Cancel
Save