Browse Source
* [feature/chore] Add Move database functions + cache * add move mem ratio to envparsing.sh * update commentpull/2726/head
17 changed files with 671 additions and 1 deletions
@ -0,0 +1,61 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"context" |
||||
"strings" |
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" |
||||
"github.com/uptrace/bun" |
||||
) |
||||
|
||||
func init() { |
||||
up := func(ctx context.Context, db *bun.DB) error { |
||||
_, err := db.ExecContext(ctx, |
||||
"ALTER TABLE ? ADD COLUMN ? CHAR(26)", |
||||
bun.Ident("accounts"), bun.Ident("move_id"), |
||||
) |
||||
if err != nil { |
||||
e := err.Error() |
||||
if !(strings.Contains(e, "already exists") || |
||||
strings.Contains(e, "duplicate column name") || |
||||
strings.Contains(e, "SQLSTATE 42701")) { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
// Create "moves" table.
|
||||
if _, err := db.NewCreateTable(). |
||||
IfNotExists(). |
||||
Model(>smodel.Move{}). |
||||
Exec(ctx); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
down := func(ctx context.Context, db *bun.DB) error { |
||||
return nil |
||||
} |
||||
|
||||
if err := Migrations.Register(up, down); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,236 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package bundb |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"net/url" |
||||
"time" |
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db" |
||||
"github.com/superseriousbusiness/gotosocial/internal/gtscontext" |
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" |
||||
"github.com/superseriousbusiness/gotosocial/internal/state" |
||||
"github.com/uptrace/bun" |
||||
) |
||||
|
||||
type moveDB struct { |
||||
db *bun.DB |
||||
state *state.State |
||||
} |
||||
|
||||
func (m *moveDB) GetMoveByID( |
||||
ctx context.Context, |
||||
id string, |
||||
) (*gtsmodel.Move, error) { |
||||
return m.getMove( |
||||
ctx, |
||||
"ID", |
||||
func(move *gtsmodel.Move) error { |
||||
return m.db. |
||||
NewSelect(). |
||||
Model(move). |
||||
Where("? = ?", bun.Ident("move.id"), id). |
||||
Scan(ctx) |
||||
}, |
||||
id, |
||||
) |
||||
} |
||||
|
||||
func (m *moveDB) GetMoveByURI( |
||||
ctx context.Context, |
||||
uri string, |
||||
) (*gtsmodel.Move, error) { |
||||
return m.getMove( |
||||
ctx, |
||||
"URI", |
||||
func(move *gtsmodel.Move) error { |
||||
return m.db. |
||||
NewSelect(). |
||||
Model(move). |
||||
Where("? = ?", bun.Ident("move.uri"), uri). |
||||
Scan(ctx) |
||||
}, |
||||
uri, |
||||
) |
||||
} |
||||
|
||||
func (m *moveDB) GetMoveByOriginTarget( |
||||
ctx context.Context, |
||||
originURI string, |
||||
targetURI string, |
||||
) (*gtsmodel.Move, error) { |
||||
return m.getMove( |
||||
ctx, |
||||
"OriginURI,TargetURI", |
||||
func(move *gtsmodel.Move) error { |
||||
return m.db. |
||||
NewSelect(). |
||||
Model(move). |
||||
Where("? = ?", bun.Ident("move.origin_uri"), originURI). |
||||
Where("? = ?", bun.Ident("move.target_uri"), targetURI). |
||||
Scan(ctx) |
||||
}, |
||||
originURI, targetURI, |
||||
) |
||||
} |
||||
|
||||
func (m *moveDB) GetLatestMoveSuccessInvolvingURIs( |
||||
ctx context.Context, |
||||
uri1 string, |
||||
uri2 string, |
||||
) (time.Time, error) { |
||||
// Get at most 1 latest Move
|
||||
// involving the provided URIs.
|
||||
var moves []*gtsmodel.Move |
||||
err := m.db. |
||||
NewSelect(). |
||||
Model(&moves). |
||||
Column("succeeded_at"). |
||||
Where("? = ?", bun.Ident("move.origin_uri"), uri1). |
||||
WhereOr("? = ?", bun.Ident("move.origin_uri"), uri2). |
||||
WhereOr("? = ?", bun.Ident("move.target_uri"), uri1). |
||||
WhereOr("? = ?", bun.Ident("move.target_uri"), uri2). |
||||
Order("id DESC"). |
||||
Limit(1). |
||||
Scan(ctx) |
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) { |
||||
return time.Time{}, err |
||||
} |
||||
|
||||
if len(moves) != 1 { |
||||
return time.Time{}, nil |
||||
} |
||||
|
||||
return moves[0].SucceededAt, nil |
||||
} |
||||
|
||||
func (m *moveDB) GetLatestMoveAttemptInvolvingURIs( |
||||
ctx context.Context, |
||||
uri1 string, |
||||
uri2 string, |
||||
) (time.Time, error) { |
||||
// Get at most 1 latest Move
|
||||
// involving the provided URIs.
|
||||
var moves []*gtsmodel.Move |
||||
err := m.db. |
||||
NewSelect(). |
||||
Model(&moves). |
||||
Column("attempted_at"). |
||||
Where("? = ?", bun.Ident("move.origin_uri"), uri1). |
||||
WhereOr("? = ?", bun.Ident("move.origin_uri"), uri2). |
||||
WhereOr("? = ?", bun.Ident("move.target_uri"), uri1). |
||||
WhereOr("? = ?", bun.Ident("move.target_uri"), uri2). |
||||
Order("id DESC"). |
||||
Limit(1). |
||||
Scan(ctx) |
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) { |
||||
return time.Time{}, err |
||||
} |
||||
|
||||
if len(moves) != 1 { |
||||
return time.Time{}, nil |
||||
} |
||||
|
||||
return moves[0].AttemptedAt, nil |
||||
} |
||||
|
||||
func (m *moveDB) getMove( |
||||
ctx context.Context, |
||||
lookup string, |
||||
dbQuery func(*gtsmodel.Move) error, |
||||
keyParts ...any, |
||||
) (*gtsmodel.Move, error) { |
||||
move, err := m.state.Caches.GTS.Move.LoadOne(lookup, func() (*gtsmodel.Move, error) { |
||||
var move gtsmodel.Move |
||||
|
||||
// Not cached! Perform database query.
|
||||
if err := dbQuery(&move); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &move, nil |
||||
}, keyParts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if gtscontext.Barebones(ctx) { |
||||
return move, nil |
||||
} |
||||
|
||||
// Populate the Move by parsing out the URIs.
|
||||
if move.Origin == nil { |
||||
move.Origin, err = url.Parse(move.OriginURI) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("error parsing Move originURI: %w", err) |
||||
} |
||||
} |
||||
|
||||
if move.Target == nil { |
||||
move.Target, err = url.Parse(move.TargetURI) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("error parsing Move originURI: %w", err) |
||||
} |
||||
} |
||||
|
||||
return move, nil |
||||
} |
||||
|
||||
func (m *moveDB) PutMove(ctx context.Context, move *gtsmodel.Move) error { |
||||
return m.state.Caches.GTS.Move.Store(move, func() error { |
||||
_, err := m.db. |
||||
NewInsert(). |
||||
Model(move). |
||||
Exec(ctx) |
||||
return err |
||||
}) |
||||
} |
||||
|
||||
func (m *moveDB) UpdateMove(ctx context.Context, move *gtsmodel.Move, columns ...string) error { |
||||
move.UpdatedAt = time.Now() |
||||
if len(columns) > 0 { |
||||
// If we're updating by column,
|
||||
// ensure "updated_at" is included.
|
||||
columns = append(columns, "updated_at") |
||||
} |
||||
|
||||
return m.state.Caches.GTS.Move.Store(move, func() error { |
||||
_, err := m.db. |
||||
NewUpdate(). |
||||
Model(move). |
||||
Column(columns...). |
||||
Where("? = ?", bun.Ident("move.id"), move.ID). |
||||
Exec(ctx) |
||||
return err |
||||
}) |
||||
} |
||||
|
||||
func (m *moveDB) DeleteMoveByID(ctx context.Context, id string) error { |
||||
defer m.state.Caches.GTS.Move.Invalidate("ID", id) |
||||
|
||||
_, err := m.db. |
||||
NewDelete(). |
||||
TableExpr("? AS ?", bun.Ident("moves"), bun.Ident("move")). |
||||
Where("? = ?", bun.Ident("move.id"), id). |
||||
Exec(ctx) |
||||
|
||||
return err |
||||
} |
||||
@ -0,0 +1,168 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package bundb_test |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/stretchr/testify/suite" |
||||
"github.com/superseriousbusiness/gotosocial/internal/db" |
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" |
||||
) |
||||
|
||||
type MoveTestSuite struct { |
||||
BunDBStandardTestSuite |
||||
} |
||||
|
||||
func (suite *MoveTestSuite) TestMoveIntegration() { |
||||
ctx := context.Background() |
||||
firstMove := >smodel.Move{ |
||||
ID: "01HPPN38MZYEC6WBTR21J6241N", |
||||
OriginURI: "https://example.org/users/my_old_account", |
||||
TargetURI: "https://somewhere.else.net/users/my_new_account", |
||||
URI: "https://example.org/users/my_old_account/activities/Move/652e8361-0182-407d-8b01-4447e7fd10c0", |
||||
} |
||||
|
||||
// Put the move.
|
||||
if err := suite.state.DB.PutMove(ctx, firstMove); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Test various ways of retrieving the Move.
|
||||
if _, err := suite.state.DB.GetMoveByID(ctx, firstMove.ID); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
if _, err := suite.state.DB.GetMoveByOriginTarget(ctx, firstMove.OriginURI, firstMove.TargetURI); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Keep the last one, and check fields set on it.
|
||||
dbMove, err := suite.state.DB.GetMoveByURI(ctx, firstMove.URI) |
||||
if err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Created/Updated should be set when
|
||||
// it's first inserted into the db.
|
||||
suite.NotZero(dbMove.CreatedAt) |
||||
suite.NotZero(dbMove.UpdatedAt) |
||||
|
||||
// URIs should be parsed and set
|
||||
// on the move on population.
|
||||
suite.NotNil(dbMove.Origin) |
||||
suite.NotNil(dbMove.Target) |
||||
|
||||
// These should not be set as
|
||||
// they have no default values.
|
||||
suite.Zero(dbMove.AttemptedAt) |
||||
suite.Zero(dbMove.SucceededAt) |
||||
|
||||
// Update the Move to emulate
|
||||
// us succeeding in processing it.
|
||||
dbMove.AttemptedAt = time.Now() |
||||
dbMove.SucceededAt = dbMove.AttemptedAt |
||||
if err := suite.state.DB.UpdateMove( |
||||
ctx, |
||||
dbMove, |
||||
"attempted_at", |
||||
"succeeded_at", |
||||
); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Store dbMove as firstMove var.
|
||||
firstMove = dbMove |
||||
|
||||
// Store another Move involving one
|
||||
// of the original URIs, and mark
|
||||
// this one as succeeded. Use a time
|
||||
// a few seconds into the future to
|
||||
// make sure it's differentiated
|
||||
// from the first move.
|
||||
secondMove := >smodel.Move{ |
||||
ID: "01HPPPNQWRMQTXRFEPKDV3A4W7", |
||||
OriginURI: "https://somewhere.else.net/users/my_new_account", |
||||
TargetURI: "http://localhost:8080/users/the_mighty_zork", |
||||
URI: "https://somewhere.else.net/activities/01HPPPPPC089VJGV0967P5YQS5", |
||||
AttemptedAt: time.Now().Add(5 * time.Second), |
||||
SucceededAt: time.Now().Add(5 * time.Second), |
||||
} |
||||
if err := suite.state.DB.PutMove(ctx, secondMove); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Test getting succeeded using the
|
||||
// URI shared between the two Moves,
|
||||
// and some random account.
|
||||
ts, err := suite.state.DB.GetLatestMoveSuccessInvolvingURIs( |
||||
ctx, |
||||
secondMove.OriginURI, |
||||
"https://a.secret.third.place/users/mystery_meat", |
||||
) |
||||
if err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Time should be equivalent to secondMove.
|
||||
suite.EqualValues(secondMove.SucceededAt.UnixMilli(), ts.UnixMilli()) |
||||
|
||||
// Test getting succeeded using
|
||||
// both URIs from the first move.
|
||||
ts, err = suite.state.DB.GetLatestMoveSuccessInvolvingURIs( |
||||
ctx, |
||||
firstMove.OriginURI, |
||||
firstMove.TargetURI, |
||||
) |
||||
if err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Time should be equivalent to secondMove.
|
||||
suite.EqualValues(secondMove.SucceededAt.UnixMilli(), ts.UnixMilli()) |
||||
|
||||
// Test getting succeeded using
|
||||
// URI from the first Move, and
|
||||
// some random account.
|
||||
ts, err = suite.state.DB.GetLatestMoveSuccessInvolvingURIs( |
||||
ctx, |
||||
firstMove.OriginURI, |
||||
"https://a.secret.third.place/users/mystery_meat", |
||||
) |
||||
if err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Time should be equivalent to firstMove.
|
||||
suite.EqualValues(firstMove.SucceededAt.UnixMilli(), ts.UnixMilli()) |
||||
|
||||
// Delete the first Move.
|
||||
if err := suite.state.DB.DeleteMoveByID(ctx, firstMove.ID); err != nil { |
||||
suite.FailNow(err.Error()) |
||||
} |
||||
|
||||
// Ensure first Move deleted.
|
||||
_, err = suite.state.DB.GetMoveByID(ctx, firstMove.ID) |
||||
suite.ErrorIs(err, db.ErrNoEntries) |
||||
} |
||||
|
||||
func TestMoveTestSuite(t *testing.T) { |
||||
suite.Run(t, new(MoveTestSuite)) |
||||
} |
||||
@ -0,0 +1,56 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package db |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" |
||||
) |
||||
|
||||
type Move interface { |
||||
// GetMoveByID gets one Move with the given internal ID.
|
||||
GetMoveByID(ctx context.Context, id string) (*gtsmodel.Move, error) |
||||
|
||||
// GetMoveByURI gets one Move with the given AP URI.
|
||||
GetMoveByURI(ctx context.Context, uri string) (*gtsmodel.Move, error) |
||||
|
||||
// GetMoveByOriginTarget gets one move with the given originURI and targetURI.
|
||||
GetMoveByOriginTarget(ctx context.Context, originURI string, targetURI string) (*gtsmodel.Move, error) |
||||
|
||||
// GetLatestMoveSuccessInvolvingURIs gets the time of
|
||||
// the latest successfully-processed Move that includes
|
||||
// either uri1 or uri2 in target or origin positions.
|
||||
GetLatestMoveSuccessInvolvingURIs(ctx context.Context, uri1 string, uri2 string) (time.Time, error) |
||||
|
||||
// GetLatestMoveAttemptInvolvingURIs gets the time
|
||||
// of the latest Move attempt that includes either
|
||||
// uri1 or uri2 in target or origin positions.
|
||||
GetLatestMoveAttemptInvolvingURIs(ctx context.Context, uri1 string, uri2 string) (time.Time, error) |
||||
|
||||
// PutMove puts the given Move in the database.
|
||||
PutMove(ctx context.Context, move *gtsmodel.Move) error |
||||
|
||||
// UpdateMove updates the given Move by primary key.
|
||||
// Updates specific columns if provided, all columns if not.
|
||||
UpdateMove(ctx context.Context, move *gtsmodel.Move, columns ...string) error |
||||
|
||||
// DeleteMoveByID deletes a move with the given internal ID.
|
||||
DeleteMoveByID(ctx context.Context, id string) error |
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package gtsmodel |
||||
|
||||
import ( |
||||
"net/url" |
||||
"time" |
||||
) |
||||
|
||||
// Move represents an ActivityPub "Move" activity
|
||||
// received (or created) by this instance.
|
||||
type Move struct { |
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database.
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was item created.
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was item last updated.
|
||||
AttemptedAt time.Time `bun:"type:timestamptz,nullzero"` // When was processing of the Move to TargetURI last attempted by our instance (zero if not yet attempted).
|
||||
SucceededAt time.Time `bun:"type:timestamptz,nullzero"` // When did the processing of the Move to TargetURI succeed according to our criteria (zero if not yet complete).
|
||||
OriginURI string `bun:",nullzero,notnull,unique:moveorigintarget"` // OriginURI of the Move. Ie., the Move Object.
|
||||
Origin *url.URL `bun:"-"` // URL corresponding to OriginURI. Not stored in the database.
|
||||
TargetURI string `bun:",nullzero,notnull,unique:moveorigintarget"` // TargetURI of the Move. Ie., the Move Target.
|
||||
Target *url.URL `bun:"-"` // URL corresponding to TargetURI. Not stored in the database.
|
||||
URI string `bun:",nullzero,notnull,unique"` // ActivityPub ID/URI of the Move Activity itself.
|
||||
} |
||||
Loading…
Reference in new issue