From 0b7b0d528515c917931a0465e78a0c34c6c55c8b Mon Sep 17 00:00:00 2001 From: biast12 Date: Thu, 29 Jan 2026 22:32:37 +0100 Subject: [PATCH] RM-239: Remove import pipeline --- database.go | 6 -- guildpurge.go | 2 - importlogs.go | 155 ---------------------------------- importmapping.go | 83 ------------------ participants.go | 13 --- serviceratings.go | 12 --- sql/import_logs/schema.sql | 29 ------- sql/import_logs/set.sql | 4 - sql/import_logs/set_run.sql | 4 - sql/import_mapping/schema.sql | 10 --- sql/import_mapping/set.sql | 3 - ticketclaims.go | 11 --- ticketlastmessage.go | 18 ---- ticketmembers.go | 14 --- tickets.go | 32 ------- 15 files changed, 396 deletions(-) delete mode 100644 importlogs.go delete mode 100644 importmapping.go delete mode 100644 sql/import_logs/schema.sql delete mode 100644 sql/import_logs/set.sql delete mode 100644 sql/import_logs/set_run.sql delete mode 100644 sql/import_mapping/schema.sql delete mode 100644 sql/import_mapping/set.sql diff --git a/database.go b/database.go index ad6247e..b4be4a7 100644 --- a/database.go +++ b/database.go @@ -52,8 +52,6 @@ type Database struct { GlobalBlacklist *GlobalBlacklist GuildLeaveTime *GuildLeaveTime GuildMetadata *GuildMetadataTable - ImportLogs *ImportLogsTable - ImportMappingTable *ImportMappingTable LegacyPremiumEntitlementGuilds *LegacyPremiumEntitlementGuilds LegacyPremiumEntitlements *LegacyPremiumEntitlements MultiPanels *MultiPanelTable @@ -147,8 +145,6 @@ func NewDatabase(pool *pgxpool.Pool) *Database { GlobalBlacklist: newGlobalBlacklist(pool), GuildLeaveTime: newGuildLeaveTime(pool), GuildMetadata: newGuildMetadataTable(pool), - ImportLogs: newImportLogs(pool), - ImportMappingTable: newImportMapping(pool), LegacyPremiumEntitlementGuilds: newLegacyPremiumEntitlementGuildsTable(pool), LegacyPremiumEntitlements: newLegacyPremiumEntitlement(pool), MultiPanels: newMultiMultiPanelTable(pool), @@ -262,8 +258,6 @@ func (d *Database) CreateTables(ctx context.Context, pool *pgxpool.Pool) { d.GlobalBlacklist, d.GuildLeaveTime, d.GuildMetadata, - d.ImportLogs, - d.ImportMappingTable, d.LegacyPremiumEntitlements, d.LegacyPremiumEntitlementGuilds, d.MultiPanels, diff --git a/guildpurge.go b/guildpurge.go index 40af1f4..0f551e0 100644 --- a/guildpurge.go +++ b/guildpurge.go @@ -68,8 +68,6 @@ func (d *Database) PurgeGuildData(ctx context.Context, guildId uint64, logger *z "custom_colours", "feedback_enabled", "guild_metadata", - "import_logs", - "import_mapping", "legacy_premium_entitlement_guilds", "naming_scheme", "on_call", diff --git a/importlogs.go b/importlogs.go deleted file mode 100644 index 99df5de..0000000 --- a/importlogs.go +++ /dev/null @@ -1,155 +0,0 @@ -package database - -import ( - "context" - _ "embed" - "strings" - "time" - - "github.com/jackc/pgx/v4/pgxpool" -) - -type ImportLogsTable struct { - *pgxpool.Pool -} - -type ImportRun struct { - RunId int `json:"run_id"` - RunType string `json:"run_type"` - Date time.Time `json:"date"` - Logs []ImportLogs `json:"logs"` -} - -type ImportLogs struct { - GuildId uint64 `json:"guild_id"` - LogType string `json:"log_type"` - RunType string `json:"run_type"` - RunId int `json:"run_id"` - RunLogId int `json:"run_log_id"` - EntityType *string `json:"entity_type"` - Message *string `json:"message"` - Date time.Time `json:"date"` -} - -var ( - //go:embed sql/import_logs/schema.sql - importLogsSchema string - - //go:embed sql/import_logs/set.sql - importLogsSet string - - //go:embed sql/import_logs/set_run.sql - importLogsSetRun string -) - -func newImportLogs(db *pgxpool.Pool) *ImportLogsTable { - return &ImportLogsTable{ - db, - } -} - -func (s ImportLogsTable) Schema() string { - return importLogsSchema -} - -func (s *ImportLogsTable) GetFinishedRuns(ctx context.Context, guildId uint64) ([]ImportRun, error) { - query := `SELECT run_id, run_type, date FROM import_logs WHERE "guild_id" = $1 AND log_type = 'RUN_COMPLETE';` - - var runs []ImportRun - - rows, err := s.Query(ctx, query, guildId) - if err != nil { - return nil, err - } - - for rows.Next() { - var mappingEntry ImportRun - if err := rows.Scan(&mappingEntry.RunId, &mappingEntry.RunType, &mappingEntry.Date); err != nil { - return nil, err - } - - runs = append(runs, mappingEntry) - } - - return runs, nil -} - -func (s *ImportLogsTable) GetRuns(ctx context.Context, guildId uint64) ([]ImportRun, error) { - query := `SELECT run_id, run_type, date FROM import_logs WHERE "guild_id" = $1 AND log_type = 'RUN_START';` - - var runs []ImportRun - - rows, err := s.Query(ctx, query, guildId) - if err != nil { - return nil, err - } - - for rows.Next() { - var mappingEntry ImportRun - if err := rows.Scan(&mappingEntry.RunId, &mappingEntry.RunType, &mappingEntry.Date); err != nil { - return nil, err - } - - runs = append(runs, mappingEntry) - } - - for i := range runs { - logs, err := s.GetRunLogs(ctx, guildId, runs[i].RunId) - if err != nil { - return nil, err - } - - runs[i].Logs = logs - } - - return runs, nil -} - -func (s *ImportLogsTable) GetRunLogs(ctx context.Context, guildId uint64, runId int) ([]ImportLogs, error) { - query := `SELECT guild_id, log_type, run_id, run_log_id, run_type, entity_type, message, date FROM import_logs WHERE guild_id = $1 AND run_id = $2 ORDER BY run_log_id ASC;` - - var logs []ImportLogs - - rows, err := s.Query(ctx, query, guildId, runId) - if err != nil { - return nil, err - } - - for rows.Next() { - var logEntry ImportLogs - if err := rows.Scan(&logEntry.GuildId, &logEntry.LogType, &logEntry.RunId, &logEntry.RunLogId, &logEntry.RunType, &logEntry.EntityType, &logEntry.Message, &logEntry.Date); err != nil { - return nil, err - } - - logs = append(logs, logEntry) - } - - return logs, nil -} - -func (s *ImportLogsTable) CreateRun(ctx context.Context, guildId uint64, runType string) (int, error) { - runCount := 1 - currentRuns, _ := s.GetRuns(ctx, guildId) - - runCount += len(currentRuns) - - _, err := s.Exec(ctx, importLogsSetRun, guildId, "RUN_START", runCount, runType) - return runCount, err -} - -func (s *ImportLogsTable) AddLog(ctx context.Context, guildId uint64, runId int, runType string, logType string, entityType string, message string) error { - return s.addLog(ctx, guildId, runId, runType, logType, entityType, message, 1) -} - -func (s *ImportLogsTable) addLog(ctx context.Context, guildId uint64, runId int, runType string, logType string, entityType string, message string, try int) error { - _, err := s.Exec(ctx, importLogsSet, guildId, logType, runId, runType, entityType, message) - if err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") { - // Try again - if try > 5 { - return err - } - - return s.addLog(ctx, guildId, runId, runType, logType, entityType, message, try+1) - } - return err -} diff --git a/importmapping.go b/importmapping.go deleted file mode 100644 index b4bbc2d..0000000 --- a/importmapping.go +++ /dev/null @@ -1,83 +0,0 @@ -package database - -import ( - "context" - _ "embed" - - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" -) - -type ImportMappingTable struct { - *pgxpool.Pool -} - -type ImportMapping struct { - GuildId uint64 `json:"guild_id"` - Area string `json:"area"` - SourceId int `json:"source_id"` - TargetId int `json:"target_id"` -} - -var ( - //go:embed sql/import_mapping/schema.sql - importMappingSchema string - - //go:embed sql/import_mapping/set.sql - importMappingSet string -) - -func newImportMapping(db *pgxpool.Pool) *ImportMappingTable { - return &ImportMappingTable{ - db, - } -} - -func (s ImportMappingTable) Schema() string { - return importMappingSchema -} - -func (s *ImportMappingTable) GetMapping(ctx context.Context, guildId uint64) (map[string]map[int]int, error) { - query := `SELECT * FROM import_mapping WHERE "guild_id" = $1;` - - rows, err := s.Query(ctx, query, guildId) - if err != nil { - return nil, err - } - - mapping := make(map[string]map[int]int) - - for rows.Next() { - var mappingEntry ImportMapping - if err := rows.Scan(&mappingEntry.GuildId, &mappingEntry.Area, &mappingEntry.SourceId, &mappingEntry.TargetId); err != nil { - return nil, err - } - - if _, ok := mapping[mappingEntry.Area]; !ok { - mapping[mappingEntry.Area] = make(map[int]int) - } - - mapping[mappingEntry.Area][mappingEntry.SourceId] = mappingEntry.TargetId - } - - return mapping, nil -} - -func (s *ImportMappingTable) Set(ctx context.Context, guildId uint64, area string, sourceId, targetId int) error { - _, err := s.Exec(ctx, importMappingSet, guildId, area, sourceId, targetId) - return err -} - -func (s *ImportMappingTable) SetBulk(ctx context.Context, guildId uint64, area string, mappings map[int]int) error { - rows := make([][]interface{}, len(mappings)) - - i := 0 - for sourceId, targetId := range mappings { - rows[i] = []interface{}{guildId, area, sourceId, targetId} - i++ - } - - _, err := s.CopyFrom(ctx, pgx.Identifier{"import_mapping"}, []string{"guild_id", "area", "source_id", "target_id"}, pgx.CopyFromRows(rows)) - - return err -} diff --git a/participants.go b/participants.go index ad5642a..09329bf 100644 --- a/participants.go +++ b/participants.go @@ -101,19 +101,6 @@ SELECT EXISTS( return } -func (p *ParticipantTable) ImportBulk(ctx context.Context, guildId uint64, participantMap map[int][]uint64) (err error) { - rows := make([][]interface{}, 0) - - for ticketId, participants := range participantMap { - for _, userId := range participants { - rows = append(rows, []interface{}{guildId, ticketId, userId}) - } - } - - _, err = p.CopyFrom(ctx, pgx.Identifier{"participant"}, []string{"guild_id", "ticket_id", "user_id"}, pgx.CopyFromRows(rows)) - return -} - func (p *ParticipantTable) Set(ctx context.Context, guildId uint64, ticketId int, userId uint64) (err error) { query := ` INSERT INTO participant("guild_id", "ticket_id", "user_id") diff --git a/serviceratings.go b/serviceratings.go index 02fe3b9..3607c67 100644 --- a/serviceratings.go +++ b/serviceratings.go @@ -150,18 +150,6 @@ func (r *ServiceRatings) GetRange(ctx context.Context, guildId uint64, lowerId, return ratings, nil } -func (r *ServiceRatings) ImportBulk(ctx context.Context, guildId uint64, ratings map[int]uint8) (err error) { - rows := make([][]interface{}, 0) - - for ticketId, rating := range ratings { - rows = append(rows, []interface{}{guildId, ticketId, rating}) - } - - _, err = r.CopyFrom(ctx, pgx.Identifier{"service_ratings"}, []string{"guild_id", "ticket_id", "rating"}, pgx.CopyFromRows(rows)) - - return -} - func (r *ServiceRatings) Set(ctx context.Context, guildId uint64, ticketId int, rating uint8) (err error) { query := ` INSERT INTO service_ratings("guild_id", "ticket_id", "rating") diff --git a/sql/import_logs/schema.sql b/sql/import_logs/schema.sql deleted file mode 100644 index 464da54..0000000 --- a/sql/import_logs/schema.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE IF NOT EXISTS import_logs ( - guild_id BIGINT NOT NULL, - log_type VARCHAR(255) NOT NULL, - run_type VARCHAR(255) NOT NULL DEFAULT 'DATA', - run_id INT NOT NULL, - run_log_id INT NOT NULL, - entity_type VARCHAR(255), - message VARCHAR(255), - date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (guild_id, run_id, run_log_id) -- Ensures uniqueness per (guild_id, run_id) -); - -CREATE OR REPLACE FUNCTION set_run_log_id() -RETURNS TRIGGER AS $$ -BEGIN - -- Get the next run_log_id for the (guild_id, run_id) pair - SELECT COALESCE(MAX(run_log_id), 0) + 1 INTO NEW.run_log_id - FROM import_logs - WHERE guild_id = NEW.guild_id AND run_id = NEW.run_id; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER before_insert_import_logs -BEFORE INSERT ON import_logs -FOR EACH ROW -WHEN (NEW.run_log_id IS NULL) -EXECUTE FUNCTION set_run_log_id(); \ No newline at end of file diff --git a/sql/import_logs/set.sql b/sql/import_logs/set.sql deleted file mode 100644 index c56088e..0000000 --- a/sql/import_logs/set.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO import_logs - (guild_id, log_type, run_id, run_type, entity_type, message) -VALUES - ($1, $2, $3, $4, $5, $6) \ No newline at end of file diff --git a/sql/import_logs/set_run.sql b/sql/import_logs/set_run.sql deleted file mode 100644 index 1a5b16a..0000000 --- a/sql/import_logs/set_run.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO import_logs - (guild_id, log_type, run_id, run_log_id, run_type) -VALUES - ($1, $2, $3, 1, $4) \ No newline at end of file diff --git a/sql/import_mapping/schema.sql b/sql/import_mapping/schema.sql deleted file mode 100644 index 9be8bc3..0000000 --- a/sql/import_mapping/schema.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TYPE mapping_area AS ENUM ('ticket', 'form', 'form_input', 'panel'); - -CREATE TABLE IF NOT EXISTS import_mapping -( - guild_id int8 NOT NULL, - area mapping_area NOT NULL, - source_id int4 NOT NULL, - target_id int4 NOT NULL, - UNIQUE NULLS NOT DISTINCT (guild_id, area, source_id, target_id) -); \ No newline at end of file diff --git a/sql/import_mapping/set.sql b/sql/import_mapping/set.sql deleted file mode 100644 index 1cf9d98..0000000 --- a/sql/import_mapping/set.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO import_mapping (guild_id, area, source_id, target_id) -VALUES ($1, $2, $3, $4) -ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/ticketclaims.go b/ticketclaims.go index 128f0bd..66bd4ec 100644 --- a/ticketclaims.go +++ b/ticketclaims.go @@ -39,17 +39,6 @@ func (c *TicketClaims) Get(ctx context.Context, guildId uint64, ticketId int) (u return } -func (c *TicketClaims) ImportBulk(ctx context.Context, guildId uint64, claims map[int]uint64) (err error) { - rows := make([][]interface{}, 0) - - for ticketId, userId := range claims { - rows = append(rows, []interface{}{guildId, ticketId, userId}) - } - - _, err = c.CopyFrom(ctx, pgx.Identifier{"ticket_claims"}, []string{"guild_id", "ticket_id", "user_id"}, pgx.CopyFromRows(rows)) - return -} - func (c *TicketClaims) Set(ctx context.Context, guildId uint64, ticketId int, userId uint64) (err error) { query := `INSERT INTO ticket_claims("guild_id", "ticket_id", "user_id") VALUES($1, $2, $3) ON CONFLICT("guild_id", "ticket_id") DO UPDATE SET "user_id" = $3;` _, err = c.Exec(ctx, query, guildId, ticketId, userId) diff --git a/ticketlastmessage.go b/ticketlastmessage.go index eabae60..0e47479 100644 --- a/ticketlastmessage.go +++ b/ticketlastmessage.go @@ -58,24 +58,6 @@ WHERE "guild_id" = $1 AND "ticket_id" = $2;` return } -func (m *TicketLastMessageTable) ImportBulk(ctx context.Context, guildId uint64, lastMessages map[int]TicketLastMessage) (err error) { - rows := make([][]interface{}, 0) - - for i, msg := range lastMessages { - rows = append(rows, []interface{}{ - guildId, - i, - msg.LastMessageId, - msg.LastMessageTime, - msg.UserId, - msg.UserIsStaff, - }) - } - - _, err = m.CopyFrom(ctx, pgx.Identifier{"ticket_last_message"}, []string{"guild_id", "ticket_id", "last_message_id", "last_message_time", "user_id", "user_is_staff"}, pgx.CopyFromRows(rows)) - return -} - func (m *TicketLastMessageTable) Set(ctx context.Context, guildId uint64, ticketId int, messageId, userId uint64, userIsStaff bool) (err error) { query := ` INSERT INTO ticket_last_message("guild_id", "ticket_id", "last_message_id", "last_message_time", "user_id", "user_is_staff") diff --git a/ticketmembers.go b/ticketmembers.go index 9d6af30..25cedc6 100644 --- a/ticketmembers.go +++ b/ticketmembers.go @@ -53,20 +53,6 @@ func (m *TicketMembers) Get(ctx context.Context, guildId uint64, ticketId int) ( return } -func (m *TicketMembers) ImportBulk(ctx context.Context, guildId uint64, ticketUsers map[int][]uint64) (err error) { - rows := make([][]interface{}, 0) - - for ticketId, users := range ticketUsers { - for _, userId := range users { - rows = append(rows, []interface{}{guildId, ticketId, userId}) - } - } - - _, err = m.CopyFrom(ctx, pgx.Identifier{"ticket_members"}, []string{"guild_id", "ticket_id", "user_id"}, pgx.CopyFromRows(rows)) - - return -} - func (m *TicketMembers) Add(ctx context.Context, guildId uint64, ticketId int, userId uint64) (err error) { query := `INSERT INTO ticket_members("guild_id", "ticket_id", "user_id") VALUES($1, $2, $3) ON CONFLICT("guild_id", "ticket_id", "user_id") DO NOTHING;` _, err = m.Exec(ctx, query, guildId, ticketId, userId) diff --git a/tickets.go b/tickets.go index 032964d..0d196c5 100644 --- a/tickets.go +++ b/tickets.go @@ -103,38 +103,6 @@ CREATE INDEX IF NOT EXISTS tickets_panel_id ON tickets("panel_id"); ` } -func (t *TicketTable) BulkImport(ctx context.Context, guildId uint64, tickets []Ticket) (err error) { - rows := make([][]interface{}, len(tickets)) - - for i, ticket := range tickets { - rows[i] = []interface{}{ - ticket.Id, - guildId, - ticket.ChannelId, - ticket.UserId, - ticket.Open, - ticket.OpenTime, - ticket.WelcomeMessageId, - ticket.PanelId, - ticket.HasTranscript, - ticket.CloseTime, - ticket.IsThread, - ticket.JoinMessageId, - ticket.NotesThreadId, - "CLOSED", - } - } - - _, err = t.CopyFrom( - ctx, - pgx.Identifier{"tickets"}, - []string{"id", "guild_id", "channel_id", "user_id", "open", "open_time", "welcome_message_id", "panel_id", "has_transcript", "close_time", "is_thread", "join_message_id", "notes_thread_id", "status"}, - pgx.CopyFromRows(rows), - ) - - return -} - func (t *TicketTable) Create(ctx context.Context, guildId, userId uint64, isThread bool, panelId *int) (id int, err error) { tx, err := t.Begin(ctx) if err != nil {