From 29ab5b1ae156f1a37dee55eb5bdff340ff663c43 Mon Sep 17 00:00:00 2001 From: biast12 Date: Thu, 29 Jan 2026 22:31:32 +0100 Subject: [PATCH] RM-239: Remove the entire import pipeline Remove the entire import pipeline and related S3 integration. Deleted backend import API and validator code (app/http/endpoints/api/export and validator files), removed s3 client (s3/s3.go) and S3 import config fields, and stopped connecting to import S3 in cmd/api/main.go. Server routes for import were removed from app/http/server.go. Also removed frontend import UI and routes (Import.svelte, ImportModal.svelte, route and sidebar/navbar links) and updated related components. This strips out the import feature and its external S3 dependency. --- app/http/endpoints/api/export/import.go | 174 --------- app/http/endpoints/api/export/model.go | 1 - .../endpoints/api/export/validator/error.go | 8 - .../api/export/validator/guild_data.go | 37 -- .../api/export/validator/guild_transcripts.go | 97 ----- .../endpoints/api/export/validator/model.go | 83 ---- .../api/export/validator/signatures.go | 52 --- .../api/export/validator/validator.go | 45 --- app/http/server.go | 6 - cmd/api/main.go | 4 - config/config.go | 8 - .../src/components/manage/ImportModal.svelte | 163 -------- frontend/src/includes/ManageSidebar.svelte | 12 - frontend/src/includes/Navbar.svelte | 17 +- frontend/src/routes.js | 3 - frontend/src/views/Import.svelte | 357 ------------------ s3/s3.go | 23 -- 17 files changed, 8 insertions(+), 1082 deletions(-) delete mode 100644 app/http/endpoints/api/export/import.go delete mode 100644 app/http/endpoints/api/export/model.go delete mode 100644 app/http/endpoints/api/export/validator/error.go delete mode 100644 app/http/endpoints/api/export/validator/guild_data.go delete mode 100644 app/http/endpoints/api/export/validator/guild_transcripts.go delete mode 100644 app/http/endpoints/api/export/validator/model.go delete mode 100644 app/http/endpoints/api/export/validator/signatures.go delete mode 100644 app/http/endpoints/api/export/validator/validator.go delete mode 100644 frontend/src/components/manage/ImportModal.svelte delete mode 100644 frontend/src/views/Import.svelte delete mode 100644 s3/s3.go diff --git a/app/http/endpoints/api/export/import.go b/app/http/endpoints/api/export/import.go deleted file mode 100644 index 5c6389a..0000000 --- a/app/http/endpoints/api/export/import.go +++ /dev/null @@ -1,174 +0,0 @@ -package api - -import ( - "context" - "fmt" - "net/http" - "net/url" - "strconv" - "time" - - "github.com/TicketsBot-cloud/common/permission" - "github.com/TicketsBot-cloud/dashboard/app" - "github.com/TicketsBot-cloud/dashboard/botcontext" - "github.com/TicketsBot-cloud/dashboard/config" - dbclient "github.com/TicketsBot-cloud/dashboard/database" - "github.com/TicketsBot-cloud/dashboard/s3" - "github.com/TicketsBot-cloud/dashboard/utils" - "github.com/gin-gonic/gin" - "github.com/minio/minio-go/v7" -) - -// func ImportHandler(ctx *gin.Context) { -// ctx.JSON(401, "This endpoint is disabled") -// } - -func CurrentQueue(ctx *gin.Context) { - guildId, userId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64) - var ( - dataBucket = config.Conf.S3Import.DataBucket - transcriptsBucket = config.Conf.S3Import.TranscriptBucket - opts = minio.ListObjectsOptions{ - Prefix: "", - Recursive: true, - } - - dataCount int - transcriptsCount int - ) - - permissionLevel, err := utils.GetPermissionLevel(ctx, guildId, userId) - if err != nil { - _ = ctx.AbortWithError(http.StatusInternalServerError, app.NewError(err, "Failed to process request")) - return - } - - if permissionLevel < permission.Admin { - ctx.JSON(403, utils.ErrorStr("You do not have permission to view import queue")) - return - } - - dataCh := s3.S3Client.ListObjects(ctx, dataBucket, opts) - transcriptsCh := s3.S3Client.ListObjects(ctx, transcriptsBucket, opts) - - for range dataCh { - dataCount++ - } - - for range transcriptsCh { - transcriptsCount++ - } - - ctx.JSON(200, gin.H{ - "data": dataCount, - "transcripts": transcriptsCount, - }) -} - -func PresignURL(ctx *gin.Context) { - guildId, userId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64) - - file_type := ctx.Query("file_type") - - bucketName := "" - - if file_type == "data" { - bucketName = config.Conf.S3Import.DataBucket - } - - if file_type == "transcripts" { - bucketName = config.Conf.S3Import.TranscriptBucket - } - - if bucketName == "" { - ctx.JSON(400, utils.ErrorStr("Invalid file type")) - return - } - - // Get "file_size" query parameter - fileSize, err := strconv.ParseInt(ctx.Query("file_size"), 10, 64) - if err != nil { - ctx.JSON(400, utils.ErrorStr("Failed to process request. Please try again.")) - return - } - - fileContentType := ctx.Query("file_content_type") - - if fileContentType == "" { - ctx.JSON(400, utils.ErrorStr("Missing file_content_type")) - return - } - - if fileContentType != "application/zip" && fileContentType != "application/x-zip-compressed" { - ctx.JSON(400, utils.ErrorStr("Invalid file_content_type")) - return - } - - // Check if file is over 1GB - if fileSize > 1024*1024*1024 { - ctx.JSON(400, utils.ErrorStr("File size too large")) - return - } - - botCtx, err := botcontext.ContextForGuild(guildId) - if err != nil { - ctx.JSON(500, utils.ErrorStr("Unable to connect to Discord. Please try again later.")) - return - } - - guild, err := botCtx.GetGuild(context.Background(), guildId) - if err != nil { - ctx.JSON(500, utils.ErrorStr("Failed to process request. Please try again.")) - return - } - - if guild.OwnerId != userId && !botCtx.IsBotAdmin(ctx, userId) { - ctx.JSON(403, utils.ErrorStr("Only the server owner can import %s", file_type)) - return - } - - // Presign URL - url, err := s3.S3Client.PresignHeader(ctx, "PUT", bucketName, fmt.Sprintf("%s/%d.zip", file_type, guildId), time.Minute*10, url.Values{}, http.Header{ - "Content-Type": []string{fileContentType}, - }) - if err != nil { - ctx.JSON(500, utils.ErrorStr("Failed to process request. Please try again.")) - return - } - - ctx.JSON(200, gin.H{ - "url": url.String(), - }) -} - -func GetRuns(ctx *gin.Context) { - guildId, userId := ctx.Keys["guildid"].(uint64), ctx.Keys["userid"].(uint64) - - permissionLevel, err := utils.GetPermissionLevel(ctx, guildId, userId) - if err != nil { - _ = ctx.AbortWithError(http.StatusInternalServerError, app.NewError(err, "Failed to process request")) - return - } - - if permissionLevel < permission.Admin { - ctx.JSON(403, utils.ErrorStr("You do not have permission to view import logs")) - return - } - - runs, err := dbclient.Client.ImportLogs.GetRuns(ctx, guildId) - if err != nil { - ctx.JSON(500, utils.ErrorStr("Failed to process request. Please try again.")) - return - } - - if len(runs) == 0 { - ctx.JSON(200, []interface{}{}) - return - } - - ctx.JSON(200, runs) -} - -func ImportHandler(ctx *gin.Context) { - ctx.JSON(401, "Imports are currently disabled - Please try again later (~24 hours)") -} diff --git a/app/http/endpoints/api/export/model.go b/app/http/endpoints/api/export/model.go deleted file mode 100644 index 778f64e..0000000 --- a/app/http/endpoints/api/export/model.go +++ /dev/null @@ -1 +0,0 @@ -package api diff --git a/app/http/endpoints/api/export/validator/error.go b/app/http/endpoints/api/export/validator/error.go deleted file mode 100644 index 6203ed7..0000000 --- a/app/http/endpoints/api/export/validator/error.go +++ /dev/null @@ -1,8 +0,0 @@ -package validator - -import "errors" - -var ( - ErrValidationFailed = errors.New("validation failed") - ErrMaximumSizeExceeded = errors.New("maximum size exceeded") -) diff --git a/app/http/endpoints/api/export/validator/guild_data.go b/app/http/endpoints/api/export/validator/guild_data.go deleted file mode 100644 index 917f738..0000000 --- a/app/http/endpoints/api/export/validator/guild_data.go +++ /dev/null @@ -1,37 +0,0 @@ -package validator - -import ( - "archive/zip" - "encoding/json" - "io" -) - -func (v *Validator) ValidateGuildData(input io.ReaderAt, size int64) (*GuildData, error) { - reader, err := zip.NewReader(input, size) - if err != nil { - return nil, err - } - - f, err := reader.Open("data.json") - if err != nil { - return nil, err - } - - defer f.Close() - - data, err := io.ReadAll(v.newLimitReader(f)) - if err != nil { - return nil, err - } - - if _, err := v.validateSignature(reader, "data.json", data); err != nil { - return nil, err - } - - var guildData GuildData - if err := json.Unmarshal(data, &guildData); err != nil { - return nil, err - } - - return &guildData, nil -} diff --git a/app/http/endpoints/api/export/validator/guild_transcripts.go b/app/http/endpoints/api/export/validator/guild_transcripts.go deleted file mode 100644 index a4dab57..0000000 --- a/app/http/endpoints/api/export/validator/guild_transcripts.go +++ /dev/null @@ -1,97 +0,0 @@ -package validator - -import ( - "archive/zip" - "io" - "regexp" - "strconv" -) - -type GuildTranscriptsOutput struct { - GuildId uint64 - // Ticket ID -> Transcript - Transcripts map[int][]byte -} - -var transcriptFileRegex = regexp.MustCompile(`^transcripts/(\d+)\.json$`) - -func (v *Validator) ValidateGuildTranscripts(input io.ReaderAt, size int64) (*GuildTranscriptsOutput, error) { - reader, err := zip.NewReader(input, size) - if err != nil { - return nil, err - } - - guildId, n, err := v.readGuildId(reader) - if err != nil { - return nil, err - } - - transcripts := make(map[int][]byte) - for _, f := range reader.File { - matches := transcriptFileRegex.FindStringSubmatch(f.Name) - if len(matches) != 2 { - continue - } - - ticketId, err := strconv.Atoi(matches[1]) - if err != nil { - continue - } - - file, err := f.Open() - if err != nil { - return nil, err - } - - b, err := io.ReadAll(v.newLimitReader(file)) - if err != nil { - return nil, err - } - - n += int64(len(b)) - if n > v.maxUncompressedSize { - return nil, ErrMaximumSizeExceeded - } - - sigSize, err := v.validateTranscriptSignature(reader, f.Name, guildId, ticketId, b) - if err != nil { - return nil, err - } - - n += sigSize - if n > v.maxUncompressedSize { - return nil, ErrMaximumSizeExceeded - } - - transcripts[ticketId] = b - } - - return &GuildTranscriptsOutput{ - GuildId: guildId, - Transcripts: transcripts, - }, nil -} - -func (v *Validator) readGuildId(reader *zip.Reader) (uint64, int64, error) { - f, err := reader.Open("guild_id.txt") - if err != nil { - return 0, 0, err - } - - b, err := io.ReadAll(v.newLimitReader(f)) - if err != nil { - return 0, 0, err - } - - guildId, err := strconv.ParseUint(string(b), 10, 64) - if err != nil { - return 0, 0, err - } - - sigSize, err := v.validateSignature(reader, "guild_id.txt", b) - if err != nil { - return 0, 0, err - } - - return guildId, int64(len(b)) + sigSize, nil -} diff --git a/app/http/endpoints/api/export/validator/model.go b/app/http/endpoints/api/export/validator/model.go deleted file mode 100644 index ca69fce..0000000 --- a/app/http/endpoints/api/export/validator/model.go +++ /dev/null @@ -1,83 +0,0 @@ -package validator - -import ( - "time" - - "github.com/TicketsBot-cloud/database" -) - -type TicketUnion[T any] struct { - TicketId int `json:"ticket_id"` - Data T `json:"data"` -} - -type GuildData struct { - GuildId uint64 `json:"guild_id,string"` - ActiveLanguage *string `json:"active_language"` - ArchiveChannel *uint64 `json:"archive_channel,string"` - ArchiveMessages []TicketUnion[database.ArchiveMessage] `json:"archive_messages"` - AutocloseSettings *database.AutoCloseSettings `json:"autoclose_settings"` - AutocloseExcluded []int `json:"autoclose_excluded"` // ticket IDs - GuildBlacklistedUsers []uint64 `json:"guild_blacklisted_users"` - ChannelCategory *uint64 `json:"channel_category,string"` - ClaimSettings *database.ClaimSettings `json:"claim_settings"` - CloseConfirmationEnabled bool `json:"close_confirmation_enabled"` - CloseReasons []TicketUnion[database.CloseMetadata] `json:"close_reasons"` - CustomColors map[int16]int `json:"custom_colors"` - EmbedFields []database.EmbedField `json:"embed_fields"` - Embeds []database.CustomEmbed `json:"embeds"` - ExitSurveyResponses []TicketUnion[ExitSurveyResponse] `json:"exit_survey_responses"` - FeedbackEnabled bool `json:"feedback_enabled"` - FirstResponseTimes []FirstResponseTime `json:"first_response_times"` - FormInputs []database.FormInput `json:"form_inputs"` - Forms []database.Form `json:"forms"` - GuildIsGloballyBlacklisted bool `json:"guild_is_globally_blacklisted"` - GuildMetadata database.GuildMetadata `json:"guild_metadata"` - MultiPanels []database.MultiPanel `json:"multi_panels"` - MultiPanelTargets map[int][]int `json:"multi_panel_targets"` // multi_panel_id -> [panel_ids] - NamingScheme *database.NamingScheme `json:"naming_scheme"` - OnCallUsers []uint64 `json:"on_call_users"` - PanelAccessControlRules map[int][]database.PanelAccessControlRule `json:"panel_access_control_rules"` // panel_id -> rules - PanelMentionUser map[int]bool `json:"panel_mention_user"` - PanelRoleMentions map[int][]uint64 `json:"panel_role_mentions"` - Panels []database.Panel `json:"panels"` - PanelTeams map[int][]int `json:"panel_teams"` // panel_id -> [team_ids] - Participants map[int][]uint64 `json:"participants"` // ticket_id -> [user_ids] - UserPermissions []Permission `json:"user_permissions"` - GuildBlacklistedRoles []uint64 `json:"guild_blacklisted_roles"` - RolePermissions []Permission `json:"role_permissions"` - ServiceRatings []TicketUnion[int16] `json:"service_ratings"` - Settings database.Settings `json:"settings"` - SupportTeamUsers map[int][]uint64 `json:"support_team_users"` // team_id -> [user_ids] - SupportTeamRoles map[int][]uint64 `json:"support_team_roles"` // team_id -> [role_ids] - SupportTeams []database.SupportTeam `json:"support_teams"` - Tags []database.Tag `json:"tags"` - TicketClaims []TicketUnion[uint64] `json:"ticket_claims"` - TicketLastMessages []TicketUnion[database.TicketLastMessage] `json:"ticket_last_messages"` - TicketLimit *int `json:"ticket_limit"` - TicketAdditionalMembers map[int][]uint64 `json:"ticket_additional_members"` // ticket_id -> [user_ids] - TicketPermissions database.TicketPermissions `json:"ticket_permissions"` - Tickets []database.Ticket `json:"tickets"` - UsersCanClose bool `json:"users_can_close"` - WelcomeMessage *string `json:"welcome_message"` -} - -// Shims - -type FirstResponseTime struct { - TicketId int `json:"ticket_id"` - UserId uint64 `json:"user_id,string"` - ResponseTime time.Duration `json:"response_time"` -} - -type Permission struct { - Snowflake uint64 `json:"snowflake,string"` - IsSupport bool `json:"is_support"` - IsAdmin bool `json:"is_admin"` -} - -type ExitSurveyResponse struct { - FormId *int `json:"form_id"` - QuestionId *int `json:"question_id"` - Response *string `json:"response"` -} diff --git a/app/http/endpoints/api/export/validator/signatures.go b/app/http/endpoints/api/export/validator/signatures.go deleted file mode 100644 index 16fed66..0000000 --- a/app/http/endpoints/api/export/validator/signatures.go +++ /dev/null @@ -1,52 +0,0 @@ -package validator - -import ( - "archive/zip" - "crypto/ed25519" - "encoding/base64" - "io" - "strconv" -) - -func (v *Validator) validateSignature(zipReader *zip.Reader, fileName string, data []byte) (int64, error) { - f, err := zipReader.Open(fileName + ".sig") - if err != nil { - return 0, err - } - - signature, err := io.ReadAll(v.newLimitReader(f)) - if err != nil { - return 0, err - } - - decoded, err := base64.RawURLEncoding.DecodeString(string(signature)) - if err != nil { - return 0, err - } - - if !ed25519.Verify(v.publicKey, data, decoded) { - return 0, ErrValidationFailed - } - - return int64(len(signature)), nil -} - -func (v *Validator) validateTranscriptSignature( - zipReader *zip.Reader, - fileName string, - guildId uint64, - ticketId int, - data []byte, -) (int64, error) { - guildIdStr := strconv.FormatUint(guildId, 10) - ticketIdStr := strconv.Itoa(ticketId) - - sigData := make([]byte, 0, len(guildIdStr)+len(ticketIdStr)+len(data)+2) - sigData = append(sigData, guildIdStr...) - sigData = append(sigData, '|') - sigData = append(sigData, ticketIdStr...) - sigData = append(sigData, '|') - sigData = append(sigData, data...) - - return v.validateSignature(zipReader, fileName, sigData) -} diff --git a/app/http/endpoints/api/export/validator/validator.go b/app/http/endpoints/api/export/validator/validator.go deleted file mode 100644 index 183f08a..0000000 --- a/app/http/endpoints/api/export/validator/validator.go +++ /dev/null @@ -1,45 +0,0 @@ -package validator - -import ( - "crypto/ed25519" - "io" -) - -type Validator struct { - publicKey ed25519.PublicKey - - maxUncompressedSize int64 - maxIndividualFileSize int64 -} - -type Option func(*Validator) - -func NewValidator(publicKey ed25519.PublicKey, options ...Option) *Validator { - v := &Validator{ - publicKey: publicKey, - maxUncompressedSize: 250 * 1024 * 1024, - maxIndividualFileSize: 1 * 1024 * 1024, - } - - for _, option := range options { - option(v) - } - - return v -} - -func WithMaxUncompressedSize(size int64) Option { - return func(v *Validator) { - v.maxUncompressedSize = size - } -} - -func WithMaxIndividualFileSize(size int64) Option { - return func(v *Validator) { - v.maxIndividualFileSize = size - } -} - -func (v *Validator) newLimitReader(r io.Reader) io.Reader { - return io.LimitReader(r, v.maxIndividualFileSize) -} diff --git a/app/http/server.go b/app/http/server.go index c86ebea..c17bc95 100644 --- a/app/http/server.go +++ b/app/http/server.go @@ -7,7 +7,6 @@ import ( "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api" "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/admin/botstaff" api_blacklist "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/blacklist" - api_import "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/export" api_forms "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/forms" api_integrations "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/integrations" api_panels "github.com/TicketsBot-cloud/dashboard/app/http/endpoints/api/panel" @@ -124,11 +123,6 @@ func StartServer(logger *zap.Logger, sm *livechat.SocketManager) { guildAuthApiSupport.GET("/settings", api_settings.GetSettingsHandler) guildAuthApiAdmin.POST("/settings", api_settings.UpdateSettingsHandler) - guildAuthApiAdmin.POST("/import", api_import.ImportHandler) - guildAuthApiAdmin.GET("/import/runs", api_import.GetRuns) - guildAuthApiAdmin.GET("/import/presign", api_import.PresignURL) - guildAuthApiAdmin.GET("/import/queue", api_import.CurrentQueue) - guildAuthApiSupport.GET("/blacklist", api_blacklist.GetBlacklistHandler) guildAuthApiSupport.POST("/blacklist", api_blacklist.AddBlacklistHandler) guildAuthApiSupport.DELETE("/blacklist/user/:user", api_blacklist.RemoveUserBlacklistHandler) diff --git a/cmd/api/main.go b/cmd/api/main.go index af0a241..f632bde 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -19,7 +19,6 @@ import ( "github.com/TicketsBot-cloud/dashboard/redis" "github.com/TicketsBot-cloud/dashboard/rpc" "github.com/TicketsBot-cloud/dashboard/rpc/cache" - "github.com/TicketsBot-cloud/dashboard/s3" "github.com/TicketsBot-cloud/dashboard/utils" "github.com/TicketsBot-cloud/gdl/rest/request" "github.com/TicketsBot-cloud/worker/i18n" @@ -83,9 +82,6 @@ func main() { logger.Info("Connecting to cache") cache.Instance = cache.NewCache() - logger.Info("Connecting to import S3") - s3.ConnectS3(config.Conf.S3Import.Endpoint, config.Conf.S3Import.AccessKey, config.Conf.S3Import.SecretKey, config.Conf.S3Import.Secure) - logger.Info("Initialising microservice clients") utils.ArchiverClient = archiverclient.NewArchiverClient(archiverclient.NewProxyRetriever(config.Conf.Bot.ObjectStore), []byte(config.Conf.Bot.AesKey)) utils.SecureProxyClient = secureproxy.NewSecureProxy(config.Conf.SecureProxyUrl) diff --git a/config/config.go b/config/config.go index 3c831ab..f760556 100644 --- a/config/config.go +++ b/config/config.go @@ -60,14 +60,6 @@ type Config struct { Uri string `env:"URI,required"` } `envPrefix:"CACHE_"` SecureProxyUrl string `env:"SECURE_PROXY_URL"` - S3Import struct { - Endpoint string `env:"ENDPOINT,required"` - Secure bool `env:"SECURE" envDefault:"true"` - AccessKey string `env:"ACCESS_KEY,required"` - SecretKey string `env:"SECRET_KEY,required"` - TranscriptBucket string `env:"TRANSCRIPT_BUCKET,required"` - DataBucket string `env:"DATA_BUCKET,required"` - } `envPrefix:"S3_IMPORT_"` } // TODO: Don't use a global variable diff --git a/frontend/src/components/manage/ImportModal.svelte b/frontend/src/components/manage/ImportModal.svelte deleted file mode 100644 index c8ee5f3..0000000 --- a/frontend/src/components/manage/ImportModal.svelte +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/frontend/src/includes/ManageSidebar.svelte b/frontend/src/includes/ManageSidebar.svelte index ea4b411..26b4f90 100644 --- a/frontend/src/includes/ManageSidebar.svelte +++ b/frontend/src/includes/ManageSidebar.svelte @@ -9,14 +9,11 @@ import SubNavigation from "./SubNavigation.svelte"; import SubNavigationLink from "./SubNavigationLink.svelte"; - import ImportModal from "../components/manage/ImportModal.svelte"; import ManageSidebarServersLink from "./ManageSidebarServersLink.svelte"; export let currentRoute; export let permissionLevel; - let importModal = false; - $: isAdmin = permissionLevel >= 2; $: isMod = permissionLevel >= 1; $: isUser = permissionLevel >= 0; @@ -81,9 +78,6 @@ }); -{#if importModal} - (importModal = false)} /> -{/if}