refact: migrate old errors save in maria to mongo for gdpr + remove table

Author: linventif

bun.lock

@@ -41,30 +41,30 @@
     "@types/node-cron": "^3.0.11",
     "@types/steamapi": "^2.2.5",
     "@types/uuid": "^10.0.0",
-    "prettier": "^3.5.3",
-    "prisma": "^6.9.0",
+    "prettier": "^3.6.1",
+    "prisma": "^6.10.1",
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",
     "typescript": "^5.8.3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.828.0",
+    "@aws-sdk/client-s3": "^3.837.0",
     "@discordjs/builders": "^1.11.2",
-    "@prisma/client": "^6.9.0",
+    "@prisma/client": "^6.10.1",
     "@sentry/cli": "^2.46.0",
     "@sentry/core": "^8.55.0",
     "@sentry/node": "^8.55.0",
     "@sentry/profiling-node": "^8.55.0",
     "@types/d3": "^7.4.3",
     "@types/jsdom": "^21.1.7",
-    "@types/node": "^22.15.32",
+    "@types/node": "^22.15.33",
     "archiver": "^7.0.1",
     "axios": "^1.10.0",
     "body-parser": "^1.20.3",
     "cors": "^2.8.5",
     "d3": "^7.9.0",
     "discord-api-types": "^0.37.120",
-    "discord.js": "^14.19.3",
+    "discord.js": "^14.21.0",
     "dotenv": "^16.5.0",
     "express": "^4.21.2",
     "express-useragent": "^1.0.15",
@@ -85,7 +85,7 @@
     "unicode-emoji": "^2.6.0",
     "uuid": "^9.0.1",
     "ws": "^8.18.2",
-    "zod": "^3.25.64",
+    "zod": "^3.25.67",
     "zod-openapi": "^4.2.4"
   }
 }

package.json

@@ -0,0 +1,26 @@
+/*
+  Warnings:
+
+  - You are about to drop the `gm_server_errors` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropForeignKey
+ALTER TABLE `gm_server_errors` DROP FOREIGN KEY `gm_server_errors_ibfk_1`;
+
+-- AlterTable
+ALTER TABLE `banUsers` MODIFY `unbanDate` DATETIME(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP + INTERVAL 2 YEAR);
+
+-- AlterTable
+ALTER TABLE `gm_discordToken` MODIFY `expirationDate` DATETIME(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP + INTERVAL 7 DAY);
+
+-- AlterTable
+ALTER TABLE `gm_panelToken` MODIFY `expirationDate` DATETIME(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP + INTERVAL 7 DAY);
+
+-- AlterTable
+ALTER TABLE `gm_user` MODIFY `token_expires` DATETIME(3) NULL DEFAULT (CURRENT_TIMESTAMP + INTERVAL 7 DAY);
+
+-- AlterTable
+ALTER TABLE `gm_users_data_request` MODIFY `expirationDate` DATETIME(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP + INTERVAL 1 DAY);
+
+-- DropTable
+DROP TABLE `gm_server_errors`;

prisma/migrations/20250626012710_remove/migration.sql

@@ -160,7 +160,6 @@ model gm_server {
   description                   String?                         @default("") @db.VarChar(255)
   isPublic                      Boolean                         @default(false)
   gm_guild                      gm_guild                        @relation(fields: [guild], references: [guild], onDelete: Cascade, map: "gm_server_ibfk_1")
-  gm_server_errors              gm_server_errors[]
   gm_server_leaderboard_options gm_server_leaderboard_options[]
   gm_server_logs_channel        gm_server_logs_channel?
   gm_server_pseudo              gm_server_pseudo[]
@@ -199,24 +198,6 @@ model gm_server_customValues {
   updatedAt DateTime @updatedAt
 }
 
-model gm_server_errors {
-  id         Int       @id @default(autoincrement())
-  serverID   String    @db.VarChar(255)
-  count      Int
-  realm      String    @db.VarChar(255)
-  error      String    @db.Text
-  stack      String    @db.Text
-  name       String    @db.VarChar(255)
-  steamID64  String    @default("") @db.VarChar(255)
-  workshopID String    @default("") @db.VarChar(255)
-  uptime     Int
-  createdAt  DateTime  @default(now())
-  updatedAt  DateTime  @updatedAt
-  gm_server  gm_server @relation(fields: [serverID], references: [id], onDelete: Cascade, map: "gm_server_errors_ibfk_1")
-
-  @@index([serverID], map: "serverID")
-}
-
 model gm_server_leaderboard_options {
   serverID     String    @db.Char(17)
   messageID    String    @db.Char(255)

prisma/schema.prisma

@@ -50,12 +50,36 @@ export class GmodErrors {
   }
 }
 
-export async function getErrorsCountByServer(serverID: string): Promise<number> {
+export async function getErrorsCountByServer(serverID: string) {
   return await collection.countDocuments({
     serverID,
   });
 }
 
+export async function getErrorsCountBySteamID(steamID64: string) {
+  return await collection.countDocuments({
+    steamID64,
+  });
+}
+
+export async function getErrorsBySteamID(steamID64: string, query: Query) {
+  return {
+    errors: await collection
+      .find({
+        steamID64,
+      })
+      .limit(query.limit)
+      .skip(query.offset)
+      .toArray(),
+    query: {
+      ...query,
+      total: await collection.countDocuments({
+        steamID64,
+      }),
+    },
+  };
+}
+
 export async function getErrorsByServer(query: Query, serverID: string) {
   return {
     errors: await collection

src/classes/gmod/GmodErrors.ts

@@ -1,7 +1,8 @@
 import { mongoClient } from '../services/mongo/index.js';
 
 const db = mongoClient.db('gmod_integration');
-const collection = db.collection('logs');
+const collectionLogs = db.collection('logs');
+const collectionErrors = db.collection('errors');
 
 interface Log {
   serverID: string;
@@ -13,11 +14,11 @@ interface Log {
 }
 
 export async function addLog(log: Log) {
-  await collection.insertOne(log);
+  await collectionLogs.insertOne(log);
 }
 
 export async function getLogsCountBySteamIDList(SteamIDS: string[]) {
-  return await collection.countDocuments({
+  return await collectionLogs.countDocuments({
     playerInvolvedSteamID64: {
       $in: SteamIDS,
     },
@@ -31,7 +32,7 @@ export async function getLogsBySteamIDList(
     offset: number | 0;
   },
 ) {
-  return await collection
+  return await collectionLogs
     .find({
       playerInvolvedSteamID64: {
         $in: SteamIDS,
@@ -43,7 +44,7 @@ export async function getLogsBySteamIDList(
 }
 
 export async function getTotalLogsByServer(serverID: string) {
-  return await collection.countDocuments({
+  return await collectionLogs.countDocuments({
     serverID,
   });
 }
@@ -62,7 +63,7 @@ export async function getLogsByServer(
     sort: string;
   },
 ) {
-  return await collection
+  return await collectionLogs
     .find({
       serverID,
     })

src/database/gm_server_logs.ts

@@ -11,6 +11,7 @@ import { createBucketIfNotExists, s3 } from '../../services/minio/index.js';
 import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
 import prisma from '../../services/prisma/index.js';
 import { Readable } from 'node:stream';
+import { getErrorsBySteamID, getErrorsCountBySteamID } from '../../classes/gmod/GmodErrors.js';
 
 export async function getUserDataGRPD(user: User) {
   const discordID = user.getDiscordID();
@@ -124,45 +125,30 @@ export async function getUserDataGRPD(user: User) {
       console.error('Error fetching server logs:', error);
     }
 
-    // Error
+    //  Error logs
     try {
-      const plyErrorsCount = await prisma.gm_server_errors.count({
-        where: {
-          steamID64,
-        },
-      });
-
+      const serverErrorsCount = await getErrorsCountBySteamID(steamID64);
       const limit = 1000;
       let offset = 0;
       let fileIndex = 1;
-      const totalFiles = Math.ceil(plyErrorsCount / limit);
-
-      while (offset < plyErrorsCount) {
-        const errorLogs = await prisma.gm_server_errors.findMany({
-          where: {
-            steamID64,
-          },
-          take: limit,
-          skip: offset,
-        });
-
+      const totalFiles = Math.ceil(serverErrorsCount / limit);
+      while (offset < serverErrorsCount) {
+        const errors = await getErrorsBySteamID(steamID64, { limit, offset });
         const stream = fs.createWriteStream(`${tempPath}/steam-errors-${fileIndex}-${totalFiles}.json`);
 
         stream.write('[');
-        for (let i = 0; i < errorLogs.length; i++) {
+        for (let i = 0; i < errors.errors.length; i++) {
           if (i !== 0) stream.write(',');
-          stream.write(JSON.stringify(errorLogs[i]));
+          stream.write(JSON.stringify(errors.errors[i]));
         }
         stream.write(']');
         stream.end();
-
         await new Promise<void>((resolve) => stream.on('finish', () => resolve()));
-
         offset += limit;
         fileIndex++;
       }
     } catch (error) {
-      console.error('Error fetching error logs:', error);
+      console.error('Error fetching server logs:', error);
     }
 
     // Screenshots