♻️ Refactor moderation code into a feature (#1614)

apps/web/src/features/moderation/index.ts

@@ -0,0 +1,86 @@
+import { env } from "@/env";
+
+import { moderateContentWithAI } from "./libs/ai-moderation";
+import { containsSuspiciousPatterns } from "./libs/pattern-moderation";
+
+/**
+ * Log the moderation status at initialization
+ * This function is automatically called when this module is imported
+ */
+function initModeration() {
+  if (env.MODERATION_ENABLED === "true") {
+    if (env.OPENAI_API_KEY) {
+      console.info("✅ Content moderation is ENABLED with AI support");
+    } else {
+      console.info(
+        "⚠️ Content moderation is ENABLED but missing OPENAI_API_KEY - AI moderation will be skipped",
+      );
+    }
+  } else {
+    console.info("ℹ️ Content moderation is DISABLED");
+  }
+}
+
+// Initialize moderation and log status
+initModeration();
+
+/**
+ * Moderates content to detect spam, inappropriate content, or abuse
+ * Uses a two-layer approach:
+ * 1. Pattern-based detection for common spam patterns
+ * 2. AI-based moderation for more sophisticated content analysis
+ *
+ * @param content Array of strings to moderate (can include undefined values which will be filtered out)
+ * @returns True if the content is flagged as inappropriate, false otherwise
+ */
+export async function moderateContent(content: Array<string | undefined>) {
+  // Skip moderation if the feature is disabled in environment
+  if (env.MODERATION_ENABLED !== "true") {
+    return false;
+  }
+
+  // Check if OpenAI API key is available
+  if (!env.OPENAI_API_KEY) {
+    console.warn(
+      "Content moderation is enabled but OPENAI_API_KEY is not set. AI-based moderation will be skipped.",
+    );
+    return false;
+  }
+
+  const textToModerate = content.filter(Boolean).join("\n");
+
+  // Log that moderation is being performed (without logging the actual content)
+  console.info(
+    `🔍 Performing content moderation check (content length: ${textToModerate.length} chars)`,
+  );
+
+  // First check for suspicious patterns (faster)
+  const hasSuspiciousPatterns = containsSuspiciousPatterns(textToModerate);
+
+  // If suspicious patterns are found, perform AI moderation
+  if (hasSuspiciousPatterns) {
+    console.info(
+      "⚠️ Suspicious patterns detected, performing AI moderation check",
+    );
+    try {
+      const isFlagged = await moderateContentWithAI(textToModerate);
+      if (isFlagged) {
+        console.warn("🚫 Content flagged by AI moderation");
+      }
+      return isFlagged;
+    } catch (error) {
+      console.error("Error during AI content moderation:", error);
+      return false;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Helper function to check if moderation is enabled
+ * @returns True if moderation is enabled, false otherwise
+ */
+export function isModerationEnabled(): boolean {
+  return env.MODERATION_ENABLED === "true";
+}

apps/web/src/features/moderation/libs/ai-moderation.ts

@@ -0,0 +1,28 @@
+import { openai } from "@ai-sdk/openai";
+import { generateText } from "ai";
+
+/**
+ * Moderates content using OpenAI's GPT-4 to detect inappropriate content
+ * @param text The text to moderate
+ * @returns True if the content is flagged as inappropriate, false otherwise
+ */
+export async function moderateContentWithAI(text: string) {
+  try {
+    const result = await generateText({
+      model: openai("gpt-4-turbo"),
+      messages: [
+        {
+          role: "system",
+          content:
+            "You are a content moderator. Analyze the following text and determine if it is attempting to misuse the app to advertise illegal drugs, prostitution, or promote illegal gambling and other illicit activities. Respond with 'FLAGGED' if detected, otherwise 'SAFE'.",
+        },
+        { role: "user", content: text },
+      ],
+    });
+
+    return result.text.includes("FLAGGED");
+  } catch (err) {
+    console.error(`❌ AI moderation failed:`, err);
+    return false;
+  }
+}

apps/web/src/features/moderation/libs/pattern-moderation.ts

@@ -0,0 +1,44 @@
+/**
+ * Checks if the provided text contains suspicious patterns that might indicate spam or abuse
+ * @param text The text to check for suspicious patterns
+ * @returns True if suspicious patterns are detected, false otherwise
+ */
+export function containsSuspiciousPatterns(text: string) {
+  if (!text.trim()) return false;
+
+  // Define all patterns
+  const repetitiveCharsPattern = /(.)\1{4,}/;
+  const allCapsPattern = /[A-Z]{5,}/;
+  const excessiveSpecialCharsPattern =
+    /[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]{4,}/;
+  const suspiciousUrlPattern = /(bit\.ly|tinyurl|goo\.gl|t\.co|is\.gd)/i;
+
+  // Email address pattern
+  const emailPattern = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/i;
+
+  // Comprehensive phone number pattern covering various formats
+  const phoneNumberPattern =
+    /(\+?\d{1,3}[-.\s]?)?(\d{3}[-.\s]?)?\d{3}[-.\s]?\d{4}|\+\d[\d\s\-\.]{5,14}|\+\d{6,15}/i;
+
+  // Detect suspicious Unicode patterns
+  const suspiciousUnicodePattern =
+    /[\u2028-\u202F\u2800-\u28FF\u3164\uFFA0\u115F\u1160\u3000\u2000-\u200F\u2028-\u202F\u205F-\u206F\uFEFF\uDB40\uDC20\uDB40\uDC21\uDB40\uDC22\uDB40\uDC23\uDB40\uDC24]/u;
+
+  // Simple leet speak pattern that detects number-letter-number patterns
+  const leetSpeakPattern = /[a-z0-9]*[0-9][a-z][0-9][a-z0-9]*/i;
+
+  return (
+    // Simple pattern checks (least intensive)
+    allCapsPattern.test(text) ||
+    repetitiveCharsPattern.test(text) ||
+    excessiveSpecialCharsPattern.test(text) ||
+    // Medium complexity patterns
+    suspiciousUrlPattern.test(text) ||
+    emailPattern.test(text) ||
+    leetSpeakPattern.test(text) ||
+    // More complex patterns
+    phoneNumberPattern.test(text) ||
+    // Most intensive pattern (Unicode handling)
+    suspiciousUnicodePattern.test(text)
+  );
+}