From 90f0d90fabc9cc71b2fdbaa44649793821a38f7c Mon Sep 17 00:00:00 2001 From: Luke Vella Date: Sat, 16 Mar 2024 11:56:07 +0700 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Improvements=20to=20house-?= =?UTF-8?q?keeping=20webhook=20(#1063)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/house-keeping.yml | 21 +-- .../src/app/api/house-keeping/[task]/route.ts | 153 ++++++++++++++++++ apps/web/src/pages/api/house-keeping.ts | 53 ------ packages/backend/trpc/routers/polls.ts | 5 +- packages/database/index.ts | 4 - .../middleware/soft-delete-middleware.ts | 34 ---- .../migration.sql | 2 + packages/database/prisma/schema.prisma | 1 + packages/ui/src/button.tsx | 4 +- 9 files changed, 173 insertions(+), 104 deletions(-) create mode 100644 apps/web/src/app/api/house-keeping/[task]/route.ts delete mode 100644 apps/web/src/pages/api/house-keeping.ts delete mode 100644 packages/database/middleware/soft-delete-middleware.ts create mode 100644 packages/database/prisma/migrations/20240315104329_index_votes_by_poll/migration.sql diff --git a/.github/workflows/house-keeping.yml b/.github/workflows/house-keeping.yml index b27b35ebd..2d0bc1077 100644 --- a/.github/workflows/house-keeping.yml +++ b/.github/workflows/house-keeping.yml @@ -1,21 +1,22 @@ name: House Keeping on: workflow_dispatch: - inputs: - query: - description: "Query params for house keeping endpoint" - required: false schedule: - cron: "0 6 * * *" # Every day at 6:00am UTC - +env: + API_SECRET: ${{ secrets.API_SECRET }} jobs: clean: name: "Clean" runs-on: ubuntu-latest steps: - - name: Call house keeping - env: - API_SECRET: ${{ secrets.API_SECRET }} + - name: Mark inactive polls as deleted run: | - curl -X "POST" --fail "https://app.rallly.co/api/house-keeping${{ github.event.inputs.query }}" \ - -H "Authorization: Bearer ${API_SECRET}" + curl -X "POST" --fail "https://app.rallly.co/api/house-keeping/delete-inactive-polls" \ + -H "Authorization: Bearer ${API_SECRET}" \ + - name: Remove deleted polls + run: | + curl -X "POST" --fail "https://app.rallly.co/api/house-keeping/remove-deleted-polls" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${API_SECRET}" \ + -d '{"take": 1000}' diff --git a/apps/web/src/app/api/house-keeping/[task]/route.ts b/apps/web/src/app/api/house-keeping/[task]/route.ts new file mode 100644 index 000000000..8d96fe6a2 --- /dev/null +++ b/apps/web/src/app/api/house-keeping/[task]/route.ts @@ -0,0 +1,153 @@ +import { prisma } from "@rallly/database"; +import { headers } from "next/headers"; +import { NextResponse } from "next/server"; + +export async function POST(req: Request, ctx: { params: { task: string } }) { + const headersList = headers(); + const authorization = headersList.get("authorization"); + + if (authorization !== `Bearer ${process.env.API_SECRET}`) { + return NextResponse.json( + { success: false }, + { + status: 401, + }, + ); + } + + switch (ctx.params.task) { + case "delete-inactive-polls": { + return await deleteInactivePolls(); + } + case "remove-deleted-polls": { + return await removeDeletedPolls(req); + } + } +} + +/** + * Marks inactive polls as deleted. Polls are inactive if they have not been + * touched in the last 30 days and all dates are in the past. + */ +async function deleteInactivePolls() { + const markedDeleted = await prisma.$executeRaw` + UPDATE polls p + SET + deleted = true, + deleted_at = NOW() + WHERE touched_at < NOW() - INTERVAL '30 days' + AND deleted = false + AND id NOT IN ( + SELECT poll_id + FROM options + WHERE poll_id = p.id + AND start_time > NOW() + ) + AND user_id NOT IN ( + SELECT id + FROM users + WHERE id IN ( + SELECT user_id + FROM user_payment_data + WHERE end_date > NOW() + ) + OR subscription_id IN ( + SELECT subscription_id + FROM subscriptions + WHERE active = true + ) + ); + `; + + return NextResponse.json({ + success: true, + summary: { + markedDeleted, + }, + }); +} + +/** + * Remove polls and corresponding data that have been marked deleted for more than 7 days. + */ +async function removeDeletedPolls(req: Request) { + const options = (await req.json()) as { take?: number } | undefined; + // First get the ids of all the polls that have been marked as deleted for at least 7 days + const deletedPolls = await prisma.poll.findMany({ + select: { + id: true, + }, + where: { + deleted: true, + deletedAt: { + lt: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + }, + }, + take: options?.take ?? 1000, + }); + + const deletedPollIds = deletedPolls.map((poll) => poll.id); + + const { count: deletedWatcherCount } = await prisma.watcher.deleteMany({ + where: { + pollId: { + in: deletedPollIds, + }, + }, + }); + + const { count: deletedVoteCount } = await prisma.vote.deleteMany({ + where: { + pollId: { + in: deletedPollIds, + }, + }, + }); + + const { count: deletedParticipantCount } = + await prisma.participant.deleteMany({ + where: { + pollId: { + in: deletedPollIds, + }, + }, + }); + + const { count: deletedOptionCount } = await prisma.option.deleteMany({ + where: { + pollId: { + in: deletedPollIds, + }, + }, + }); + + const { count: deletedCommentCount } = await prisma.comment.deleteMany({ + where: { + pollId: { + in: deletedPollIds, + }, + }, + }); + + const { count: deletedPollCount } = await prisma.poll.deleteMany({ + where: { + id: { + in: deletedPollIds, + }, + }, + }); + + return NextResponse.json({ + success: true, + summary: { + deleted: { + votes: deletedVoteCount, + options: deletedOptionCount, + participants: deletedParticipantCount, + comments: deletedCommentCount, + watchers: deletedWatcherCount, + polls: deletedPollCount, + }, + }, + }); +} diff --git a/apps/web/src/pages/api/house-keeping.ts b/apps/web/src/pages/api/house-keeping.ts deleted file mode 100644 index 707ea475a..000000000 --- a/apps/web/src/pages/api/house-keeping.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { prisma } from "@rallly/database"; -import { NextApiRequest, NextApiResponse } from "next"; - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - if (req.method !== "POST") { - res.setHeader("Allow", "POST"); - res.status(405).end("Method not allowed"); - return; - } - - const { authorization } = req.headers; - - if (authorization !== `Bearer ${process.env.API_SECRET}`) { - res.status(401).json({ success: false }); - return; - } - - const deletedPolls = await prisma.$executeRaw` - UPDATE polls p - SET - deleted = true, - deleted_at = NOW() - WHERE touched_at < NOW() - INTERVAL '30 days' - AND deleted = false - AND id NOT IN ( - SELECT poll_id - FROM options - WHERE poll_id = p.id - AND start_time > NOW() - ) - AND user_id NOT IN ( - SELECT id - FROM users - WHERE id IN ( - SELECT user_id - FROM user_payment_data - WHERE end_date > NOW() - ) - OR subscription_id IN ( - SELECT subscription_id - FROM subscriptions - WHERE active = true - ) - ); - `; - - res.status(200).json({ - softDeleted: deletedPolls, - }); -} diff --git a/packages/backend/trpc/routers/polls.ts b/packages/backend/trpc/routers/polls.ts index c86ab5c01..85031fe9e 100644 --- a/packages/backend/trpc/routers/polls.ts +++ b/packages/backend/trpc/routers/polls.ts @@ -237,7 +237,10 @@ export const polls = router({ ) .mutation(async ({ input: { urlId } }) => { const pollId = await getPollIdFromAdminUrlId(urlId); - await prisma.poll.delete({ where: { id: pollId } }); + await prisma.poll.update({ + where: { id: pollId }, + data: { deleted: true, deletedAt: new Date() }, + }); }), touch: publicProcedure .input( diff --git a/packages/database/index.ts b/packages/database/index.ts index 2a412bdf5..9f455963e 100644 --- a/packages/database/index.ts +++ b/packages/database/index.ts @@ -1,7 +1,5 @@ import { PrismaClient } from "@rallly/database"; -import { softDeleteMiddleware } from "./middleware/soft-delete-middleware"; - export * from "@prisma/client"; declare global { @@ -12,6 +10,4 @@ declare global { export const prisma = global.prisma || new PrismaClient(); -softDeleteMiddleware(prisma, "Poll"); - if (process.env.NODE_ENV !== "production") global.prisma = prisma; diff --git a/packages/database/middleware/soft-delete-middleware.ts b/packages/database/middleware/soft-delete-middleware.ts deleted file mode 100644 index a7ea759b2..000000000 --- a/packages/database/middleware/soft-delete-middleware.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Prisma, PrismaClient } from "@rallly/database"; - -export const softDeleteMiddleware = ( - prisma: PrismaClient, - model: Prisma.ModelName, -) => { - prisma.$use(async (params, next) => { - // We use middleware to handle soft deletes - // See: https://www.prisma.io/docs/concepts/components/prisma-client/middleware/soft-delete-middleware - if (params.model === model) { - if (params.action === "delete") { - // Delete queries - // Change action to an update - params.action = "update"; - params.args["data"] = { deleted: true, deletedAt: new Date() }; - } - if (params.action == "deleteMany") { - // Delete many queries - params.action = "updateMany"; - if (params.args.data != undefined) { - params.args.data["deleted"] = true; - } else { - params.args["data"] = { deleted: true, deletedAt: new Date() }; - } - } - if (params.action === "findFirst") { - // Add 'deleted' filter - // ID filter maintained - params.args.where["deleted"] = params.args.where["deleted"] || false; - } - } - return next(params); - }); -}; diff --git a/packages/database/prisma/migrations/20240315104329_index_votes_by_poll/migration.sql b/packages/database/prisma/migrations/20240315104329_index_votes_by_poll/migration.sql new file mode 100644 index 000000000..8338f1fbe --- /dev/null +++ b/packages/database/prisma/migrations/20240315104329_index_votes_by_poll/migration.sql @@ -0,0 +1,2 @@ +-- CreateIndex +CREATE INDEX "votes_poll_id_idx" ON "votes" USING HASH ("poll_id"); diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 8af14b885..69d3b977b 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -226,6 +226,7 @@ model Vote { @@index([participantId], type: Hash) @@index([optionId], type: Hash) + @@index([pollId], type: Hash) @@map("votes") } diff --git a/packages/ui/src/button.tsx b/packages/ui/src/button.tsx index ef22809d3..3996b37c4 100644 --- a/packages/ui/src/button.tsx +++ b/packages/ui/src/button.tsx @@ -8,7 +8,7 @@ import { cn } from "@rallly/ui"; const buttonVariants = cva( cn( - "inline-flex border font-medium disabled:text-muted-foreground disabled:bg-muted disabled:pointer-events-none select-none items-center justify-center whitespace-nowrap rounded-md border", + "inline-flex shadow-[inset_0_0.5px_0_0.5px_rgb(255,255,255,0.4),_inset_0_-0.5px_0_0.5px_rgb(0,0,0,0.1)] border font-medium disabled:text-muted-foreground disabled:bg-muted disabled:pointer-events-none select-none items-center justify-center whitespace-nowrap rounded-md border", "focus-visible:ring-offset-input-background focus-visible:border-primary-400 focus-visible:ring-2 focus-visible:ring-indigo-100", ), { @@ -19,7 +19,7 @@ const buttonVariants = cva( destructive: "bg-destructive text-destructive-foreground focus:ring-offset-1 active:bg-destructive border-destructive hover:bg-destructive/90", default: - "rounded-md px-3.5 py-2.5 data-[state=open]:shadow-none data-[state=open]:bg-gray-100 active:bg-gray-200 focus:border-gray-300 hover:bg-gray-100 bg-gray-50", + "rounded-md px-3.5 shadow-inner py-2.5 data-[state=open]:shadow-none data-[state=open]:bg-gray-100 active:bg-gray-200 focus:border-gray-300 hover:bg-gray-100 bg-gray-50", secondary: "bg-secondary text-secondary-foreground hover:bg-secondary/80", ghost: "border-transparent hover:bg-gray-200 active:bg-gray-300",