From 8f4bdad8c5c08b52227e700359e275cfea645d6b Mon Sep 17 00:00:00 2001 From: Luke Vella Date: Mon, 23 May 2022 08:26:10 +0100 Subject: [PATCH] Update house-keeping policy (#185) --- prisma/middlewares/softDeleteMiddleware.ts | 6 +- .../migration.sql | 4 + prisma/schema.prisma | 1 + public/locales/en/app.json | 2 +- .../poll/manage-poll/delete-poll-form.tsx | 3 +- src/pages/api/house-keeping.ts | 60 ++--- tests/house-keeping.spec.ts | 252 ++++++++++++++++++ 7 files changed, 289 insertions(+), 39 deletions(-) create mode 100644 prisma/migrations/20220522165453_add_deleted_at_column/migration.sql create mode 100644 tests/house-keeping.spec.ts diff --git a/prisma/middlewares/softDeleteMiddleware.ts b/prisma/middlewares/softDeleteMiddleware.ts index dbb4ef336..aafe6e194 100644 --- a/prisma/middlewares/softDeleteMiddleware.ts +++ b/prisma/middlewares/softDeleteMiddleware.ts @@ -12,7 +12,7 @@ export const softDeleteMiddleware = ( // Delete queries // Change action to an update params.action = "update"; - params.args["data"] = { deleted: true }; + params.args["data"] = { deleted: true, deletedAt: new Date() }; } if (params.action == "deleteMany") { // Delete many queries @@ -20,7 +20,7 @@ export const softDeleteMiddleware = ( if (params.args.data != undefined) { params.args.data["deleted"] = true; } else { - params.args["data"] = { deleted: true }; + params.args["data"] = { deleted: true, deletedAt: new Date() }; } } if (params.action === "findUnique" || params.action === "findFirst") { @@ -29,7 +29,7 @@ export const softDeleteMiddleware = ( params.action = "findFirst"; // Add 'deleted' filter // ID filter maintained - params.args.where["deleted"] = false; + params.args.where["deleted"] = params.args.where["deleted"] || false; } } return next(params); diff --git a/prisma/migrations/20220522165453_add_deleted_at_column/migration.sql b/prisma/migrations/20220522165453_add_deleted_at_column/migration.sql new file mode 100644 index 000000000..9deb52970 --- /dev/null +++ b/prisma/migrations/20220522165453_add_deleted_at_column/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "polls" ADD COLUMN "deleted_at" TIMESTAMP(3); + +UPDATE "polls" SET "deleted_at" = now() WHERE "deleted" = true; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3083c8c6b..123801f87 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -52,6 +52,7 @@ model Poll { closed Boolean @default(false) notifications Boolean @default(false) deleted Boolean @default(false) + deletedAt DateTime? @map("deleted_at") touchedAt DateTime @default(now()) @map("touched_at") @@map("polls") diff --git a/public/locales/en/app.json b/public/locales/en/app.json index 272fe0805..a5a33e1f6 100644 --- a/public/locales/en/app.json +++ b/public/locales/en/app.json @@ -55,7 +55,7 @@ "no": "No", "ifNeedBe": "If need be", "areYouSure": "Are you sure?", - "deletePollDescription": "All data related to this poll will be deleted. This action cannot be undone. To confirm, please type “{{confirmText}}” in to the input below:", + "deletePollDescription": "All data related to this poll will be deleted. To confirm, please type “{{confirmText}}” in to the input below:", "deletePoll": "Delete poll", "demoPollNotice": "Demo polls are automatically deleted after 1 day" } diff --git a/src/components/poll/manage-poll/delete-poll-form.tsx b/src/components/poll/manage-poll/delete-poll-form.tsx index 7e0079902..060e53706 100644 --- a/src/components/poll/manage-poll/delete-poll-form.tsx +++ b/src/components/poll/manage-poll/delete-poll-form.tsx @@ -54,8 +54,7 @@ export const DeletePollForm: React.VoidFunctionComponent<{ i18nKey="deletePollDescription" values={{ confirmText }} components={{ - b: , - s: , + s: , }} />

diff --git a/src/pages/api/house-keeping.ts b/src/pages/api/house-keeping.ts index 57a971b8f..c4cc58abb 100644 --- a/src/pages/api/house-keeping.ts +++ b/src/pages/api/house-keeping.ts @@ -5,11 +5,7 @@ import { NextApiRequest, NextApiResponse } from "next"; import { prisma } from "~/prisma/db"; /** - * This endpoint will permanently delete polls that: - * * have been soft deleted OR - * * are demo polls that are older than 1 day OR - * * polls that have not been accessed for 30 days - * All dependant records are also deleted. + * DANGER: This endpoint will permanently delete polls. */ export default async function handler( req: NextApiRequest, @@ -21,17 +17,6 @@ export default async function handler( return; } - // Batch size is the max number of polls we will attempt to delete - // We can adjust this value through the batchSize query param - const parsedBatchSizeQueryParam = - typeof req.query["batchSize"] === "string" - ? parseInt(req.query["batchSize"]) - : NaN; - - // If not specified we default to a max of 500 polls - const batchSize = - parsedBatchSizeQueryParam > 0 ? parsedBatchSizeQueryParam : 500; - const { authorization } = req.headers; if (authorization !== `Bearer ${process.env.API_SECRET}`) { @@ -39,12 +24,26 @@ export default async function handler( return; } - const pollIds = ( + // soft delete polls that have not been accessed for over 30 days + const inactivePolls = await prisma.poll.deleteMany({ + where: { + deleted: false, + touchedAt: { + lte: addDays(new Date(), -30), + }, + }, + }); + + // Permantly delete old demos and polls that have been soft deleted for 7 days + const pollIdsToDelete = ( await prisma.poll.findMany({ where: { OR: [ { deleted: true, + deletedAt: { + lte: addDays(new Date(), -7), + }, }, // demo polls that are 1 day old { @@ -53,12 +52,6 @@ export default async function handler( lte: addDays(new Date(), -1), }, }, - // polls that have not been accessed for over 30 days - { - touchedAt: { - lte: addDays(new Date(), -30), - }, - }, ], }, select: { @@ -67,16 +60,15 @@ export default async function handler( orderBy: { createdAt: "asc", // oldest first }, - take: batchSize, }) ).map(({ urlId }) => urlId); - if (pollIds.length !== 0) { + if (pollIdsToDelete.length !== 0) { // Delete links await prisma.link.deleteMany({ where: { pollId: { - in: pollIds, + in: pollIdsToDelete, }, }, }); @@ -84,7 +76,7 @@ export default async function handler( await prisma.comment.deleteMany({ where: { pollId: { - in: pollIds, + in: pollIdsToDelete, }, }, }); @@ -92,7 +84,7 @@ export default async function handler( await prisma.vote.deleteMany({ where: { pollId: { - in: pollIds, + in: pollIdsToDelete, }, }, }); @@ -101,7 +93,7 @@ export default async function handler( await prisma.participant.deleteMany({ where: { pollId: { - in: pollIds, + in: pollIdsToDelete, }, }, }); @@ -110,18 +102,20 @@ export default async function handler( await prisma.option.deleteMany({ where: { pollId: { - in: pollIds, + in: pollIdsToDelete, }, }, }); // Delete polls - prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join( - pollIds, + // Using execute raw to bypass soft delete middelware + await prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join( + pollIdsToDelete, )})`; } res.status(200).json({ - deleted: pollIds, + inactive: inactivePolls.count, + deleted: pollIdsToDelete.length, }); } diff --git a/tests/house-keeping.spec.ts b/tests/house-keeping.spec.ts new file mode 100644 index 000000000..34c36496b --- /dev/null +++ b/tests/house-keeping.spec.ts @@ -0,0 +1,252 @@ +import { expect, test } from "@playwright/test"; +import { Prisma } from "@prisma/client"; +import { addDays } from "date-fns"; + +import { prisma } from "../prisma/db"; + +/** + * House keeping policy: + * * Demo polls are hard deleted after one day + * * Polls are soft deleted after 30 days of inactivity + * * Soft deleted polls are hard deleted after 7 days of being soft deleted + */ +test.beforeAll(async ({ request, baseURL }) => { + await prisma.poll.createMany({ + data: [ + // Active Poll + { + title: "Active Poll", + urlId: "active-poll", + type: "date", + userId: "user1", + }, + // Poll that has been deleted 6 days ago + { + title: "Deleted poll", + urlId: "deleted-poll-6d", + type: "date", + userId: "user1", + deleted: true, + deletedAt: addDays(new Date(), -6), + }, + // Poll that has been deleted 7 days ago + { + title: "Deleted poll 7d", + urlId: "deleted-poll-7d", + type: "date", + userId: "user1", + deleted: true, + deletedAt: addDays(new Date(), -7), + }, + // Poll that has been inactive for 29 days + { + title: "Still active", + urlId: "still-active-poll", + type: "date", + userId: "user1", + touchedAt: addDays(new Date(), -29), + }, + // Poll that has been inactive for 30 days + { + title: "Inactive poll", + urlId: "inactive-poll", + type: "date", + userId: "user1", + touchedAt: addDays(new Date(), -30), + }, + // Demo poll + { + demo: true, + title: "Demo poll", + urlId: "demo-poll-new", + type: "date", + userId: "user1", + createdAt: new Date(), + }, + // Old demo poll + { + demo: true, + title: "Demo poll", + urlId: "demo-poll-old", + type: "date", + userId: "user1", + createdAt: addDays(new Date(), -2), + }, + ], + }); + + await prisma.option.createMany({ + data: [ + { + id: "option-1", + value: "2022-02-22", + pollId: "deleted-poll-7d", + }, + { + id: "option-2", + value: "2022-02-23", + pollId: "deleted-poll-7d", + }, + { + id: "option-3", + value: "2022-02-24", + pollId: "deleted-poll-7d", + }, + ], + }); + + await prisma.participant.create({ + data: { id: "participant-1", name: "Luke", pollId: "deleted-poll-7d" }, + }); + + await prisma.vote.createMany({ + data: [ + { + optionId: "option-1", + type: "yes", + participantId: "participant-1", + pollId: "deleted-poll-7d", + }, + { + optionId: "option-2", + type: "no", + participantId: "participant-1", + pollId: "deleted-poll-7d", + }, + { + optionId: "option-3", + type: "yes", + participantId: "participant-1", + pollId: "deleted-poll-7d", + }, + ], + }); + + // call house-keeping endpoint + const res = await request.post(`${baseURL}/api/house-keeping`, { + headers: { + Authorization: `Bearer ${process.env.API_SECRET}`, + }, + }); + + expect(await res.json()).toMatchObject({ + inactive: 1, + deleted: 2, + }); +}); + +test("should keep active polls", async () => { + const poll = await prisma.poll.findUnique({ + where: { + urlId: "active-poll", + }, + }); + + // expect active poll to not be deleted + expect(poll).not.toBeNull(); + expect(poll?.deleted).toBeFalsy(); +}); + +test("should keep polls that have been soft deleted for less than 7 days", async () => { + const deletedPoll6d = await prisma.poll.findFirst({ + where: { + urlId: "deleted-poll-6d", + deleted: true, + }, + }); + + // expect a poll that has been deleted for 6 days to + expect(deletedPoll6d).not.toBeNull(); +}); + +test("should hard delete polls that have been soft deleted for 7 days", async () => { + const deletedPoll7d = await prisma.poll.findFirst({ + where: { + urlId: "deleted-poll-7d", + deleted: true, + }, + }); + + expect(deletedPoll7d).toBeNull(); + + const participants = await prisma.participant.findMany({ + where: { + pollId: "deleted-poll-7d", + }, + }); + + expect(participants.length).toBe(0); + + const votes = await prisma.vote.findMany({ + where: { + pollId: "deleted-poll-7d", + }, + }); + + expect(votes.length).toBe(0); + + const options = await prisma.option.findMany({ + where: { + pollId: "deleted-poll-7d", + }, + }); + + expect(options.length).toBe(0); +}); + +test("should keep polls that are still active", async () => { + const stillActivePoll = await prisma.poll.findUnique({ + where: { + urlId: "still-active-poll", + }, + }); + + expect(stillActivePoll).not.toBeNull(); + expect(stillActivePoll?.deleted).toBeFalsy(); +}); + +test("should soft delete polls that are inactive", async () => { + const inactivePoll = await prisma.poll.findFirst({ + where: { + urlId: "inactive-poll", + deleted: true, + }, + }); + + expect(inactivePoll).not.toBeNull(); + expect(inactivePoll?.deleted).toBeTruthy(); + expect(inactivePoll?.deletedAt).toBeTruthy(); +}); + +test("should keep new demo poll", async () => { + const demoPoll = await prisma.poll.findFirst({ + where: { + urlId: "demo-poll-new", + }, + }); + + expect(demoPoll).not.toBeNull(); +}); + +test("should delete old demo poll", async () => { + const oldDemoPoll = await prisma.poll.findFirst({ + where: { + urlId: "demo-poll-old", + }, + }); + + expect(oldDemoPoll).toBeNull(); +}); + +// Teardown +test.afterAll(async () => { + await prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join([ + "active-poll", + "deleted-poll-6d", + "deleted-poll-7d", + "still-active-poll", + "inactive-poll", + "demo-poll-new", + "demo-poll-old", + ])})`; +});