Update house-keeping policy (#185)

prisma/middlewares/softDeleteMiddleware.ts

@@ -12,7 +12,7 @@ export const softDeleteMiddleware = (
         // Delete queries
         // Change action to an update
         params.action = "update";
-        params.args["data"] = { deleted: true };
+        params.args["data"] = { deleted: true, deletedAt: new Date() };
       }
       if (params.action == "deleteMany") {
         // Delete many queries
@@ -20,7 +20,7 @@ export const softDeleteMiddleware = (
         if (params.args.data != undefined) {
           params.args.data["deleted"] = true;
         } else {
-          params.args["data"] = { deleted: true };
+          params.args["data"] = { deleted: true, deletedAt: new Date() };
         }
       }
       if (params.action === "findUnique" || params.action === "findFirst") {
@@ -29,7 +29,7 @@ export const softDeleteMiddleware = (
         params.action = "findFirst";
         // Add 'deleted' filter
         // ID filter maintained
-        params.args.where["deleted"] = false;
+        params.args.where["deleted"] = params.args.where["deleted"] || false;
       }
     }
     return next(params);

prisma/migrations/20220522165453_add_deleted_at_column/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "polls" ADD COLUMN     "deleted_at" TIMESTAMP(3);
+
+UPDATE "polls" SET "deleted_at" = now() WHERE "deleted" = true;

prisma/schema.prisma

@@ -52,6 +52,7 @@ model Poll {
   closed        Boolean       @default(false)
   notifications Boolean       @default(false)
   deleted       Boolean       @default(false)
+  deletedAt     DateTime?     @map("deleted_at")
   touchedAt     DateTime      @default(now()) @map("touched_at")
 
   @@map("polls")

public/locales/en/app.json

@@ -55,7 +55,7 @@
   "no": "No",
   "ifNeedBe": "If need be",
   "areYouSure": "Are you sure?",
-  "deletePollDescription": "All data related to this poll will be deleted. This action <b>cannot be undone</b>. To confirm, please type <s>“{{confirmText}}”</s> in to the input below:",
+  "deletePollDescription": "All data related to this poll will be deleted. To confirm, please type <s>“{{confirmText}}”</s> in to the input below:",
   "deletePoll": "Delete poll",
   "demoPollNotice": "Demo polls are automatically deleted after 1 day"
 }

src/components/poll/manage-poll/delete-poll-form.tsx

@@ -54,8 +54,7 @@ export const DeletePollForm: React.VoidFunctionComponent<{
             i18nKey="deletePollDescription"
             values={{ confirmText }}
             components={{
-              b: <strong />,
-              s: <span className="whitespace-nowrap" />,
+              s: <span className="whitespace-nowrap font-mono" />,
             }}
           />
         </p>

src/pages/api/house-keeping.ts

@@ -5,11 +5,7 @@ import { NextApiRequest, NextApiResponse } from "next";
 import { prisma } from "~/prisma/db";
 
 /**
- * This endpoint will permanently delete polls that:
- * * have been soft deleted OR
- * * are demo polls that are older than 1 day OR
- * * polls that have not been accessed for 30 days
- * All dependant records are also deleted.
+ * DANGER: This endpoint will permanently delete polls.
  */
 export default async function handler(
   req: NextApiRequest,
@@ -21,17 +17,6 @@ export default async function handler(
     return;
   }
 
-  // Batch size is the max number of polls we will attempt to delete
-  // We can adjust this value through the batchSize query param
-  const parsedBatchSizeQueryParam =
-    typeof req.query["batchSize"] === "string"
-      ? parseInt(req.query["batchSize"])
-      : NaN;
-
-  // If not specified we default to a max of 500 polls
-  const batchSize =
-    parsedBatchSizeQueryParam > 0 ? parsedBatchSizeQueryParam : 500;
-
   const { authorization } = req.headers;
 
   if (authorization !== `Bearer ${process.env.API_SECRET}`) {
@@ -39,12 +24,26 @@ export default async function handler(
     return;
   }
 
-  const pollIds = (
+  // soft delete polls that have not been accessed for over 30 days
+  const inactivePolls = await prisma.poll.deleteMany({
+    where: {
+      deleted: false,
+      touchedAt: {
+        lte: addDays(new Date(), -30),
+      },
+    },
+  });
+
+  // Permantly delete old demos and polls that have been soft deleted for 7 days
+  const pollIdsToDelete = (
     await prisma.poll.findMany({
       where: {
         OR: [
           {
             deleted: true,
+            deletedAt: {
+              lte: addDays(new Date(), -7),
+            },
           },
           // demo polls that are 1 day old
           {
@@ -53,12 +52,6 @@ export default async function handler(
               lte: addDays(new Date(), -1),
             },
           },
-          // polls that have not been accessed for over 30 days
-          {
-            touchedAt: {
-              lte: addDays(new Date(), -30),
-            },
-          },
         ],
       },
       select: {
@@ -67,16 +60,15 @@ export default async function handler(
       orderBy: {
         createdAt: "asc", // oldest first
       },
-      take: batchSize,
     })
   ).map(({ urlId }) => urlId);
 
-  if (pollIds.length !== 0) {
+  if (pollIdsToDelete.length !== 0) {
     // Delete links
     await prisma.link.deleteMany({
       where: {
         pollId: {
-          in: pollIds,
+          in: pollIdsToDelete,
         },
       },
     });
@@ -84,7 +76,7 @@ export default async function handler(
     await prisma.comment.deleteMany({
       where: {
         pollId: {
-          in: pollIds,
+          in: pollIdsToDelete,
         },
       },
     });
@@ -92,7 +84,7 @@ export default async function handler(
     await prisma.vote.deleteMany({
       where: {
         pollId: {
-          in: pollIds,
+          in: pollIdsToDelete,
         },
       },
     });
@@ -101,7 +93,7 @@ export default async function handler(
     await prisma.participant.deleteMany({
       where: {
         pollId: {
-          in: pollIds,
+          in: pollIdsToDelete,
         },
       },
     });
@@ -110,18 +102,20 @@ export default async function handler(
     await prisma.option.deleteMany({
       where: {
         pollId: {
-          in: pollIds,
+          in: pollIdsToDelete,
         },
       },
     });
 
     // Delete polls
-    prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join(
-      pollIds,
+    // Using execute raw to bypass soft delete middelware
+    await prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join(
+      pollIdsToDelete,
     )})`;
   }
 
   res.status(200).json({
-    deleted: pollIds,
+    inactive: inactivePolls.count,
+    deleted: pollIdsToDelete.length,
   });
 }

tests/house-keeping.spec.ts

@@ -0,0 +1,252 @@
+import { expect, test } from "@playwright/test";
+import { Prisma } from "@prisma/client";
+import { addDays } from "date-fns";
+
+import { prisma } from "../prisma/db";
+
+/**
+ * House keeping policy:
+ * * Demo polls are hard deleted after one day
+ * * Polls are soft deleted after 30 days of inactivity
+ * * Soft deleted polls are hard deleted after 7 days of being soft deleted
+ */
+test.beforeAll(async ({ request, baseURL }) => {
+  await prisma.poll.createMany({
+    data: [
+      // Active Poll
+      {
+        title: "Active Poll",
+        urlId: "active-poll",
+        type: "date",
+        userId: "user1",
+      },
+      // Poll that has been deleted 6 days ago
+      {
+        title: "Deleted poll",
+        urlId: "deleted-poll-6d",
+        type: "date",
+        userId: "user1",
+        deleted: true,
+        deletedAt: addDays(new Date(), -6),
+      },
+      // Poll that has been deleted 7 days ago
+      {
+        title: "Deleted poll 7d",
+        urlId: "deleted-poll-7d",
+        type: "date",
+        userId: "user1",
+        deleted: true,
+        deletedAt: addDays(new Date(), -7),
+      },
+      // Poll that has been inactive for 29 days
+      {
+        title: "Still active",
+        urlId: "still-active-poll",
+        type: "date",
+        userId: "user1",
+        touchedAt: addDays(new Date(), -29),
+      },
+      // Poll that has been inactive for 30 days
+      {
+        title: "Inactive poll",
+        urlId: "inactive-poll",
+        type: "date",
+        userId: "user1",
+        touchedAt: addDays(new Date(), -30),
+      },
+      // Demo poll
+      {
+        demo: true,
+        title: "Demo poll",
+        urlId: "demo-poll-new",
+        type: "date",
+        userId: "user1",
+        createdAt: new Date(),
+      },
+      // Old demo poll
+      {
+        demo: true,
+        title: "Demo poll",
+        urlId: "demo-poll-old",
+        type: "date",
+        userId: "user1",
+        createdAt: addDays(new Date(), -2),
+      },
+    ],
+  });
+
+  await prisma.option.createMany({
+    data: [
+      {
+        id: "option-1",
+        value: "2022-02-22",
+        pollId: "deleted-poll-7d",
+      },
+      {
+        id: "option-2",
+        value: "2022-02-23",
+        pollId: "deleted-poll-7d",
+      },
+      {
+        id: "option-3",
+        value: "2022-02-24",
+        pollId: "deleted-poll-7d",
+      },
+    ],
+  });
+
+  await prisma.participant.create({
+    data: { id: "participant-1", name: "Luke", pollId: "deleted-poll-7d" },
+  });
+
+  await prisma.vote.createMany({
+    data: [
+      {
+        optionId: "option-1",
+        type: "yes",
+        participantId: "participant-1",
+        pollId: "deleted-poll-7d",
+      },
+      {
+        optionId: "option-2",
+        type: "no",
+        participantId: "participant-1",
+        pollId: "deleted-poll-7d",
+      },
+      {
+        optionId: "option-3",
+        type: "yes",
+        participantId: "participant-1",
+        pollId: "deleted-poll-7d",
+      },
+    ],
+  });
+
+  // call house-keeping endpoint
+  const res = await request.post(`${baseURL}/api/house-keeping`, {
+    headers: {
+      Authorization: `Bearer ${process.env.API_SECRET}`,
+    },
+  });
+
+  expect(await res.json()).toMatchObject({
+    inactive: 1,
+    deleted: 2,
+  });
+});
+
+test("should keep active polls", async () => {
+  const poll = await prisma.poll.findUnique({
+    where: {
+      urlId: "active-poll",
+    },
+  });
+
+  // expect active poll to not be deleted
+  expect(poll).not.toBeNull();
+  expect(poll?.deleted).toBeFalsy();
+});
+
+test("should keep polls that have been soft deleted for less than 7 days", async () => {
+  const deletedPoll6d = await prisma.poll.findFirst({
+    where: {
+      urlId: "deleted-poll-6d",
+      deleted: true,
+    },
+  });
+
+  // expect a poll that has been deleted for 6 days to
+  expect(deletedPoll6d).not.toBeNull();
+});
+
+test("should hard delete polls that have been soft deleted for 7 days", async () => {
+  const deletedPoll7d = await prisma.poll.findFirst({
+    where: {
+      urlId: "deleted-poll-7d",
+      deleted: true,
+    },
+  });
+
+  expect(deletedPoll7d).toBeNull();
+
+  const participants = await prisma.participant.findMany({
+    where: {
+      pollId: "deleted-poll-7d",
+    },
+  });
+
+  expect(participants.length).toBe(0);
+
+  const votes = await prisma.vote.findMany({
+    where: {
+      pollId: "deleted-poll-7d",
+    },
+  });
+
+  expect(votes.length).toBe(0);
+
+  const options = await prisma.option.findMany({
+    where: {
+      pollId: "deleted-poll-7d",
+    },
+  });
+
+  expect(options.length).toBe(0);
+});
+
+test("should keep polls that are still active", async () => {
+  const stillActivePoll = await prisma.poll.findUnique({
+    where: {
+      urlId: "still-active-poll",
+    },
+  });
+
+  expect(stillActivePoll).not.toBeNull();
+  expect(stillActivePoll?.deleted).toBeFalsy();
+});
+
+test("should soft delete polls that are inactive", async () => {
+  const inactivePoll = await prisma.poll.findFirst({
+    where: {
+      urlId: "inactive-poll",
+      deleted: true,
+    },
+  });
+
+  expect(inactivePoll).not.toBeNull();
+  expect(inactivePoll?.deleted).toBeTruthy();
+  expect(inactivePoll?.deletedAt).toBeTruthy();
+});
+
+test("should keep new demo poll", async () => {
+  const demoPoll = await prisma.poll.findFirst({
+    where: {
+      urlId: "demo-poll-new",
+    },
+  });
+
+  expect(demoPoll).not.toBeNull();
+});
+
+test("should delete old demo poll", async () => {
+  const oldDemoPoll = await prisma.poll.findFirst({
+    where: {
+      urlId: "demo-poll-old",
+    },
+  });
+
+  expect(oldDemoPoll).toBeNull();
+});
+
+// Teardown
+test.afterAll(async () => {
+  await prisma.$executeRaw`DELETE FROM polls WHERE url_id IN (${Prisma.join([
+    "active-poll",
+    "deleted-poll-6d",
+    "deleted-poll-7d",
+    "still-active-poll",
+    "inactive-poll",
+    "demo-poll-new",
+    "demo-poll-old",
+  ])})`;
+});