📦 EqualifyEverything / equalify

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112import { parser } from "@aws-lambda-powertools/parser/middleware";
import middy from "@middy/core";

import { logger, metrics } from "./telemetry.ts";
import { scansSchema } from "../../../shared/types/scansSchema.zod.ts";
import { MetricUnit } from "@aws-lambda-powertools/metrics";

import { SQSClient, SendMessageBatchCommand } from "@aws-sdk/client-sqs";
const sqsClient = new SQSClient({ region: "us-east-2" });
const htmlQueueUrl =
  "https://sqs.us-east-2.amazonaws.com/380610849750/scanHtml.fifo";

const pdfQueueUrl =
  "https://sqs.us-east-2.amazonaws.com/380610849750/scanPdf.fifo";

export const handler = middy()
  .use(parser({ schema: scansSchema }))
  .handler(async (event): Promise<void> => {
    logger.info(`Received ${event.urls?.length || 0} URLs to route`);
    
    // Check for no URLs in request
    if(event.urls?.length ===0){
      logger.info(`No URLs received in request, exiting!`);
      return;
    }
    // get the type="html" URLs
    const htmlUrls = event.urls.filter((item) => {
      return item.type === "html";
    });
    logger.info(`Found ${htmlUrls.length} HTML URLs and ${event.urls.length - htmlUrls.length} PDF URLs`);

    // we can pass 10 events at a time to SQS
    const HtmlBatches = chunkArray(htmlUrls, 10);

    // for each batch, send to SQS
    for (const batch of HtmlBatches) {
      const formattedMessages = batch.map((item) => {
        return {
          MessageGroupId: item.auditId,
          Id: item.urlId,
          MessageDeduplicationId: `${item.scanId}-${item.urlId}`,
          MessageBody: JSON.stringify({
            data: item,
          }),
        };
      });
      const command = new SendMessageBatchCommand({
        QueueUrl: htmlQueueUrl,
        Entries: formattedMessages,
      });
      try {
        const response = await sqsClient.send(command);
        if (response.Successful && response.Successful.length > 0) {
          logger.info(`HTML Batch send successful: ${response.Successful.length} messages sent`);
        }
        if (response.Failed && response.Failed.length > 0) {
          logger.error(`HTML Messages failed to send: ${JSON.stringify(response.Failed)}`);
        }
      } catch (error) {
        logger.error("Error sending HTML batch:", error as Error);
      }
    }

    // PDF routing
    const pdfUrls = event.urls.filter((item) => {
      return item.type === "pdf";
    });

    // we can pass 10 events at a time to SQS
    const PdfBatches = chunkArray(pdfUrls, 10);
    // for each batch, send to SQS
    for (const batch of PdfBatches) {
      const formattedMessages = batch.map((item) => {
        return {
          MessageGroupId: item.auditId,
          Id: item.urlId,
          MessageDeduplicationId: `${item.scanId}-${item.urlId}`,
          MessageBody: JSON.stringify({
            data: item,
          }),
        };
      });
      const command = new SendMessageBatchCommand({
        QueueUrl: pdfQueueUrl,
        Entries: formattedMessages,
      });
      try {
        const response = await sqsClient.send(command);
        if (response.Successful && response.Successful.length > 0) {
          logger.info(`PDF Batch send successful: ${response.Successful.length} messages sent`);
        }
        if (response.Failed && response.Failed.length > 0) {
          logger.error(`PDF Messages failed to send: ${JSON.stringify(response.Failed)}`);
        }
      } catch (error) {
        logger.error("Error sending PDF batch:", error as Error);
      }
    }


    logger.info("Finished sending batch");
    metrics.addMetric("scanRequest", MetricUnit.Count, 1);
  });

function chunkArray<T>(array: T[], chunkSize: number): T[][] {
  const result: T[][] = [];
  for (let i = 0; i < array.length; i += chunkSize) {
    result.push(array.slice(i, i + chunkSize));
  }
  return result;
}