Rendering: Support new concurrency mode: contextPerRenderKey (#314)

* allow scaled thumbnails

* added support for full page images

* add `acceptBeforeUnload` handler

* use png files

* produce requested thumbnail size

* use browser per domain concurrency

* browser per render key

* browser per render key

* poolpeteer!

* merge conflict fixes

* add debugging options

* update yarnlock

* fix: disable `headed` by default

* reduce the default worker shutdownTimeout

* yarn lock update

* add the ability to emulate network conditions

* add domain to the groupId

* #44449: rollback `mode` changes in default.json/dev.json

* #44449: use poolpeteer only for `contextPerRenderKey`

* Update package.json

Co-authored-by: Agnès Toulet <[email protected]>

* Update src/config.ts

Co-authored-by: Agnès Toulet <[email protected]>

* Update src/config.ts

Co-authored-by: Agnès Toulet <[email protected]>

* fix: remove `workerCreationDelay`

* yarn lock update

Co-authored-by: Ryan McKinley <[email protected]>
Co-authored-by: Agnès Toulet <[email protected]>

Author: 3 people

default.json

@@ -33,7 +33,9 @@
     "headed": false,
 
     "mode": "default",
+    "emulateNetworkConditions": false,
     "clustering": {
+      "monitor": false,
       "mode": "browser",
       "maxConcurrency": 5,
       "timeout": 30

dev.json

@@ -33,7 +33,16 @@
     "headed": false,
 
     "mode": "default",
+    "emulateNetworkConditions": false,
+    "networkConditions": {
+      "downloadThroughput": 4000000,
+      "uploadThroughput": 4000000,
+      "latency": 100,
+      "offline": false
+    },
+
     "clustering": {
+      "monitor": true,
       "mode": "browser",
       "maxConcurrency": 5,
       "timeout": 30

package.json

@@ -14,6 +14,7 @@
     "prettier:write": "prettier --list-different \"**/*.ts\" --write",
     "precommit": "npm run eslint & npm run typecheck",
     "watch": "tsc-watch --onSuccess \"node build/app.js server --config=dev.json\"",
+    "watch:debug": "tsc-watch --onSuccess \"cross-env DEBUG=puppeteer-cluster:* node build/app.js server --config=dev.json\"",
     "build": "tsc",
     "start": "node build/app.js --config=dev.json"
   },
@@ -30,8 +31,9 @@
     "morgan": "^1.9.0",
     "on-finished": "^2.3.0",
     "prom-client": "^11.5.3",
-    "puppeteer": "^10.0.0",
+    "puppeteer": "^13.1.3",
     "puppeteer-cluster": "^0.22.0",
+    "poolpeteer": "^0.22.0",
     "sharp": "0.29.3",
     "unique-filename": "^1.1.0",
     "winston": "^3.2.1"
@@ -40,6 +42,7 @@
     "@grafana/eslint-config": "^2.5.0",
     "@types/express": "^4.11.1",
     "@types/node": "^14.14.41",
+    "cross-env": "7.0.3",
     "@typescript-eslint/eslint-plugin": "^4.32.0",
     "@typescript-eslint/parser": "^4.32.0",
     "eslint": "^7.32.0",

src/browser/browser.ts

@@ -148,6 +148,11 @@ export class Browser {
   }
 
   async preparePage(page: puppeteer.Page, options: RenderOptions) {
+    if (this.config.emulateNetworkConditions && this.config.networkConditions) {
+      const client = await page.target().createCDPSession();
+      await client.send('Network.emulateNetworkConditions', this.config.networkConditions);
+    }
+
     if (options.renderKey) {
       if (this.config.verboseLogging) {
         this.log.debug('Setting cookie for page', 'renderKey', options.renderKey, 'domain', options.domain);

src/browser/clustered.ts

@@ -1,4 +1,5 @@
-import { Cluster } from 'puppeteer-cluster';
+import { Cluster as PoolpeteerCluster } from 'poolpeteer';
+import { Cluster as PuppeteerCluster } from 'puppeteer-cluster';
 import { ImageRenderOptions, RenderOptions } from '../types';
 import { Browser, RenderResponse, RenderCSVResponse, Metrics } from './browser';
 import { Logger } from '../logger';
@@ -10,12 +11,17 @@ enum RenderType {
 }
 
 interface ClusterOptions {
+  groupId?: string;
   options: RenderOptions | ImageRenderOptions;
   renderType: RenderType;
 }
 
 type ClusterResponse = RenderResponse | RenderCSVResponse;
 
+const contextPerRenderKey = 'contextPerRenderKey';
+
+type Cluster<JobData = any, ReturnData = any> = PuppeteerCluster<JobData, ReturnData> | PoolpeteerCluster<JobData, ReturnData>;
+
 export class ClusteredBrowser extends Browser {
   cluster: Cluster<ClusterOptions, ClusterResponse>;
   clusteringConfig: ClusteringConfig;
@@ -25,21 +31,45 @@ export class ClusteredBrowser extends Browser {
     super(config, log, metrics);
 
     this.clusteringConfig = config.clustering;
-    this.concurrency = Cluster.CONCURRENCY_BROWSER;
+    this.concurrency = PuppeteerCluster.CONCURRENCY_BROWSER;
 
     if (this.clusteringConfig.mode === 'context') {
-      this.concurrency = Cluster.CONCURRENCY_CONTEXT;
+      this.concurrency = PuppeteerCluster.CONCURRENCY_CONTEXT;
+    }
+
+    if (this.clusteringConfig.mode === contextPerRenderKey) {
+      this.concurrency = PoolpeteerCluster.CONCURRENCY_CONTEXT_PER_REQUEST_GROUP;
     }
   }
 
-  async start(): Promise<void> {
+  shouldUsePoolpeteer(): boolean {
+    return this.clusteringConfig.mode === contextPerRenderKey;
+  }
+
+  async createCluster(): Promise<Cluster<ClusterOptions, ClusterResponse>> {
     const launcherOptions = this.getLauncherOptions({});
-    this.cluster = await Cluster.launch({
+
+    const clusterOptions = {
       concurrency: this.concurrency,
+      workerShutdownTimeout: 5000,
+      monitor: this.clusteringConfig.monitor,
       maxConcurrency: this.clusteringConfig.maxConcurrency,
       timeout: this.clusteringConfig.timeout * 1000,
       puppeteerOptions: launcherOptions,
-    });
+    };
+
+    // TODO use poolpeteer by default after initial release and testing (8.5?)
+    if (this.shouldUsePoolpeteer()) {
+      this.log.debug('Launching Browser cluster with poolpeteer');
+      return PoolpeteerCluster.launch(clusterOptions);
+    }
+
+    this.log.debug('Launching Browser cluster with puppeteer-cluster');
+    return PuppeteerCluster.launch(clusterOptions);
+  }
+
+  async start(): Promise<void> {
+    this.cluster = await this.createCluster();
     await this.cluster.task(async ({ page, data }) => {
       if (data.options.timezone) {
         // set timezone
@@ -61,13 +91,21 @@ export class ClusteredBrowser extends Browser {
     });
   }
 
+  private getGroupId = (options: ImageRenderOptions | RenderOptions) => {
+    if (this.clusteringConfig.mode === contextPerRenderKey) {
+      return `${options.domain}${options.renderKey}`;
+    }
+
+    return undefined;
+  };
+
   async render(options: ImageRenderOptions): Promise<RenderResponse> {
     this.validateImageOptions(options);
-    return this.cluster.execute({ options, renderType: RenderType.PNG });
+    return this.cluster.execute({ groupId: this.getGroupId(options), options, renderType: RenderType.PNG });
   }
 
   async renderCSV(options: RenderOptions): Promise<RenderCSVResponse> {
     this.validateRenderOptions(options);
-    return this.cluster.execute({ options, renderType: RenderType.CSV });
+    return this.cluster.execute({ groupId: this.getGroupId(options), options, renderType: RenderType.CSV });
   }
 }

src/config.ts

@@ -1,11 +1,20 @@
 import * as fs from 'fs';
 
 export interface ClusteringConfig {
+  monitor: boolean;
   mode: string;
   maxConcurrency: number;
   timeout: number;
 }
 
+// https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-emulateNetworkConditions
+type NetworkConditions = {
+  offline: boolean;
+  downloadThroughput: number;
+  uploadThroughput: number;
+  latency: number;
+};
+
 export interface RenderingConfig {
   chromeBin?: string;
   args: string[];
@@ -24,6 +33,8 @@ export interface RenderingConfig {
   dumpio: boolean;
   timingMetrics: boolean;
   headed?: boolean;
+  networkConditions?: NetworkConditions;
+  emulateNetworkConditions: boolean;
 }
 
 export interface MetricsConfig {
@@ -78,10 +89,12 @@ const defaultRenderingConfig: RenderingConfig = {
   maxDeviceScaleFactor: 4,
   mode: 'default',
   clustering: {
+    monitor: false,
     mode: 'browser',
     maxConcurrency: 5,
     timeout: 30,
   },
+  emulateNetworkConditions: false,
   verboseLogging: false,
   dumpio: false,
   timingMetrics: false,