feat: file downloads

Author: blakebyrnes

client/lib/CoreTab.ts

@@ -8,6 +8,7 @@ import IWaitForOptions from '@secret-agent/interfaces/IWaitForOptions';
 import IScreenshotOptions from '@secret-agent/interfaces/IScreenshotOptions';
 import IFrameMeta from '@secret-agent/interfaces/IFrameMeta';
 import IFileChooserPrompt from '@secret-agent/interfaces/IFileChooserPrompt';
+import IDownload, { IDownloadState } from '@secret-agent/interfaces/IDownload';
 import CoreCommandQueue from './CoreCommandQueue';
 import CoreEventHeap from './CoreEventHeap';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
@@ -17,6 +18,7 @@ import ConnectionToCore from '../connections/ConnectionToCore';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
 import { createDialog } from './Dialog';
 import CoreSession from './CoreSession';
+import Download, { createDownload } from './Download';
 
 export default class CoreTab implements IJsPathEventTarget {
   public tabId: number;
@@ -32,6 +34,7 @@ export default class CoreTab implements IJsPathEventTarget {
   private readonly connection: ConnectionToCore;
   private readonly mainFrameId: number;
   private readonly coreSession: CoreSession;
+  private readonly downloadsById = new Map<string, Download>();
 
   constructor(
     meta: ISessionMeta & { sessionName: string },
@@ -57,6 +60,7 @@ export default class CoreTab implements IJsPathEventTarget {
     this.eventHeap.registerEventInterceptors({
       resource: createResource.bind(null, resolvedThis),
       dialog: createDialog.bind(null, resolvedThis),
+      'download-started': this.createDownload.bind(resolvedThis),
     });
   }
 
@@ -176,4 +180,16 @@ export default class CoreTab implements IJsPathEventTarget {
     const session = this.connection.getSession(this.sessionId);
     session?.removeTab(this);
   }
+
+  private createDownload(download: IDownload): Download {
+    const newDownload = createDownload(Promise.resolve(this), download);
+    this.downloadsById.set(download.id, newDownload);
+    return newDownload;
+  }
+
+  private onDownloadProgress(data: IDownloadState): void {
+    const download = this.downloadsById.get(data.id);
+    if (!download) return;
+    Object.assign(download, data);
+  }
 }

client/lib/Download.ts

@@ -0,0 +1,50 @@
+import StateMachine from 'awaited-dom/base/StateMachine';
+import Resolvable from '@secret-agent/commons/Resolvable';
+import IDownload from '@secret-agent/interfaces/IDownload';
+import CoreTab from './CoreTab';
+
+const { getState, setState } = StateMachine<Download, IState>();
+
+interface IState {
+  coreTab: Promise<CoreTab>;
+  downloadPromise: Resolvable<void>;
+  complete: boolean;
+}
+
+export default class Download {
+  id: string;
+  url: string;
+  path: string;
+  suggestedFilename: string;
+
+  progress: number;
+  totalBytes: number;
+  canceled: boolean;
+
+  get complete(): boolean {
+    return getState(this).complete;
+  }
+
+  set complete(value) {
+    setState(this, { complete: value });
+    if (value) getState(this).downloadPromise.resolve();
+  }
+
+  waitForFinished(): Promise<void> {
+    return getState(this).downloadPromise.promise;
+  }
+
+  async saveAs(): Promise<Buffer> {
+    // todo: add streaming ability
+  }
+}
+
+export function createDownload(coreTab: Promise<CoreTab>, data: IDownload): Download {
+  const download = new Download();
+  Object.assign(download, data);
+  setState(download, {
+    coreTab,
+    downloadPromise: new Resolvable<void>(),
+  });
+  return download;
+}

client/lib/Tab.ts

@@ -35,6 +35,7 @@ import CoreFrameEnvironment from './CoreFrameEnvironment';
 import IAwaitedOptions from '../interfaces/IAwaitedOptions';
 import Dialog from './Dialog';
 import FileChooser from './FileChooser';
+import Download from './Download';
 
 const awaitedPathState = StateMachine<
   any,
@@ -53,6 +54,7 @@ export interface IState {
 interface IEventType {
   resource: Resource | WebsocketResource;
   dialog: Dialog;
+  download: Download;
 }
 
 const propertyKeys: (keyof Tab)[] = [

core/lib/CorePlugins.ts

@@ -26,6 +26,7 @@ import { PluginTypes } from '@secret-agent/interfaces/IPluginTypes';
 import requirePlugins from '@secret-agent/plugin-utils/lib/utils/requirePlugins';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
 import IDeviceProfile from '@secret-agent/interfaces/IDeviceProfile';
+import IPuppetContext from '@secret-agent/interfaces/IPuppetContext';
 import Core from '../index';
 
 const DefaultBrowserEmulatorId = 'default-browser-emulator';
@@ -136,6 +137,12 @@ export default class CorePlugins implements ICorePlugins {
     this.instances.filter(p => p.onTlsConfiguration).forEach(p => p.onTlsConfiguration(settings));
   }
 
+  public async onNewPuppetContext(context: IPuppetContext): Promise<void> {
+    await Promise.all(
+      this.instances.filter(p => p.onNewPuppetContext).map(p => p.onNewPuppetContext(context)),
+    );
+  }
+
   public async onNewPuppetPage(page: IPuppetPage): Promise<void> {
     await Promise.all(
       this.instances.filter(p => p.onNewPuppetPage).map(p => p.onNewPuppetPage(page)),

core/lib/Tab.ts

@@ -23,6 +23,7 @@ import IFrameMeta from '@secret-agent/interfaces/IFrameMeta';
 import { LoadStatus } from '@secret-agent/interfaces/INavigation';
 import IPuppetDialog from '@secret-agent/interfaces/IPuppetDialog';
 import IFileChooserPrompt from '@secret-agent/interfaces/IFileChooserPrompt';
+import IDownload, { IDownloadState } from '@secret-agent/interfaces/IDownload';
 import FrameNavigations from './FrameNavigations';
 import CommandRecorder from './CommandRecorder';
 import FrameEnvironment from './FrameEnvironment';
@@ -622,6 +623,9 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     page.on('page-callback-triggered', this.onPageCallback.bind(this));
     page.on('dialog-opening', this.onDialogOpening.bind(this));
     page.on('filechooser', this.onFileChooser.bind(this));
+    page.on('download-started', this.onDownloadStarted.bind(this));
+    page.on('download-progress', this.onDownloadProgress.bind(this));
+    page.on('download-finished', this.onDownloadFinished.bind(this));
 
     // resource requested should registered before navigations so we can grab nav on new tab anchor clicks
     page.on('resource-will-be-requested', this.onResourceWillBeRequested.bind(this), true);
@@ -848,6 +852,28 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     this.sessionState.captureError(this.id, this.mainFrameId, `events.error`, error);
   }
 
+  /////// DOWNLOADS ////////////////////////////////////////////////////////////////////////////////
+
+  private onDownloadStarted(event: IPuppetPageEvents['download-started']): void {
+    this.emit('download-started', event);
+  }
+
+  private onDownloadProgress(event: IPuppetPageEvents['download-progress']): void {
+    this.emit('download-progress', {
+      ...event,
+      canceled: false,
+      complete: false,
+    });
+  }
+
+  private onDownloadFinished(event: IPuppetPageEvents['download-finished']): void {
+    this.emit('download-progress', {
+      ...event,
+      complete: true,
+      progress: 100,
+    });
+  }
+
   /////// DIALOGS //////////////////////////////////////////////////////////////////////////////////
 
   private onDialogOpening(event: IPuppetPageEvents['dialog-opening']): void {
@@ -881,6 +907,8 @@ interface ITabEventParams {
   'resource-requested': IResourceMeta;
   resource: IResourceMeta;
   dialog: IPuppetDialog;
+  'download-started': IDownload;
+  'download-progress': IDownloadState;
   'websocket-message': IWebsocketResourceMessage;
   'child-tab-created': Tab;
 }

interfaces/ICorePlugin.ts

@@ -16,6 +16,7 @@ import IViewport from './IViewport';
 import IGeolocation from './IGeolocation';
 import IDeviceProfile from './IDeviceProfile';
 import IHttp2ConnectSettings from './IHttp2ConnectSettings';
+import IPuppetContext from './IPuppetContext';
 
 export default interface ICorePlugin
   extends ICorePluginMethods,
@@ -116,6 +117,7 @@ export interface IBrowserEmulatorMethods {
   beforeHttpRequest?(request: IHttpResourceLoadDetails): Promise<any> | void;
   beforeHttpResponse?(resource: IHttpResourceLoadDetails): Promise<any> | void;
 
+  onNewPuppetContext?(context: IPuppetContext): Promise<any>;
   onNewPuppetPage?(page: IPuppetPage): Promise<any>;
   onNewPuppetWorker?(worker: IPuppetWorker): Promise<any>;
 

interfaces/IDownload.ts

@@ -0,0 +1,14 @@
+export default interface IDownload {
+  id: string;
+  path: string;
+  suggestedFilename: string;
+  url: string;
+}
+
+export interface IDownloadState {
+  id: string;
+  totalBytes: number;
+  complete: boolean;
+  progress: number;
+  canceled: boolean;
+}

interfaces/IPuppetContext.ts

@@ -22,6 +22,8 @@ export default interface IPuppetContext extends ITypedEventEmitter<IPuppetContex
     cookies: (Omit<ICookie, 'expires'> & { expires?: string | Date | number })[],
     origins?: string[],
   ): Promise<void>;
+
+  enableDownloads(downloadsPath: string): Promise<void>;
 }
 
 export interface IPuppetPageOptions {

interfaces/IPuppetPage.ts

@@ -65,4 +65,12 @@ export interface IPuppetPageEvents extends IPuppetFrameManagerEvents, IPuppetNet
   filechooser: { frameId: string; selectMultiple: boolean; objectId: string };
   'page-error': { frameId: string; error: Error };
   'page-callback-triggered': { name: string; frameId: string; payload: any };
+  'download-started': {
+    id: string;
+    path: string;
+    suggestedFilename: string;
+    url: string;
+  };
+  'download-progress': { id: string; totalBytes: number; progress: number };
+  'download-finished': { id: string; totalBytes: number; canceled: boolean };
 }

plugin-utils/lib/BrowserEngine.ts

@@ -10,6 +10,7 @@ export default class BrowserEngine implements IBrowserEngine {
   public fullVersion: string;
   public executablePath: string;
   public executablePathEnvVar: string;
+  public userDataDir?: string;
 
   public readonly launchArguments: string[] = [];
 

plugins/default-browser-emulator/index.ts

@@ -15,6 +15,9 @@ import IUserAgentOption from '@secret-agent/interfaces/IUserAgentOption';
 import BrowserEngine from '@secret-agent/plugin-utils/lib/BrowserEngine';
 import IGeolocation from '@secret-agent/interfaces/IGeolocation';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
+import IPuppetContext from '@secret-agent/interfaces/IPuppetContext';
+import * as Path from 'path';
+import * as os from 'os';
 import Viewports from './lib/Viewports';
 import setWorkerDomOverrides from './lib/setWorkerDomOverrides';
 import setPageDomOverrides from './lib/setPageDomOverrides';
@@ -43,6 +46,8 @@ const dataLoader = new DataLoader(__dirname);
 export const latestBrowserEngineId = 'chrome-88-0';
 export const latestChromeBrowserVersion = { major: '88', minor: '0' };
 
+let sessionDirCounter = 0;
+
 @BrowserEmulatorClassDecorator
 export default class DefaultBrowserEmulator extends BrowserEmulator {
   public static id = dataLoader.pkg.name.replace('@secret-agent/', '');
@@ -51,6 +56,7 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
   public locale: string;
   public viewport: IViewport;
   public geolocation: IGeolocation;
+  public userDataDir: string;
 
   protected readonly data: IBrowserData;
   private readonly domOverridesBuilder: DomOverridesBuilder;
@@ -109,6 +115,10 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
     configureHttp2Session(this, this.data, request, settings);
   }
 
+  public async onNewPuppetContext(context: IPuppetContext): Promise<any> {
+    await context.enableDownloads(this.userDataDir);
+  }
+
   public onNewPuppetPage(page: IPuppetPage): Promise<any> {
     // Don't await here! we want to queue all these up to run before the debugger resumes
     const devtools = page.devtoolsSession;
@@ -154,6 +164,13 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
       disableDevtools?: boolean;
     },
   ): void {
+    const dataDir = Path.join(
+      os.tmpdir(),
+      browserEngine.fullVersion.replace('.', '-'),
+      `${String(Date.now()).substr(0, 10)}-${(sessionDirCounter += 1)}`,
+    );
+    browserEngine.userDataDir = dataDir;
+
     configureBrowserLaunchArgs(browserEngine, options);
   }
 }

plugins/default-browser-emulator/lib/helpers/configureBrowserLaunchArgs.ts

@@ -1,10 +1,6 @@
-import * as Path from 'path';
-import * as os from 'os';
 import BrowserEngine from '@secret-agent/plugin-utils/lib/BrowserEngine';
 import { defaultScreen } from '../Viewports';
 
-let sessionDirCounter = 0;
-
 export function configureBrowserLaunchArgs(
   engine: BrowserEngine,
   options: {
@@ -60,12 +56,7 @@ export function configureBrowserLaunchArgs(
   );
 
   if (options.showBrowser) {
-    const dataDir = Path.join(
-      os.tmpdir(),
-      engine.fullVersion.replace('.', '-'),
-      `${String(Date.now()).substr(0, 10)}-${(sessionDirCounter += 1)}`,
-    );
-    engine.launchArguments.push(`--user-data-dir=${dataDir}`); // required to allow multiple browsers to be headed
+    engine.launchArguments.push(`--user-data-dir=${engine.userDataDir}`); // required to allow multiple browsers to be headed
 
     if (!options.disableDevtools) engine.launchArguments.push('--auto-open-devtools-for-tabs');
   } else {

puppet-chrome/lib/Browser.ts

@@ -56,7 +56,9 @@ export class Browser extends TypedEventEmitter<IBrowserEvents> implements IPuppe
       ...proxySettings,
     });
 
-    return new BrowserContext(this, plugins, browserContextId, logger, proxy);
+    const context = new BrowserContext(this, plugins, browserContextId, logger, proxy);
+    if (plugins.onNewPuppetContext) await plugins.onNewPuppetContext(context);
+    return context;
   }
 
   public async getFeatures(): Promise<{

puppet-chrome/lib/BrowserContext.ts

@@ -28,21 +28,20 @@ import { Page } from './Page';
 import { Browser } from './Browser';
 import { DevtoolsSession } from './DevtoolsSession';
 import Frame from './Frame';
-
 import CookieParam = Protocol.Network.CookieParam;
 import TargetInfo = Protocol.Target.TargetInfo;
 
 export class BrowserContext
   extends TypedEventEmitter<IPuppetContextEvents>
-  implements IPuppetContext
-{
+  implements IPuppetContext {
   public logger: IBoundLog;
 
   public workersById = new Map<string, IPuppetWorker>();
   public pagesById = new Map<string, Page>();
   public plugins: ICorePlugins;
   public proxy: IProxyConnectionOptions;
   public readonly id: string;
+  public downloadsPath?: string;
 
   private attachedTargetIds = new Set<string>();
   private pageOptionsByTargetId = new Map<string, IPuppetPageOptions>();
@@ -119,6 +118,15 @@ export class BrowserContext
     }
   }
 
+  async enableDownloads(downloadsPath: string): Promise<any> {
+    this.downloadsPath = downloadsPath;
+    await this.sendWithBrowserDevtoolsSession('Browser.setDownloadBehavior', {
+      behavior: 'allowAndName',
+      browserContextId: this.id,
+      downloadPath: downloadsPath,
+    });
+  }
+
   initializePage(page: Page): Promise<any> {
     if (this.pageOptionsByTargetId.get(page.targetId)?.runPageScripts === false) return;