feat: file downloads

fix: download-progress not wired up

feat: add ability to download actual file

Author: blakebyrnes

client/lib/CoreTab.ts

@@ -8,6 +8,8 @@ import IWaitForOptions from '@secret-agent/interfaces/IWaitForOptions';
 import IScreenshotOptions from '@secret-agent/interfaces/IScreenshotOptions';
 import IFrameMeta from '@secret-agent/interfaces/IFrameMeta';
 import IFileChooserPrompt from '@secret-agent/interfaces/IFileChooserPrompt';
+import IDownload from '@secret-agent/interfaces/IDownload';
+import { URL } from 'url';
 import CoreCommandQueue from './CoreCommandQueue';
 import CoreEventHeap from './CoreEventHeap';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
@@ -17,6 +19,7 @@ import ConnectionToCore from '../connections/ConnectionToCore';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
 import { createDialog } from './Dialog';
 import CoreSession from './CoreSession';
+import Download, { createDownload } from './Download';
 
 export default class CoreTab implements IJsPathEventTarget {
   public tabId: number;
@@ -32,6 +35,7 @@ export default class CoreTab implements IJsPathEventTarget {
   private readonly connection: ConnectionToCore;
   private readonly mainFrameId: number;
   private readonly coreSession: CoreSession;
+  private readonly downloadsById = new Map<string, Download>();
 
   constructor(
     meta: ISessionMeta & { sessionName: string },
@@ -57,6 +61,7 @@ export default class CoreTab implements IJsPathEventTarget {
     this.eventHeap.registerEventInterceptors({
       resource: createResource.bind(null, resolvedThis),
       dialog: createDialog.bind(null, resolvedThis),
+      download: this.createDownload.bind(this),
     });
   }
 
@@ -176,4 +181,20 @@ export default class CoreTab implements IJsPathEventTarget {
     const session = this.connection.getSession(this.sessionId);
     session?.removeTab(this);
   }
+
+  public async deleteDownload(id: string): Promise<void> {
+    await this.commandQueue.run('Tab.deleteDownload', id);
+  }
+
+  private createDownload(download: IDownload): Download {
+    const newDownload = createDownload(this, download);
+    this.downloadsById.set(download.id, newDownload);
+    newDownload.downloadUrl = this.connection.hostOrError.then(host => {
+      if (host instanceof Error) return null;
+      const hostUrl = new URL(download.downloadPath, host);
+      hostUrl.protocol = 'http:';
+      return hostUrl.href;
+    });
+    return newDownload;
+  }
 }

client/lib/Download.ts

@@ -0,0 +1,98 @@
+import StateMachine from 'awaited-dom/base/StateMachine';
+import Resolvable from '@secret-agent/commons/Resolvable';
+import IDownload, { IDownloadState } from '@secret-agent/interfaces/IDownload';
+import { httpGet } from '@secret-agent/commons/downloadFile';
+import CoreTab from './CoreTab';
+
+const { getState, setState } = StateMachine<Download, IState>();
+
+interface IState {
+  coreTab: CoreTab;
+  downloadPromise: Resolvable<void>;
+  complete: boolean;
+}
+
+export default class Download {
+  id: string;
+  url: string;
+  path: string;
+  suggestedFilename: string;
+
+  progress = 0;
+  totalBytes = 0;
+  canceled = false;
+
+  #downloadUrl: Promise<string>;
+
+  get complete(): boolean {
+    return getState(this).complete;
+  }
+
+  set complete(value) {
+    setState(this, { complete: value });
+    if (value) {
+      getState(this).downloadPromise.resolve();
+    }
+  }
+
+  waitForFinished(): Promise<void> {
+    return getState(this).downloadPromise.promise;
+  }
+
+  set downloadUrl(value: Promise<string>) {
+    this.#downloadUrl = value;
+  }
+
+  async delete(): Promise<void> {
+    const coreTab = await getState(this).coreTab;
+    await coreTab.deleteDownload(this.id);
+  }
+
+  async data(): Promise<Buffer> {
+    await this.waitForFinished();
+    const url = await this.#downloadUrl;
+    const downloaderPromise = new Resolvable<Buffer>();
+    const request = httpGet(url, async response => {
+      if (response.statusCode !== 200) {
+        const error = new Error(
+          `Download failed: server returned code ${response.statusCode}. URL: ${url}`,
+        );
+        // consume response data to free up memory
+        response.resume();
+        downloaderPromise.reject(error);
+        return;
+      }
+      const buffer: Buffer[] = [];
+      for await (const chunk of response) {
+        buffer.push(chunk);
+      }
+      downloaderPromise.resolve(Buffer.concat(buffer));
+    });
+    request.once('error', downloaderPromise.reject);
+    return downloaderPromise.promise;
+  }
+}
+
+export function createDownload(coreTab: CoreTab, data: IDownload): Download {
+  const download = new Download();
+  Object.assign(download, data);
+  setState(download, {
+    coreTab,
+    downloadPromise: new Resolvable<void>(),
+  });
+
+  function onDownloadProgress(progress: IDownloadState): void {
+    if (progress.complete) {
+      coreTab
+        .removeEventListener(['downloads', progress.id], 'download-progress', onDownloadProgress)
+        .catch(() => null);
+    }
+    Object.assign(download, progress);
+  }
+
+  coreTab
+    .addEventListener(['downloads', download.id], 'download-progress', onDownloadProgress)
+    .catch(() => null);
+
+  return download;
+}

client/lib/Tab.ts

@@ -34,6 +34,7 @@ import CoreFrameEnvironment from './CoreFrameEnvironment';
 import IAwaitedOptions from '../interfaces/IAwaitedOptions';
 import Dialog from './Dialog';
 import FileChooser from './FileChooser';
+import Download from './Download';
 
 const awaitedPathState = StateMachine<
   any,
@@ -52,6 +53,7 @@ export interface IState {
 interface IEventType {
   resource: Resource | WebsocketResource;
   dialog: Dialog;
+  download: Download;
 }
 
 const propertyKeys: (keyof Tab)[] = [

commons/downloadFile.ts

@@ -60,7 +60,7 @@ export function httpGet(
   return request;
 }
 
-function getRequestOptionsWithProxy(url: string): RequestOptions {
+export function getRequestOptionsWithProxy(url: string): RequestOptions {
   const urlParsed = parse(url);
 
   const options: https.RequestOptions = {

copyfiles.js

@@ -16,7 +16,13 @@ const copyArgs = [
   '.*ignore',
 ];
 if (isBuild) {
-  copyArgs.push('testing/*/**', 'core/test/html/**', 'puppet/test/*/**', 'yarn.lock');
+  copyArgs.push(
+    'testing/*/**',
+    'core/test/html/**',
+    'full-client/test/html/**',
+    'puppet/test/*/**',
+    'yarn.lock',
+  );
 }
 
 for (const workspace of workspaces) {

core/lib/AwaitedEventListener.ts

@@ -50,6 +50,14 @@ export default class AwaitedEventListener {
         // need to give client time to register function sending events
         process.nextTick(() => tab.sessionState.onWebsocketMessages(resourceId, listener.listenFn));
       }
+      if (listener.jsPath && listener.jsPath[0] === 'downloads') {
+        const downloadId = listener.jsPath[1] as string;
+        listener.listenFn = download => {
+          if (download.id === downloadId) this.triggerListenersWithId(listenerId, download);
+        };
+        tab.on('download-progress', listener.listenFn);
+        process.nextTick(() => listener.listenFn(tab.session.downloadsById.get(downloadId)));
+      }
     } else if (type && tab) {
       if (type !== 'close') {
         listener.listenFn = this.triggerListenersWithType.bind(this, type);

core/lib/CorePlugins.ts

@@ -26,6 +26,7 @@ import { PluginTypes } from '@secret-agent/interfaces/IPluginTypes';
 import requirePlugins from '@secret-agent/plugin-utils/lib/utils/requirePlugins';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
 import IDeviceProfile from '@secret-agent/interfaces/IDeviceProfile';
+import IPuppetContext from '@secret-agent/interfaces/IPuppetContext';
 import Core from '../index';
 
 const DefaultBrowserEmulatorId = 'default-browser-emulator';
@@ -136,6 +137,12 @@ export default class CorePlugins implements ICorePlugins {
     this.instances.filter(p => p.onTlsConfiguration).forEach(p => p.onTlsConfiguration(settings));
   }
 
+  public async onNewPuppetContext(context: IPuppetContext): Promise<void> {
+    await Promise.all(
+      this.instances.filter(p => p.onNewPuppetContext).map(p => p.onNewPuppetContext(context)),
+    );
+  }
+
   public async onNewPuppetPage(page: IPuppetPage): Promise<void> {
     await Promise.all(
       this.instances.filter(p => p.onNewPuppetPage).map(p => p.onNewPuppetPage(page)),

core/lib/Session.ts

@@ -21,6 +21,7 @@ import IViewport from '@secret-agent/interfaces/IViewport';
 import IJsPathResult from '@secret-agent/interfaces/IJsPathResult';
 import ISessionCreateOptions from '@secret-agent/interfaces/ISessionCreateOptions';
 import IGeolocation from '@secret-agent/interfaces/IGeolocation';
+import IDownload, { IDownloadState } from '@secret-agent/interfaces/IDownload';
 import SessionState from './SessionState';
 import AwaitedEventListener from './AwaitedEventListener';
 import GlobalPool from './GlobalPool';
@@ -60,6 +61,7 @@ export default class Session extends TypedEventEmitter<{
   public userProfile?: IUserProfile;
 
   public tabsById = new Map<number, Tab>();
+  public downloadsById = new Map<string, IDownload & Partial<IDownloadState>>();
 
   public get isClosing() {
     return this._isClosing;

core/lib/Tab.ts

@@ -23,6 +23,10 @@ import IFrameMeta from '@secret-agent/interfaces/IFrameMeta';
 import { LoadStatus } from '@secret-agent/interfaces/INavigation';
 import IPuppetDialog from '@secret-agent/interfaces/IPuppetDialog';
 import IFileChooserPrompt from '@secret-agent/interfaces/IFileChooserPrompt';
+import IDownload, { IDownloadState } from '@secret-agent/interfaces/IDownload';
+import { encode } from 'querystring';
+import * as Fs from 'fs';
+import { existsAsync } from '@secret-agent/commons/fileUtils';
 import FrameNavigations from './FrameNavigations';
 import CommandRecorder from './CommandRecorder';
 import FrameEnvironment from './FrameEnvironment';
@@ -108,7 +112,7 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     windowOpenParams?: { url: string; windowName: string; loaderId: string },
   ) {
     super();
-    this.setEventsToLog(['child-tab-created', 'close']);
+    this.setEventsToLog(['child-tab-created', 'close', 'dialog', 'download', 'download-progress']);
     this.id = session.nextTabId();
     this.logger = log.createChild(module, {
       tabId: this.id,
@@ -139,6 +143,7 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     this.commandRecorder = new CommandRecorder(this, this.session, this.id, this.mainFrameId, [
       this.focus,
       this.dismissDialog,
+      this.deleteDownload,
       this.getFrameEnvironments,
       this.goto,
       this.goBack,
@@ -437,6 +442,12 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     return this.puppetPage.dismissDialog(accept, promptText);
   }
 
+  public async deleteDownload(id: string): Promise<void> {
+    const download = this.session.downloadsById.get(id);
+    if (!download) return;
+    if (await existsAsync(download.path)) await Fs.promises.unlink(download.path);
+  }
+
   public async waitForNewTab(options: IWaitForOptions = {}): Promise<Tab> {
     // last command is the one running right now
     const startCommandId = Number.isInteger(options.sinceCommandId)
@@ -634,6 +645,9 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     page.on('page-callback-triggered', this.onPageCallback.bind(this));
     page.on('dialog-opening', this.onDialogOpening.bind(this));
     page.on('filechooser', this.onFileChooser.bind(this));
+    page.on('download-started', this.onDownloadStarted.bind(this));
+    page.on('download-progress', this.onDownloadProgress.bind(this));
+    page.on('download-finished', this.onDownloadFinished.bind(this));
 
     // resource requested should registered before navigations so we can grab nav on new tab anchor clicks
     page.on('resource-will-be-requested', this.onResourceWillBeRequested.bind(this), true);
@@ -910,6 +924,42 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     this.sessionState.captureError(this.id, this.mainFrameId, `events.error`, error);
   }
 
+  /////// DOWNLOADS ////////////////////////////////////////////////////////////////////////////////
+
+  private onDownloadStarted(event: IPuppetPageEvents['download-started']): void {
+    const broadcast = event as IDownload;
+    broadcast.downloadPath = `/downloads?${encode({
+      id: event.id,
+      sessionId: this.sessionId,
+    })}`;
+    this.session.downloadsById.set(event.id, broadcast);
+    this.emit('download', broadcast);
+  }
+
+  private onDownloadProgress(event: IPuppetPageEvents['download-progress']): void {
+    const state = this.session.downloadsById.get(event.id);
+    const broadcast = {
+      ...event,
+      canceled: false,
+      complete: false,
+    };
+    if (state) Object.assign(state, broadcast);
+
+    this.emit('download-progress', broadcast);
+  }
+
+  private onDownloadFinished(event: IPuppetPageEvents['download-finished']): void {
+    const state = this.session.downloadsById.get(event.id);
+    const broadcast = {
+      ...event,
+      complete: true,
+      progress: 100,
+    };
+    if (state) Object.assign(state, broadcast);
+
+    this.emit('download-progress', broadcast);
+  }
+
   /////// DIALOGS //////////////////////////////////////////////////////////////////////////////////
 
   private onDialogOpening(event: IPuppetPageEvents['dialog-opening']): void {
@@ -943,6 +993,8 @@ interface ITabEventParams {
   'resource-requested': IResourceMeta;
   resource: IResourceMeta;
   dialog: IPuppetDialog;
+  download: IDownload;
+  'download-progress': IDownloadState;
   'websocket-message': IWebsocketResourceMessage;
   'child-tab-created': Tab;
 }

core/server/ConnectionToClient.ts

@@ -294,12 +294,14 @@ export default class ConnectionToClient extends TypedEventEmitter<{
       }
 
       const session = Session.get(meta.sessionId);
-      session.sessionState.nextCommandMeta = commandMeta;
       if (!session) {
+        if (method === 'close' || method === 'removeEventListener') return;
+
         return new SessionClosedOrMissingError(
           `The requested command (${command}) references a session that is closed or invalid.`,
         );
       }
+      session.sessionState.nextCommandMeta = commandMeta;
 
       return await this[method](meta, ...args);
     }