.eslintrc.js

@@ -100,6 +100,7 @@ module.exports = {
         'no-console': 'off',
         '@typescript-eslint/no-unused-vars': 'off',
         'import/no-extraneous-dependencies': 'off',
+        '@typescript-eslint/no-shadow': 'off',
       },
     },
     {

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: ulixee

.github/workflows/new-version.yml

@@ -44,6 +44,23 @@ jobs:
           tag_name="${GITHUB_REF##*/}"
           hub release edit "${assets[@]}" -m "" "$tag_name"
 
+  website:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v1
+
+      - name: Install Node.js, NPM and Yarn
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      - name: Yarn install
+        run: yarn install --network-timeout 1000000
+        env:
+          SA_SKIP_DOWNLOAD: true
+
       - name: Build Webite
         run: yarn build
         working-directory: ./website

CHANGELOG.md

@@ -3,6 +3,142 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.6.5](https://github.com/ulixee/secret-agent/compare/v1.6.4...v1.6.5) (2022-05-25)
+
+
+### Bug Fixes
+
+* **cookies:** sub domain cookies not saving ([712808d](https://github.com/ulixee/secret-agent/commit/712808d745ab9057ed0ad66edb4e249de63b276f))
+* **core:** file picker not working after profile ([4f7c737](https://github.com/ulixee/secret-agent/commit/4f7c737a50dc1b5683bdf551c27fea8368f26f31))
+* **core:** speed for many user profile sites ([54a4061](https://github.com/ulixee/secret-agent/commit/54a406141ff851875d14e3ae342e00f76d240b1e))
+* dependencies ([65cae71](https://github.com/ulixee/secret-agent/commit/65cae71af84e00d1dca4a74ac45318af43641a1c))
+* github no longer allowing git prefix ([9693c89](https://github.com/ulixee/secret-agent/commit/9693c89267eeaeb48f98f91a363658230010210b))
+* lint issues ([db3f353](https://github.com/ulixee/secret-agent/commit/db3f3535636707ed2fd9c7f5dcc29f7d4bd07cc7))
+
+
+
+
+
+## [1.6.4](https://github.com/ulixee/secret-agent/compare/v1.6.3...v1.6.4) (2022-01-24)
+
+
+### Bug Fixes
+
+* **mitm:** lookup public ip should use https ([5a68228](https://github.com/ulixee/secret-agent/commit/5a6822851a71aff940900ea4f41365c997314ee2))
+
+
+
+
+
+## [1.6.3](https://github.com/ulixee/secret-agent/compare/v1.6.2...v1.6.3) (2022-01-14)
+
+
+### Bug Fixes
+
+* **client:** wait for all command queue flushes ([95664f5](https://github.com/ulixee/secret-agent/commit/95664f570b6b7e233ea6b6227ad5f363cce52287))
+* **core:** cleanup event listener memory ([6f7d7bb](https://github.com/ulixee/secret-agent/commit/6f7d7bb449714876ab9a90ec4697c012d8c7bff3))
+* **core:** in page nav not resolving resource ([2539d78](https://github.com/ulixee/secret-agent/commit/2539d7809d0709d709cff9bba48f411361e8ebc9))
+
+
+
+
+
+## [1.6.2](https://github.com/ulixee/secret-agent/compare/v1.6.1...v1.6.2) (2021-12-20)
+
+
+### Bug Fixes
+
+* **client:** reload should return a resource ([83b80b5](https://github.com/ulixee/secret-agent/commit/83b80b5b3d70bc06344fb64d31fed1434ac98114)), closes [#393](https://github.com/ulixee/secret-agent/issues/393)
+* **core:** navigations directly to hash fixed ([c02f61b](https://github.com/ulixee/secret-agent/commit/c02f61b549d34c438614ab8f63c8fce17af60d2c)), closes [#404](https://github.com/ulixee/secret-agent/issues/404)
+
+
+### Features
+
+* **client:** return resource in waitForLocation ([253b33b](https://github.com/ulixee/secret-agent/commit/253b33b3528952432528949ab0dc2c8b4cf9a50c))
+* **core:** dialogs should run out of command line ([52a737f](https://github.com/ulixee/secret-agent/commit/52a737ff7ca380b00a1f156aa7dacf5e9af0bcfb)), closes [#397](https://github.com/ulixee/secret-agent/issues/397)
+
+
+
+
+
+## [1.6.1](https://github.com/ulixee/secret-agent/compare/v1.5.15...v1.6.1) (2021-12-14)
+
+
+### Bug Fixes
+
+* **core:** frame navigations on redirects ([de3ea24](https://github.com/ulixee/secret-agent/commit/de3ea248a0fda5b8d08c7de961bac4f9fdc0b83e))
+* **core:** handle detached when elements not there ([5b46bd9](https://github.com/ulixee/secret-agent/commit/5b46bd97cf1ca1aa3bc0fed625064764726872f4))
+* **puppet:** extract navigation loader ([77d8e2e](https://github.com/ulixee/secret-agent/commit/77d8e2e34286469a2df1d9e00e631c6e773760eb))
+* **replay:** fix ability to replay data attributes ([9440995](https://github.com/ulixee/secret-agent/commit/944099560870c8479f0be59426fa194daf17ab4d))
+
+
+### Features
+
+* **plugins:** add frames to execute js ([65258a3](https://github.com/ulixee/secret-agent/commit/65258a3fa6ad23f7263bd0f90d86bebf751d8e94))
+
+
+
+
+
+# [1.6.0](https://github.com/ulixee/secret-agent/compare/v1.5.15...v1.6.0) (2021-12-14)
+
+
+### Bug Fixes
+
+* **core:** handle detached when elements not there ([5b46bd9](https://github.com/ulixee/secret-agent/commit/5b46bd97cf1ca1aa3bc0fed625064764726872f4))
+* **puppet:** extract navigation loader ([77d8e2e](https://github.com/ulixee/secret-agent/commit/77d8e2e34286469a2df1d9e00e631c6e773760eb))
+* **replay:** fix ability to replay data attributes ([9440995](https://github.com/ulixee/secret-agent/commit/944099560870c8479f0be59426fa194daf17ab4d))
+
+
+### Features
+
+* **plugins:** add frames to execute js ([65258a3](https://github.com/ulixee/secret-agent/commit/65258a3fa6ad23f7263bd0f90d86bebf751d8e94))
+
+
+
+
+
+## [1.5.15](https://github.com/ulixee/secret-agent/compare/v1.5.14...v1.5.15) (2021-11-24)
+
+
+### Bug Fixes
+
+* **client:** handle ws closing during shutdown ([09bfe29](https://github.com/ulixee/secret-agent/commit/09bfe29f54a4307fe0df1370a35a87d7950e9378))
+* **mitm:** clean invalid response characters ([5d85921](https://github.com/ulixee/secret-agent/commit/5d859217d7cf6aa870058e4234a079c0027e35b3))
+* **plugins:** improve user agent selector ([cd708bb](https://github.com/ulixee/secret-agent/commit/cd708bbc8e46705376f9cf3bd438a59c0063b6e7))
+* **puppet:** better handling of default loader ([2979ba7](https://github.com/ulixee/secret-agent/commit/2979ba7b323aab6896f9c2f006ca4bbbe40e69cb))
+* **puppet:** consisten timeout handling ([ba6bcee](https://github.com/ulixee/secret-agent/commit/ba6bcee3384811824ae3dc3789c5c3fc6088905a))
+* **puppet:** don't use contexts we didn't initiate ([710acab](https://github.com/ulixee/secret-agent/commit/710acabd1b4c0275d95048cb9f0053e2775f4d77))
+* **puppet:** fix flakiness on frame stopped ([002a516](https://github.com/ulixee/secret-agent/commit/002a51601123565b774f3589ddd9e35494516035))
+* **puppet:** fix shutdown issues (port from hero) ([2b70752](https://github.com/ulixee/secret-agent/commit/2b70752054edae95701243cc59afa48e113856fe))
+
+
+### Features
+
+* **plugins:** mask public ip in webrtc ([14d3c67](https://github.com/ulixee/secret-agent/commit/14d3c673327c149084ae32cd71c944cf60a84df3))
+
+
+
+
+
+## [1.5.14](https://github.com/ulixee/secret-agent/compare/v1.5.13...v1.5.14) (2021-11-15)
+
+
+### Bug Fixes
+
+* **core:** don't lose old user-profile storage ([9fe9ae4](https://github.com/ulixee/secret-agent/commit/9fe9ae4d6827a93f6b681dc10455d4695369b37f)), closes [#342](https://github.com/ulixee/secret-agent/issues/342)
+* **human-emulator:** fix updated traget point ([98eabd1](https://github.com/ulixee/secret-agent/commit/98eabd183660fd2cbbc44575f2703ae86e5e2950))
+* **puppet:** fix stack trace for protocol error ([8a5b4bf](https://github.com/ulixee/secret-agent/commit/8a5b4bf16b53268509b0ad93d3db74559f6de699))
+
+
+### Features
+
+* **mitm:** add documentUrl to resources ([7fcbf8f](https://github.com/ulixee/secret-agent/commit/7fcbf8faf84a72360d490f14c72910ea54619dc6))
+
+
+
+
+
 
 
 **Note:** Version bump only for package @secret-agent/monorepo

README.md

@@ -1,6 +1,12 @@
+## SecretAgent 2.0 === Hero
+
+> :bell: SecretAgent 2.0 is named "[Hero](https://github.com/ulixee/hero)" and currently in alpha testing. We're ready for developers to begin to switch over - it's a pretty easy transition ([migration guide](https://ulixee.org/docs/hero/help/migration-guide)). Follow along with development of Hero [here](https://github.com/ulixee/hero) or check out the latest npm packages `@ulixee/hero-fullstack`.
+
+---
+
 # SecretAgent
 
-SecretAgent is a web browser that's built for scraping. 
+SecretAgent is a web browser that's built for scraping.
 
 - [x] **Built for scraping** - it's the first modern headless browsers designed specifically for scraping instead of just automated testing.
 - [x] **Designed for web developers** - We've recreated a fully compliant DOM directly in NodeJS allowing you bypass the headaches of previous scraper tools.

client/CHANGELOG.md

@@ -3,6 +3,105 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.6.5](https://github.com/ulixee/secret-agent/compare/v1.6.4...v1.6.5) (2022-05-25)
+
+**Note:** Version bump only for package @secret-agent/client
+
+
+
+
+
+## [1.6.4](https://github.com/ulixee/secret-agent/compare/v1.6.3...v1.6.4) (2022-01-24)
+
+**Note:** Version bump only for package @secret-agent/client
+
+
+
+
+
+## [1.6.3](https://github.com/ulixee/secret-agent/compare/v1.6.2...v1.6.3) (2022-01-14)
+
+
+### Bug Fixes
+
+* **client:** wait for all command queue flushes ([95664f5](https://github.com/ulixee/secret-agent/commit/95664f570b6b7e233ea6b6227ad5f363cce52287))
+* **core:** cleanup event listener memory ([6f7d7bb](https://github.com/ulixee/secret-agent/commit/6f7d7bb449714876ab9a90ec4697c012d8c7bff3))
+
+
+
+
+
+## [1.6.2](https://github.com/ulixee/secret-agent/compare/v1.6.1...v1.6.2) (2021-12-20)
+
+
+### Bug Fixes
+
+* **client:** reload should return a resource ([83b80b5](https://github.com/ulixee/secret-agent/commit/83b80b5b3d70bc06344fb64d31fed1434ac98114)), closes [#393](https://github.com/ulixee/secret-agent/issues/393)
+
+
+### Features
+
+* **client:** return resource in waitForLocation ([253b33b](https://github.com/ulixee/secret-agent/commit/253b33b3528952432528949ab0dc2c8b4cf9a50c))
+* **core:** dialogs should run out of command line ([52a737f](https://github.com/ulixee/secret-agent/commit/52a737ff7ca380b00a1f156aa7dacf5e9af0bcfb)), closes [#397](https://github.com/ulixee/secret-agent/issues/397)
+
+
+
+
+
+## [1.6.1](https://github.com/ulixee/secret-agent/compare/v1.5.15...v1.6.1) (2021-12-14)
+
+
+### Bug Fixes
+
+* **core:** handle detached when elements not there ([5b46bd9](https://github.com/ulixee/secret-agent/commit/5b46bd97cf1ca1aa3bc0fed625064764726872f4))
+
+
+### Features
+
+* **plugins:** add frames to execute js ([65258a3](https://github.com/ulixee/secret-agent/commit/65258a3fa6ad23f7263bd0f90d86bebf751d8e94))
+
+
+
+
+
+# [1.6.0](https://github.com/ulixee/secret-agent/compare/v1.5.15...v1.6.0) (2021-12-14)
+
+
+### Bug Fixes
+
+* **core:** handle detached when elements not there ([5b46bd9](https://github.com/ulixee/secret-agent/commit/5b46bd97cf1ca1aa3bc0fed625064764726872f4))
+
+
+### Features
+
+* **plugins:** add frames to execute js ([65258a3](https://github.com/ulixee/secret-agent/commit/65258a3fa6ad23f7263bd0f90d86bebf751d8e94))
+
+
+
+
+
+## [1.5.15](https://github.com/ulixee/secret-agent/compare/v1.5.14...v1.5.15) (2021-11-24)
+
+
+### Bug Fixes
+
+* **client:** handle ws closing during shutdown ([09bfe29](https://github.com/ulixee/secret-agent/commit/09bfe29f54a4307fe0df1370a35a87d7950e9378))
+
+
+
+
+
+## [1.5.14](https://github.com/ulixee/secret-agent/compare/v1.5.13...v1.5.14) (2021-11-15)
+
+
+### Features
+
+* **mitm:** add documentUrl to resources ([7fcbf8f](https://github.com/ulixee/secret-agent/commit/7fcbf8faf84a72360d490f14c72910ea54619dc6))
+
+
+
+
+
 
 
 **Note:** Version bump only for package @secret-agent/client

client/connections/ConnectionFactory.ts

@@ -40,7 +40,7 @@ If you meant to connect to a remote host, include the "host" parameter for your
       }
     };
 
-    connection.connect().then(onError).catch(onError);
+    connection.connect(true).then(onError).catch(onError);
 
     return connection;
   }

client/connections/ConnectionToCore.ts

@@ -30,6 +30,7 @@ export default abstract class ConnectionToCore extends TypedEventEmitter<{
   public readonly hostOrError: Promise<string | Error>;
   public options: IConnectionToCoreOptions;
   public isDisconnecting = false;
+  public isConnectionTerminated = false;
 
   protected resolvedHost: string;
 
@@ -45,6 +46,7 @@ export default abstract class ConnectionToCore extends TypedEventEmitter<{
 
   private connectRequestId: string;
   private disconnectRequestId: string;
+  private didAutoConnect = false;
   private coreSessions: CoreSessions;
   private readonly pendingRequestsById = new Map<string, IResolvablePromiseWithId>();
   private lastId = 0;
@@ -81,14 +83,20 @@ export default abstract class ConnectionToCore extends TypedEventEmitter<{
   protected abstract createConnection(): Promise<Error | null>;
   protected abstract destroyConnection(): Promise<any>;
 
-  public async connect(): Promise<Error | null> {
+  public async connect(isAutoConnect = false): Promise<Error | null> {
     if (!this.connectPromise) {
+      this.didAutoConnect = isAutoConnect;
       this.connectPromise = new Resolvable();
       try {
         const startTime = new Date();
         const connectError = await this.createConnection();
         if (connectError) throw connectError;
-        if (this.isDisconnecting) throw new DisconnectedFromCoreError(this.resolvedHost);
+        if (this.isDisconnecting) {
+          if (this.coreSessions.size > 0 && !this.didAutoConnect) {
+            throw new DisconnectedFromCoreError(this.resolvedHost);
+          }
+          this.connectPromise.resolve();
+        }
         // can be resolved if canceled by a disconnect
         if (this.connectPromise.isResolved) return;
 
@@ -227,10 +235,12 @@ export default abstract class ConnectionToCore extends TypedEventEmitter<{
       sessionId: null,
     });
 
+    const hasSessions = this.coreSessions?.size > 0;
+
     this.cancelPendingRequests();
 
     if (this.connectPromise) {
-      if (!this.connectPromise.isResolved) {
+      if (!this.connectPromise.isResolved && hasSessions && !this.didAutoConnect) {
         this.connectPromise.resolve(new DisconnectedFromCoreError(this.resolvedHost));
       } else if (beforeClose) {
         await beforeClose();
@@ -287,11 +297,30 @@ export default abstract class ConnectionToCore extends TypedEventEmitter<{
     session?.onEvent(meta, listenerId, eventArgs);
   }
 
+  protected async onConnectionTerminated(): Promise<void> {
+    if (this.isConnectionTerminated) return;
+
+    this.isConnectionTerminated = true;
+    await this.internalDisconnect();
+    if (this.connectRequestId) {
+      this.onResponse(this.connectRequestId, {
+        data: this.didAutoConnect ? new DisconnectedFromCoreError(this.resolvedHost) : null,
+      });
+    }
+    if (this.disconnectRequestId) {
+      this.onResponse(this.disconnectRequestId, {
+        data: null,
+      });
+    }
+  }
+
   protected onResponse(id: string, message: ICoreResponsePayload): void {
     const pending = this.pendingRequestsById.get(id);
     if (!pending) return;
     this.pendingRequestsById.delete(id);
     const isInternalRequest = this.connectRequestId === id || this.disconnectRequestId === id;
+    if (this.disconnectRequestId === id) this.disconnectRequestId = null;
+    if (this.connectRequestId === id) this.connectRequestId = null;
 
     if (message.data instanceof Error) {
       let responseError = message.data;