Merge branch 'develop'

Author: vasconsaurus

CODEOWNERS

@@ -1 +1 @@
-* @caiosba @melsawy @vasconsaurus
+* @caiosba @melsawy @vasconsaurus @jayjay-w

Gemfile

@@ -47,7 +47,7 @@ gem 'rack-cors', '>= 2.0.2', :require => 'rack/cors'
 gem 'rails-perftest'
 gem 'sidekiq', '< 8'
 gem 'redis', '4.3.1'
-gem 'nokogiri', '1.16.2', require: false
+gem 'nokogiri', '1.16.5', require: false
 gem 'htmlentities', require: false
 gem 'rack-protection', '2.0.1'
 gem 'loofah', '2.19.1', require: false

Gemfile.lock

@@ -21,60 +21,60 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actioncable (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionmailbox (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionmailer (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.7)
-      actionview (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionpack (6.1.7.8)
+      actionview (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actiontext (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionview (6.1.7.8)
+      activesupport (= 6.1.7.8)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.1.7.7)
-      activesupport (= 6.1.7.7)
+    activejob (6.1.7.8)
+      activesupport (= 6.1.7.8)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.7)
-      activesupport (= 6.1.7.7)
-    activerecord (6.1.7.7)
-      activemodel (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
-    activestorage (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    activemodel (6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activerecord (6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activestorage (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.7)
+    activesupport (6.1.7.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -106,7 +106,7 @@ GEM
     byebug (11.1.3)
     codeclimate-test-reporter (1.0.8)
       simplecov (<= 0.13)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.1)
     connection_pool (2.4.1)
     crack (0.4.5)
       rexml
@@ -143,7 +143,7 @@ GEM
     heapy (0.2.0)
       thor
     htmlentities (4.3.4)
-    i18n (1.14.1)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     jmespath (1.6.2)
     json (2.6.3)
@@ -170,7 +170,7 @@ GEM
     method_source (1.0.0)
     mini_histogram (0.3.1)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.7)
     minitest (5.10.1)
     minitest-retry (0.2.2)
       minitest (>= 5.0)
@@ -187,7 +187,7 @@ GEM
     net-smtp (0.4.0.1)
       net-protocol
     nio4r (2.7.0)
-    nokogiri (1.16.2)
+    nokogiri (1.16.5)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     open_uri_redirections (0.2.1)
@@ -270,7 +270,7 @@ GEM
     public_suffix (4.0.7)
     puma (5.6.8)
       nio4r (~> 2.0)
-    racc (1.7.3)
+    racc (1.8.0)
     rack (2.2.8.1)
     rack-cors (2.0.2)
       rack (>= 2.0.0)
@@ -279,20 +279,20 @@ GEM
     rack-test (2.1.0)
       rack (>= 1.3)
     railroady (1.6.0)
-    rails (6.1.7.7)
-      actioncable (= 6.1.7.7)
-      actionmailbox (= 6.1.7.7)
-      actionmailer (= 6.1.7.7)
-      actionpack (= 6.1.7.7)
-      actiontext (= 6.1.7.7)
-      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activemodel (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    rails (6.1.7.8)
+      actioncable (= 6.1.7.8)
+      actionmailbox (= 6.1.7.8)
+      actionmailer (= 6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actiontext (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.7)
+      railties (= 6.1.7.8)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -305,23 +305,24 @@ GEM
     rails-html-sanitizer (1.4.4)
       loofah (~> 2.19, >= 2.19.1)
     rails-perftest (0.0.7)
-    railties (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    railties (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
     rake (13.1.0)
     redis (4.3.1)
-    redis-client (0.18.0)
+    redis-client (0.22.1)
       connection_pool
     request_store (1.5.1)
       rack (>= 1.4)
     responders (3.1.0)
       actionpack (>= 5.2)
       railties (>= 5.2)
     retryable (3.0.5)
-    rexml (3.2.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
     rspec-expectations (3.12.3)
@@ -364,11 +365,11 @@ GEM
     sentry-sidekiq (5.10.0)
       sentry-ruby (~> 5.10.0)
       sidekiq (>= 3.0)
-    sidekiq (7.2.0)
+    sidekiq (7.2.4)
       concurrent-ruby (< 2)
       connection_pool (>= 2.3.0)
       rack (>= 2.2.4)
-      redis-client (>= 0.14.0)
+      redis-client (>= 0.19.0)
     simplecov (0.13.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -386,6 +387,7 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    strscan (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.3.1)
@@ -409,7 +411,7 @@ GEM
     websocket-extensions (0.1.5)
     yt (0.25.40)
       activesupport
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.15)
 
 PLATFORMS
   ruby
@@ -432,7 +434,7 @@ DEPENDENCIES
   minitest-retry
   mocha (~> 1.14.0)
   net-http
-  nokogiri (= 1.16.2)
+  nokogiri (= 1.16.5)
   open_uri_redirections
   opentelemetry-exporter-otlp
   opentelemetry-instrumentation-action_pack

README.md

@@ -454,6 +454,16 @@ To enable sampling for Honeycomb, set the following configuration (either in `co
 
 **Note**: If sampling behavior is changed in Pender, we will also need to update the behavior to match in any other application reporting to Honeycomb. More [here](https://docs.honeycomb.io/getting-data-in/opentelemetry/ruby/#sampling)
 
+### URL Parameters Normalization
+
+Some service providers include URL parameters for tracking purposes that can be safely removed. Pender parsers can define a list of such parameters to be removed during the URL normalization process.
+
+To define URL parameters to be removed, a parser class should implement the `urls_parameters_to_remove` method, which returns an array of strings representing the parameters to be stripped. For example:
+
+```ruby
+def urls_parameters_to_remove
+  ['ighs']
+end
 
 #### Environment overrides
 

app/models/concerns/provider_twitter.rb

@@ -14,7 +14,7 @@ def oembed_url(_ = nil)
   def tweet_lookup(tweet_id)
     params = {
       "ids": tweet_id,
-      "tweet.fields": "author_id,created_at,text",
+      "tweet.fields": "author_id,created_at,text,lang",
       "expansions": "author_id,attachments.media_keys",
       "user.fields": "profile_image_url,username,url",
       "media.fields": "url",

app/models/media.rb

@@ -64,6 +64,7 @@ def initialize(attributes = {})
     self.follow_redirections
     self.url = RequestHelper.normalize_url(self.url) unless self.get_canonical_url
     self.try_https
+    self.remove_parser_specific_parameters
     self.parser = nil
   end
 
@@ -275,6 +276,37 @@ def try_https
     end
   end
 
+  def remove_parser_specific_parameters
+    parser_class = self.class.find_parser_class(self.url)
+    return unless parser_class&.respond_to?(:urls_parameters_to_remove)
+
+    params_to_remove = parser_class.urls_parameters_to_remove
+    return unless params_to_remove.any? { |param| self.url.include?(param) }
+
+    uri = URI.parse(self.url)
+    query_params = URI.decode_www_form(uri.query || '').to_h
+
+    params_to_remove.each do |param|
+      query_params.keys.each do |key|
+        query_params.delete(key) if key == param
+      end
+    end
+
+    new_query = query_params.empty? ? nil : URI.encode_www_form(query_params)
+    uri.query = new_query
+
+    result_url = uri.to_s
+    result_url += '/' if url.end_with?('/') && !result_url.end_with?('/')
+    self.url = result_url
+  end
+
+  def self.find_parser_class(url)
+    PARSERS.each do |parser|
+      return parser if parser.patterns.any? { |pattern| pattern.match?(url) }
+    end
+    nil
+  end
+
   def get_html(header_options = {}, force_proxy = false)
     RequestHelper.get_html(self.url, self.method(:set_error), header_options, force_proxy)
   end

app/models/parser/dropbox_item.rb

@@ -7,7 +7,7 @@ def type
 
       def patterns
         [
-          /^https?:\/\/(www\.)?dropbox\.com\/sh?\/([^\/]+)/,
+          /^https?:\/\/(www\.)?dropbox\.com\/([^\/]+)/,
           /^https?:\/\/([^\.]+\.)?(dropboxusercontent|dropbox)\.com\/s\/([^\/]+)/,
         ]
       end

app/models/parser/instagram_item.rb

@@ -13,6 +13,10 @@ def type
       def patterns
         [INSTAGRAM_ITEM_URL, INSTAGRAM_HOME_URL]
       end
+
+      def urls_parameters_to_remove
+        ['igsh']
+      end
     end
 
     private

app/models/parser/instagram_profile.rb

@@ -12,6 +12,10 @@ def type
       def patterns
         [INSTAGRAM_PROFILE_URL]
       end
+
+      def urls_parameters_to_remove
+        ['igsh']
+      end
     end
 
     private

app/models/parser/twitter_item.rb

@@ -2,7 +2,7 @@ module Parser
   class TwitterItem < Base
     include ProviderTwitter
 
-    TWITTER_ITEM_URL = /^https?:\/\/([^\.]+\.)?twitter\.com\/((%23|#)!\/)?(?<user>[^\/]+)\/status\/(?<id>[0-9]+).*/
+    TWITTER_ITEM_URL = /^https?:\/\/([^\.]+\.)?(twitter|x)\.com\/((%23|#)!\/)?(?<user>[^\/]+)\/status\/(?<id>[0-9]+).*/
 
     class << self
       def type
@@ -49,7 +49,7 @@ def parse_data_for_parser(_doc, _original_url, _jsonld_array)
             description: raw_data['text'].squish,
             author_picture: raw_user_data['profile_image_url'].gsub('_normal', ''),
             published_at: raw_data['created_at'],
-            html: html_for_twitter_item(url),
+            html: html_for_twitter_item(url,raw_data['lang']),
             author_name: raw_user_data['name'],
           })  
         end
@@ -67,11 +67,13 @@ def get_twitter_item_picture(parsed_data)
       item_media ? item_media.dig(0, 'url') : ''
     end
 
-    def html_for_twitter_item(url)
+    def html_for_twitter_item(url,lang)
       '<blockquote class="twitter-tweet">' +
-      '<a href="' + url + '"></a>' +
+        '<p lang="' + lang + '" dir="ltr"></p>' +
+        '<a href="' + url.gsub(/x\.com/, 'twitter.com') + '"></a>' +
       '</blockquote>' +
-      '<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>'
+      '<script async src="https://platform.twitter.com/widgets.js" charset="utf-8">' +
+      '</script>'
     end
   end
 end