Use banned domain list from the hyperphish api

README.md

@@ -25,23 +25,24 @@ notifications: 525783962023362616
 action_log: 525783962023362616
 pfp_log: 525783962023362616
 ban_message: "You have been banned from The Coding Den. Reason: `%s`.\nTo appeal, visit <https://tcd.one/appeal>"
+domain_poll_frequency_min: 15
 groups:
   - name: badware
     mode: delete
-  	words:
+  	phrases:
   	  - skype
   	  - teamspeak
     whitelist:
       - discord
   - name: spam
     mode: softban
-    words:
+    phrases:
       - somepornsite101010.com
     whitelist:
       - discord
   - name: slurs
     mode: log
-    words:
+    phrases:
       - jap
     whitelist:
       - japan

example.config.yml

@@ -11,23 +11,24 @@ notifications: 525783962023362616
 action_log: 525783962023362616
 pfp_log: 525783962023362616
 ban_message: "You have been banned from The Coding Den. Reason: `%s`.\nTo appeal, visit <https://tcd.one/appeal>"
+domain_poll_frequency_min: 15
 groups:
   - name: badware
     mode: delete
-    words:
+    phrases:
       - skype
       - teamspeak
     whitelist:
       - discord
   - name: spam
     mode: softban
-    words:
+    phrases:
       - somepornsite101010.com
     whitelist:
       - discord
   - name: slurs
     mode: log
-    words:
+    phrases:
       - jap
     whitelist:
       - japan

spec/spec_helper.cr

@@ -26,6 +26,7 @@ class Slash
     @spam_detection_cache = SpamDetectionCache.new(spam_detection_config)
     @last_spam_notice = Time.monotonic
     @ban_cache = LimitedSizeCache(UInt64).new(1)
+    @domain_poll_frequency = 15u16
   end
 
   def detect_group_and_word(string)

src/config.cr

@@ -27,7 +27,9 @@ class Slash
 
     property ban_message : String?
 
-    def initialize(@admins, @whitelist_roles, groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, @spam_detection_config, @ban_message)
+    property domain_poll_frequency_min : UInt16 = 15u16
+
+    def initialize(@admins, @whitelist_roles, groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, @spam_detection_config, @ban_message, @domain_poll_frequency_min)
       @groups = groups.values
     end
   end
@@ -45,11 +47,11 @@ class Slash
     action_log = c.action_log || c.notifs
     pfp_log = c.pfp_log || c.notifs
 
-    return c.admins, c.whitelist_roles, g, action_log, c.notifs, pfp_log, c.linkedjoins, c.dhash, c.spam_detection_config, c.ban_message
+    return c.admins, c.whitelist_roles, g, action_log, c.notifs, pfp_log, c.linkedjoins, c.dhash, c.spam_detection_config, c.ban_message, c.domain_poll_frequency_min
   end
 
   private def write
-    c = Config.new(@admins, @whitelist_roles, @groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, @spam_detection_cache.config, @ban_message)
+    c = Config.new(@admins, @whitelist_roles, @groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, @spam_detection_cache.config, @ban_message, @domain_poll_frequency)
 
     string = c.to_yaml
     File.write(ARGV[0], string)

src/handle_message.cr

@@ -5,6 +5,33 @@ class Slash
 
     return unless content = msg.content
 
+    # (1) Check banned domains
+    banned_domain = @banned_domains.find { |domain| content.includes?(domain) }
+    unless banned_domain.nil?
+      unless ban(user, "Scam detected")
+        begin
+          @client.delete_message(msg.channel_id, msg.id)
+        rescue exception : Discord::CodeException
+          Log.error { "Could not delete message from #{user.id}. Error-code: #{exception.error_code}" }
+        end
+      end
+      embed_fields = [
+        Discord::EmbedField.new("User", "<@#{user.id}>", true),
+        Discord::EmbedField.new("Used Domain", "**#{banned_domain}**", true),
+        Discord::EmbedField.new("Action", "**Ban**", true),
+        Discord::EmbedField.new("User ID", user.id.to_s, true),
+        Discord::EmbedField.new("Channel", "<##{msg.channel_id}>", true),
+        Discord::EmbedField.new("Type", msg.is_a?(Discord::Message) ? "NEW" : "EDIT", true),
+      ]
+      embed_desc = "[[JUMP]](#{link(msg)})"
+      embed_author = Discord::EmbedAuthor.new("#{user.username}##{user.discriminator}", icon_url: user.avatar_url)
+      msg_text = "```#{content.gsub('`', '\'')}```"
+      embed = Discord::Embed.new(author: embed_author, description: embed_desc, timestamp: Time.utc, fields: embed_fields, colour: Color::RED)
+      @client.create_message(@action_log, msg_text, embed)
+      return
+    end
+
+    # (2) Check for spam
     spam_result = @spam_detection_cache.check(msg)
     spam_detected = spam_result.should_ban
     if spam_detected
@@ -19,34 +46,36 @@ class Slash
       spam_result.user_ids.each do |user_id|
         ban(user_id, send_dm: false) unless user_id == user.id.to_u64
       end
-    end
-    if spam_detected && (Time.monotonic - @last_spam_notice) > 1.minute
-      @last_spam_notice = Time.monotonic
+      if (Time.monotonic - @last_spam_notice) > 1.minute
+        @last_spam_notice = Time.monotonic
 
-      recent_messages = spam_result.findings.sort_by { |message| message.time }.last(5)
-      embed_desc = String.build do |builder|
-        builder << "**Message triggering detection**\n<##{msg.channel_id}> [[Link]](#{get_message_link(msg)}) #{user.mention}:```\n#{content.gsub('`', "")}```\n"
-        builder << "**Similar messages**: #{spam_result.findings.size} with average similarity #{spam_result.average_similarity}\n"
-        builder << "**Last #{recent_messages.size} Matches**:"
-        recent_messages.each do |result|
-          builder << "\n<##{result.channel_id}> [[Link]](#{get_message_link(result.channel_id, result.message_id)}) <@#{result.author_id}>: "
-          if result.content.size > 100
-            builder << "`#{result.content.gsub('`', "")[0, 100]}...`"
-          else
-            builder << "`#{result.content.gsub('`', "")}`"
+        recent_messages = spam_result.findings.sort_by { |message| message.time }.last(5)
+        embed_desc = String.build do |builder|
+          builder << "**Message triggering detection**\n<##{msg.channel_id}> [[Link]](#{get_message_link(msg)}) #{user.mention}:```\n#{content.gsub('`', "")}```\n"
+          builder << "**Similar messages**: #{spam_result.findings.size} with average similarity #{spam_result.average_similarity}\n"
+          builder << "**Last #{recent_messages.size} Matches**:"
+          recent_messages.each do |result|
+            builder << "\n<##{result.channel_id}> [[Link]](#{get_message_link(result.channel_id, result.message_id)}) <@#{result.author_id}>: "
+            if result.content.size > 100
+              builder << "`#{result.content.gsub('`', "")[0, 100]}...`"
+            else
+              builder << "`#{result.content.gsub('`', "")}`"
+            end
           end
         end
-      end
 
-      if embed_desc.size > 5900
-        embed_desc = embed_desc[0..5900] + "\n..."
-      end
+        if embed_desc.size > 5900
+          embed_desc = embed_desc[0..5900] + "\n..."
+        end
 
-      embed_author = Discord::EmbedAuthor.new("#{user.username}##{user.discriminator}", icon_url: user.avatar_url)
-      embed = Discord::Embed.new(title: "Spam detected", author: embed_author, description: embed_desc, timestamp: Time.utc, colour: Color::RED)
-      @client.create_message(@action_log, ZWS, embed)
+        embed_author = Discord::EmbedAuthor.new("#{user.username}##{user.discriminator}", icon_url: user.avatar_url)
+        embed = Discord::Embed.new(title: "Spam detected", author: embed_author, description: embed_desc, timestamp: Time.utc, colour: Color::RED)
+        @client.create_message(@action_log, ZWS, embed)
+      end
+      return
     end
 
+    # (3) Check configured group/phrase matching. This is the last step in order to not have conditional fall-through in log/delete cases
     group, word = detect_group_and_word(content)
     return unless (group && word)
 

src/slash.cr

@@ -47,6 +47,10 @@ class Slash
   @linkedjoins : Hash(UInt64, UInt64)
   @linkedjoins_cache : Hash(UInt64, Array(UInt64?))
 
+  @banned_domains = Set(String).new
+
+  @domain_poll_frequency : UInt16
+
   module Color
     GRAY   = 0x808080_u32
     BLUE   = 0x34a8eb_u32
@@ -64,7 +68,7 @@ class Slash
 
     @dhash = Hash(String, BigInt).new
 
-    @admins, @whitelist_roles, @groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, spam_detection_config, @ban_message = load
+    @admins, @whitelist_roles, @groups, @action_log, @notifs, @pfp_log, @linkedjoins, @dhash, spam_detection_config, @ban_message, @domain_poll_frequency = load
 
     @spam_detection_cache = SpamDetectionCache.new(spam_detection_config)
     @ban_cache = LimitedSizeCache(UInt64).new(spam_detection_config.size)
@@ -78,6 +82,8 @@ class Slash
     spawn do
       @pfp_lock.send(nil)
     end
+
+    start_banned_domain_loop()
   end
 
   def run
@@ -207,4 +213,24 @@ class Slash
     end
     true
   end
+
+  private def start_banned_domain_loop
+    spawn do
+      while true
+        begin
+          response = HTTP::Client.get("https://api.hyperphish.com/gimme-domains", headers: HTTP::Headers{"User-Agent" => "Mini Slash, protector of the realm - github.com/TheCodingDen/mini-slash"})
+          if response.success?
+            @banned_domains = Array(String).from_json(response.body)
+          else
+            Log.warn { "Could not refresh banned domains! Statuscode: #{response.status_code}" }
+          end
+        rescue e
+          Log.error { "Fatally failed while trying to refresh banned domains. Exception message: #{e.message}" }
+        else
+          Log.info { "Refreshed banned domain list" }
+        end
+        sleep @domain_poll_frequency.minutes
+      end
+    end
+  end
 end