From 02ec0eddb591da02ad9e04a432458ad25e2a7dc2 Mon Sep 17 00:00:00 2001 From: Hans Lemuet Date: Mon, 12 Oct 2026 16:36:24 +0200 Subject: [PATCH 1/3] perf: look up disposable domains without building a Set of them ValidEmail2.disposable_emails loads the ~180,000 disposable domains into a Set: one String object per domain plus its hash entry, about 35 MB of memory once the first email is checked. The file is sorted (see pull_mailchecker_emails.rb), so lookups now go through ValidEmail2.disposable_domains, a SortedDomainList keeping the file as a single String (2.6 MB) searched by bisection. disposable_emails still returns the same Set, built on demand. Once it is loaded, disposable_domains returns it, so code adding domains to it or removing them keeps working. --- lib/valid_email2.rb | 9 +++++++ lib/valid_email2/address.rb | 4 +-- lib/valid_email2/sorted_domain_list.rb | 36 ++++++++++++++++++++++++++ spec/benchmark_spec.rb | 8 ++++++ spec/sorted_domain_list_spec.rb | 21 +++++++++++++++ spec/valid_email2_spec.rb | 20 ++++++++++++++ 6 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 lib/valid_email2/sorted_domain_list.rb create mode 100644 spec/sorted_domain_list_spec.rb diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index 9622d93a..a40bb8af 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "valid_email2/email_validator" +require "valid_email2/sorted_domain_list" module ValidEmail2 DENY_LIST_FILE = "config/deny_listed_email_domains.yml" @@ -8,10 +9,18 @@ module ValidEmail2 DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__) class << self + # The disposable domains as a Set, which takes about 14 MB. Lookups use disposable_domains, which + # returns this Set once it is loaded, so domains added to it or removed from it still count. def disposable_emails @disposable_emails ||= load_file(DISPOSABLE_FILE) end + # The disposable domains lookups use: the Set of disposable_emails if it was loaded, otherwise + # the sorted file searched by bisection, which takes about 3 MB. + def disposable_domains + @disposable_emails || (@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE)) + end + def deny_list @deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new end diff --git a/lib/valid_email2/address.rb b/lib/valid_email2/address.rb index 48c72075..f6119b2c 100644 --- a/lib/valid_email2/address.rb +++ b/lib/valid_email2/address.rb @@ -85,7 +85,7 @@ def disposable? end def disposable_domain? - valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails) + valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains) end def allow_listed? @@ -121,7 +121,7 @@ def valid_domain_label?(label) end def disposable_mx_server? - mx_server_is_in?(ValidEmail2.disposable_emails) + mx_server_is_in?(ValidEmail2.disposable_domains) end # TODO: (PS) keep this for backward compatibility with the test setup described in the reamde diff --git a/lib/valid_email2/sorted_domain_list.rb b/lib/valid_email2/sorted_domain_list.rb new file mode 100644 index 00000000..79fb7520 --- /dev/null +++ b/lib/valid_email2/sorted_domain_list.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ValidEmail2 + # A list of domains read from a sorted file, one domain per line, and kept as a single String + # searched by bisection. A Set of the same domains takes several times the memory: each domain + # becomes a String object of its own, plus its hash table entry. + class SortedDomainList + NEWLINE = "\n".b.freeze + + def initialize(path) + @list = File.binread(path).freeze + end + + def include?(domain) + domain = domain.to_s.b + low = 0 + high = @list.bytesize + + while low < high + middle = (low + high) / 2 + start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1 + stop = @list.index(NEWLINE, start) || @list.bytesize + line = @list.byteslice(start, stop - start) + return true if line == domain + + if line < domain + low = stop + 1 + else + high = start + end + end + + false + end + end +end diff --git a/spec/benchmark_spec.rb b/spec/benchmark_spec.rb index f2fd5a4c..625a061c 100644 --- a/spec/benchmark_spec.rb +++ b/spec/benchmark_spec.rb @@ -16,6 +16,14 @@ expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times end + it "disposable_domain? has acceptable lookup performance without the Set" do + address = ValidEmail2::Address.new("test@example.com") + list = ValidEmail2::SortedDomainList.new(ValidEmail2::DISPOSABLE_FILE) + allow(ValidEmail2).to receive(:disposable_domains).and_return(list) + + expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times + end + it "disposable_mx_server? has acceptable lookup performance" do address = ValidEmail2::Address.new("test@gmail.com") diff --git a/spec/sorted_domain_list_spec.rb b/spec/sorted_domain_list_spec.rb new file mode 100644 index 00000000..6c4a87f9 --- /dev/null +++ b/spec/sorted_domain_list_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ValidEmail2::SortedDomainList do + let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) } + let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) } + + # The bisection relies on it; pull_mailchecker_emails.rb writes it so. + it "searches a sorted file without duplicates" do + expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true) + end + + it "finds every domain of the file" do + expect(domains.reject { |domain| list.include?(domain) }).to be_empty + end + + it "finds no domain missing from the file" do + expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty + end +end diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index 70cca55a..cef552bc 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -311,6 +311,26 @@ def set_allow_list end end + describe ".disposable_domains" do + around do |example| + loaded = ValidEmail2.instance_variable_get(:@disposable_emails) + ValidEmail2.instance_variable_set(:@disposable_emails, nil) + example.run + ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + end + + it "checks a disposable domain without loading the Set of disposable_emails" do + expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) + expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + end + + it "uses the Set of disposable_emails once it is loaded, with the domains added to it" do + ValidEmail2.disposable_emails << "not-yet-listed.example" + + expect(ValidEmail2::Address.new("foo@not-yet-listed.example").disposable_domain?).to be(true) + end + end + describe "with deny list validation" do it "is valid if the domain is not deny-listed" do user = TestUserDisallowDenyListed.new(email: "foo@gmail.com") From b8541491992e2f80cdd7bbef93f130ebe4a9b830 Mon Sep 17 00:00:00 2001 From: Hans Lemuet Date: Mon, 9 Nov 2026 22:45:01 +0100 Subject: [PATCH 2/3] feat: make the sorted disposable domains list opt-in Lookups use the Set of disposable_emails by default again. Setting ValidEmail2.disposable_domains_lookup = :sorted_list switches them to the SortedDomainList, trading lookup speed for memory. --- README.md | 12 +++++++++++ lib/valid_email2.rb | 28 ++++++++++++++++++++----- spec/benchmark_spec.rb | 8 ++++--- spec/sorted_domain_list_spec.rb | 4 ---- spec/valid_email2_spec.rb | 37 +++++++++++++++++++++++---------- 5 files changed, 66 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 6b5557a4..c58aff44 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,18 @@ If you want to allow multibyte characters, set it explicitly. ValidEmail2::Address.permitted_multibyte_characters_regex = /[ÆæØøÅåÄäÖöÞþÐð]/ ``` +### Memory usage of the disposable domains list + +By default, the disposable domains are loaded into a `Set` (`ValidEmail2.disposable_emails`), which adds about 35 MB to the process for the fastest lookups. +To trade lookup speed for memory, look them up in the sorted list file instead: it takes about 3 MB, and a lookup takes a few microseconds. + +```ruby +# config/initializers/valid_email2.rb +ValidEmail2.disposable_domains_lookup = :sorted_list +``` + +With this option, domains added to or removed from `ValidEmail2.disposable_emails` are ignored by the validations. + ### Test environment If you are validating `mx` then your specs will fail without an internet connection. diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index a40bb8af..4a9b8829 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -7,18 +7,36 @@ module ValidEmail2 DENY_LIST_FILE = "config/deny_listed_email_domains.yml" ALLOW_LIST_FILE = "config/allow_listed_email_domains.yml" DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__) + DISPOSABLE_DOMAINS_LOOKUPS = %i[set sorted_list].freeze class << self - # The disposable domains as a Set, which takes about 14 MB. Lookups use disposable_domains, which - # returns this Set once it is loaded, so domains added to it or removed from it still count. + # How disposable domains are looked up: + # - :set (default): the Set of disposable_emails, about 35 MB, with the fastest lookups; + # - :sorted_list: a SortedDomainList, about 3 MB, with lookups of a few microseconds. + def disposable_domains_lookup + @disposable_domains_lookup || :set + end + + def disposable_domains_lookup=(lookup) + unless DISPOSABLE_DOMAINS_LOOKUPS.include?(lookup) + raise ArgumentError, "disposable_domains_lookup must be one of #{DISPOSABLE_DOMAINS_LOOKUPS.inspect}, " \ + "got #{lookup.inspect}" + end + + @disposable_domains_lookup = lookup + end + def disposable_emails @disposable_emails ||= load_file(DISPOSABLE_FILE) end - # The disposable domains lookups use: the Set of disposable_emails if it was loaded, otherwise - # the sorted file searched by bisection, which takes about 3 MB. + # The disposable domains lookups use, as chosen by disposable_domains_lookup. def disposable_domains - @disposable_emails || (@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE)) + if disposable_domains_lookup == :sorted_list + @sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE) + else + disposable_emails + end end def deny_list diff --git a/spec/benchmark_spec.rb b/spec/benchmark_spec.rb index 625a061c..00a64540 100644 --- a/spec/benchmark_spec.rb +++ b/spec/benchmark_spec.rb @@ -16,12 +16,14 @@ expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times end - it "disposable_domain? has acceptable lookup performance without the Set" do + it "disposable_domain? has acceptable lookup performance with the sorted_list lookup" do address = ValidEmail2::Address.new("test@example.com") - list = ValidEmail2::SortedDomainList.new(ValidEmail2::DISPOSABLE_FILE) - allow(ValidEmail2).to receive(:disposable_domains).and_return(list) + ValidEmail2.disposable_domains_lookup = :sorted_list + ValidEmail2.disposable_domains expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times + ensure + ValidEmail2.disposable_domains_lookup = :set end it "disposable_mx_server? has acceptable lookup performance" do diff --git a/spec/sorted_domain_list_spec.rb b/spec/sorted_domain_list_spec.rb index 6c4a87f9..c811febb 100644 --- a/spec/sorted_domain_list_spec.rb +++ b/spec/sorted_domain_list_spec.rb @@ -11,10 +11,6 @@ expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true) end - it "finds every domain of the file" do - expect(domains.reject { |domain| list.include?(domain) }).to be_empty - end - it "finds no domain missing from the file" do expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty end diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index cef552bc..1298b10a 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -312,22 +312,37 @@ def set_allow_list end describe ".disposable_domains" do - around do |example| - loaded = ValidEmail2.instance_variable_get(:@disposable_emails) - ValidEmail2.instance_variable_set(:@disposable_emails, nil) - example.run - ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + it "uses the Set of disposable_emails by default" do + expect(ValidEmail2.disposable_domains).to be(ValidEmail2.disposable_emails) end - it "checks a disposable domain without loading the Set of disposable_emails" do - expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) - expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + it "rejects an unknown lookup" do + expect { ValidEmail2.disposable_domains_lookup = :array }.to raise_error(ArgumentError, /:array/) end - it "uses the Set of disposable_emails once it is loaded, with the domains added to it" do - ValidEmail2.disposable_emails << "not-yet-listed.example" + context "with the sorted_list lookup" do + around do |example| + loaded = ValidEmail2.instance_variable_get(:@disposable_emails) + ValidEmail2.instance_variable_set(:@disposable_emails, nil) + ValidEmail2.disposable_domains_lookup = :sorted_list + example.run + ensure + ValidEmail2.disposable_domains_lookup = :set + ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + end - expect(ValidEmail2::Address.new("foo@not-yet-listed.example").disposable_domain?).to be(true) + it "checks a disposable domain without loading the Set of disposable_emails" do + expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) + expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + end + + it "has all entries from the other list" do + disposable_domains = ValidEmail2.disposable_domains + disposable_emails = ValidEmail2.disposable_emails + expect(disposable_domains).to be_a(ValidEmail2::SortedDomainList) + expect(disposable_emails).to be_a(Set) + expect(disposable_emails.reject { |domain| disposable_domains.include?(domain) }).to be_empty + end end end From beffcef58838518a5b66d134ec73ca431f446641 Mon Sep 17 00:00:00 2001 From: Hans Lemuet Date: Tue, 10 Nov 2026 13:20:21 +0100 Subject: [PATCH 3/3] style: drop inspect from the unknown lookup's error message As suggested in review. --- lib/valid_email2.rb | 4 ++-- spec/valid_email2_spec.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index 4a9b8829..1ae22599 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -19,8 +19,8 @@ def disposable_domains_lookup def disposable_domains_lookup=(lookup) unless DISPOSABLE_DOMAINS_LOOKUPS.include?(lookup) - raise ArgumentError, "disposable_domains_lookup must be one of #{DISPOSABLE_DOMAINS_LOOKUPS.inspect}, " \ - "got #{lookup.inspect}" + raise ArgumentError, "disposable_domains_lookup must be one of #{DISPOSABLE_DOMAINS_LOOKUPS}, " \ + "got #{lookup}" end @disposable_domains_lookup = lookup diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index 1298b10a..10edfa9e 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -317,7 +317,7 @@ def set_allow_list end it "rejects an unknown lookup" do - expect { ValidEmail2.disposable_domains_lookup = :array }.to raise_error(ArgumentError, /:array/) + expect { ValidEmail2.disposable_domains_lookup = :array }.to raise_error(ArgumentError, /got array/) end context "with the sorted_list lookup" do