diff --git a/README.md b/README.md index 6b5557a..c58aff4 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,18 @@ If you want to allow multibyte characters, set it explicitly. ValidEmail2::Address.permitted_multibyte_characters_regex = /[ÆæØøÅåÄäÖöÞþÐð]/ ``` +### Memory usage of the disposable domains list + +By default, the disposable domains are loaded into a `Set` (`ValidEmail2.disposable_emails`), which adds about 35 MB to the process for the fastest lookups. +To trade lookup speed for memory, look them up in the sorted list file instead: it takes about 3 MB, and a lookup takes a few microseconds. + +```ruby +# config/initializers/valid_email2.rb +ValidEmail2.disposable_domains_lookup = :sorted_list +``` + +With this option, domains added to or removed from `ValidEmail2.disposable_emails` are ignored by the validations. + ### Test environment If you are validating `mx` then your specs will fail without an internet connection. diff --git a/lib/valid_email2.rb b/lib/valid_email2.rb index 9622d93..1ae2259 100644 --- a/lib/valid_email2.rb +++ b/lib/valid_email2.rb @@ -1,17 +1,44 @@ # frozen_string_literal: true require "valid_email2/email_validator" +require "valid_email2/sorted_domain_list" module ValidEmail2 DENY_LIST_FILE = "config/deny_listed_email_domains.yml" ALLOW_LIST_FILE = "config/allow_listed_email_domains.yml" DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__) + DISPOSABLE_DOMAINS_LOOKUPS = %i[set sorted_list].freeze class << self + # How disposable domains are looked up: + # - :set (default): the Set of disposable_emails, about 35 MB, with the fastest lookups; + # - :sorted_list: a SortedDomainList, about 3 MB, with lookups of a few microseconds. + def disposable_domains_lookup + @disposable_domains_lookup || :set + end + + def disposable_domains_lookup=(lookup) + unless DISPOSABLE_DOMAINS_LOOKUPS.include?(lookup) + raise ArgumentError, "disposable_domains_lookup must be one of #{DISPOSABLE_DOMAINS_LOOKUPS}, " \ + "got #{lookup}" + end + + @disposable_domains_lookup = lookup + end + def disposable_emails @disposable_emails ||= load_file(DISPOSABLE_FILE) end + # The disposable domains lookups use, as chosen by disposable_domains_lookup. + def disposable_domains + if disposable_domains_lookup == :sorted_list + @sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE) + else + disposable_emails + end + end + def deny_list @deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new end diff --git a/lib/valid_email2/address.rb b/lib/valid_email2/address.rb index 48c7207..f6119b2 100644 --- a/lib/valid_email2/address.rb +++ b/lib/valid_email2/address.rb @@ -85,7 +85,7 @@ def disposable? end def disposable_domain? - valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails) + valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains) end def allow_listed? @@ -121,7 +121,7 @@ def valid_domain_label?(label) end def disposable_mx_server? - mx_server_is_in?(ValidEmail2.disposable_emails) + mx_server_is_in?(ValidEmail2.disposable_domains) end # TODO: (PS) keep this for backward compatibility with the test setup described in the reamde diff --git a/lib/valid_email2/sorted_domain_list.rb b/lib/valid_email2/sorted_domain_list.rb new file mode 100644 index 0000000..79fb752 --- /dev/null +++ b/lib/valid_email2/sorted_domain_list.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ValidEmail2 + # A list of domains read from a sorted file, one domain per line, and kept as a single String + # searched by bisection. A Set of the same domains takes several times the memory: each domain + # becomes a String object of its own, plus its hash table entry. + class SortedDomainList + NEWLINE = "\n".b.freeze + + def initialize(path) + @list = File.binread(path).freeze + end + + def include?(domain) + domain = domain.to_s.b + low = 0 + high = @list.bytesize + + while low < high + middle = (low + high) / 2 + start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1 + stop = @list.index(NEWLINE, start) || @list.bytesize + line = @list.byteslice(start, stop - start) + return true if line == domain + + if line < domain + low = stop + 1 + else + high = start + end + end + + false + end + end +end diff --git a/spec/benchmark_spec.rb b/spec/benchmark_spec.rb index f2fd5a4..00a6454 100644 --- a/spec/benchmark_spec.rb +++ b/spec/benchmark_spec.rb @@ -16,6 +16,16 @@ expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times end + it "disposable_domain? has acceptable lookup performance with the sorted_list lookup" do + address = ValidEmail2::Address.new("test@example.com") + ValidEmail2.disposable_domains_lookup = :sorted_list + ValidEmail2.disposable_domains + + expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times + ensure + ValidEmail2.disposable_domains_lookup = :set + end + it "disposable_mx_server? has acceptable lookup performance" do address = ValidEmail2::Address.new("test@gmail.com") diff --git a/spec/sorted_domain_list_spec.rb b/spec/sorted_domain_list_spec.rb new file mode 100644 index 0000000..c811feb --- /dev/null +++ b/spec/sorted_domain_list_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ValidEmail2::SortedDomainList do + let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) } + let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) } + + # The bisection relies on it; pull_mailchecker_emails.rb writes it so. + it "searches a sorted file without duplicates" do + expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true) + end + + it "finds no domain missing from the file" do + expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty + end +end diff --git a/spec/valid_email2_spec.rb b/spec/valid_email2_spec.rb index 70cca55..10edfa9 100644 --- a/spec/valid_email2_spec.rb +++ b/spec/valid_email2_spec.rb @@ -311,6 +311,41 @@ def set_allow_list end end + describe ".disposable_domains" do + it "uses the Set of disposable_emails by default" do + expect(ValidEmail2.disposable_domains).to be(ValidEmail2.disposable_emails) + end + + it "rejects an unknown lookup" do + expect { ValidEmail2.disposable_domains_lookup = :array }.to raise_error(ArgumentError, /got array/) + end + + context "with the sorted_list lookup" do + around do |example| + loaded = ValidEmail2.instance_variable_get(:@disposable_emails) + ValidEmail2.instance_variable_set(:@disposable_emails, nil) + ValidEmail2.disposable_domains_lookup = :sorted_list + example.run + ensure + ValidEmail2.disposable_domains_lookup = :set + ValidEmail2.instance_variable_set(:@disposable_emails, loaded) + end + + it "checks a disposable domain without loading the Set of disposable_emails" do + expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true) + expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil + end + + it "has all entries from the other list" do + disposable_domains = ValidEmail2.disposable_domains + disposable_emails = ValidEmail2.disposable_emails + expect(disposable_domains).to be_a(ValidEmail2::SortedDomainList) + expect(disposable_emails).to be_a(Set) + expect(disposable_emails.reject { |domain| disposable_domains.include?(domain) }).to be_empty + end + end + end + describe "with deny list validation" do it "is valid if the domain is not deny-listed" do user = TestUserDisallowDenyListed.new(email: "foo@gmail.com")