Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,18 @@ If you want to allow multibyte characters, set it explicitly.
ValidEmail2::Address.permitted_multibyte_characters_regex = /[ÆæØøÅåÄäÖöÞþÐð]/
```

### Memory usage of the disposable domains list

By default, the disposable domains are loaded into a `Set` (`ValidEmail2.disposable_emails`), which adds about 35 MB to the process for the fastest lookups.
To trade lookup speed for memory, look them up in the sorted list file instead: it takes about 3 MB, and a lookup takes a few microseconds.

```ruby
# config/initializers/valid_email2.rb
ValidEmail2.disposable_domains_lookup = :sorted_list
```

With this option, domains added to or removed from `ValidEmail2.disposable_emails` are ignored by the validations.

### Test environment

If you are validating `mx` then your specs will fail without an internet connection.
Expand Down
27 changes: 27 additions & 0 deletions lib/valid_email2.rb
Original file line number Diff line number Diff line change
@@ -1,17 +1,44 @@
# frozen_string_literal: true

require "valid_email2/email_validator"
require "valid_email2/sorted_domain_list"

module ValidEmail2
DENY_LIST_FILE = "config/deny_listed_email_domains.yml"
ALLOW_LIST_FILE = "config/allow_listed_email_domains.yml"
DISPOSABLE_FILE = File.expand_path('../config/disposable_email_domains.txt', __dir__)
DISPOSABLE_DOMAINS_LOOKUPS = %i[set sorted_list].freeze

class << self
# How disposable domains are looked up:
# - :set (default): the Set of disposable_emails, about 35 MB, with the fastest lookups;
# - :sorted_list: a SortedDomainList, about 3 MB, with lookups of a few microseconds.
def disposable_domains_lookup
@disposable_domains_lookup || :set
end

def disposable_domains_lookup=(lookup)
unless DISPOSABLE_DOMAINS_LOOKUPS.include?(lookup)
raise ArgumentError, "disposable_domains_lookup must be one of #{DISPOSABLE_DOMAINS_LOOKUPS}, " \
"got #{lookup}"
end

@disposable_domains_lookup = lookup
end

def disposable_emails
@disposable_emails ||= load_file(DISPOSABLE_FILE)
end

# The disposable domains lookups use, as chosen by disposable_domains_lookup.
def disposable_domains
if disposable_domains_lookup == :sorted_list
@sorted_disposable_domains ||= SortedDomainList.new(DISPOSABLE_FILE)
else
disposable_emails
end
end

def deny_list
@deny_list ||= load_if_exists(DENY_LIST_FILE) || Set.new
end
Expand Down
4 changes: 2 additions & 2 deletions lib/valid_email2/address.rb
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def disposable?
end

def disposable_domain?
valid? && domain_is_in?(address.domain, ValidEmail2.disposable_emails)
valid? && domain_is_in?(address.domain, ValidEmail2.disposable_domains)
end

def allow_listed?
Expand Down Expand Up @@ -121,7 +121,7 @@ def valid_domain_label?(label)
end

def disposable_mx_server?
mx_server_is_in?(ValidEmail2.disposable_emails)
mx_server_is_in?(ValidEmail2.disposable_domains)
end

# TODO: (PS) keep this for backward compatibility with the test setup described in the reamde
Expand Down
36 changes: 36 additions & 0 deletions lib/valid_email2/sorted_domain_list.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# frozen_string_literal: true

module ValidEmail2
# A list of domains read from a sorted file, one domain per line, and kept as a single String
# searched by bisection. A Set of the same domains takes several times the memory: each domain
# becomes a String object of its own, plus its hash table entry.
class SortedDomainList
NEWLINE = "\n".b.freeze

def initialize(path)
@list = File.binread(path).freeze
end

def include?(domain)
domain = domain.to_s.b
low = 0
high = @list.bytesize

while low < high
middle = (low + high) / 2
start = (@list.rindex(NEWLINE, middle - 1) || -1) + 1
stop = @list.index(NEWLINE, start) || @list.bytesize
line = @list.byteslice(start, stop - start)
return true if line == domain

if line < domain
low = stop + 1
else
high = start
end
end

false
end
end
end
10 changes: 10 additions & 0 deletions spec/benchmark_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@
expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times
end

it "disposable_domain? has acceptable lookup performance with the sorted_list lookup" do
address = ValidEmail2::Address.new("test@example.com")
ValidEmail2.disposable_domains_lookup = :sorted_list
ValidEmail2.disposable_domains

expect { address.disposable_domain? }.to perform_under(0.0001).sec.sample(10).times
ensure
ValidEmail2.disposable_domains_lookup = :set
end

it "disposable_mx_server? has acceptable lookup performance" do
address = ValidEmail2::Address.new("test@gmail.com")

Expand Down
17 changes: 17 additions & 0 deletions spec/sorted_domain_list_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# frozen_string_literal: true

require "spec_helper"

describe ValidEmail2::SortedDomainList do
let(:domains) { File.readlines(ValidEmail2::DISPOSABLE_FILE, chomp: true) }
let(:list) { described_class.new(ValidEmail2::DISPOSABLE_FILE) }

# The bisection relies on it; pull_mailchecker_emails.rb writes it so.
it "searches a sorted file without duplicates" do
expect(domains.each_cons(2).all? { |a, b| a.b < b.b }).to be(true)
end

it "finds no domain missing from the file" do
expect(%w[gmail.com example.com 0 zzzzzzzzzz.zz].select { |domain| list.include?(domain) }).to be_empty
end
end
35 changes: 35 additions & 0 deletions spec/valid_email2_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,41 @@ def set_allow_list
end
end

describe ".disposable_domains" do
it "uses the Set of disposable_emails by default" do
expect(ValidEmail2.disposable_domains).to be(ValidEmail2.disposable_emails)
end

it "rejects an unknown lookup" do
expect { ValidEmail2.disposable_domains_lookup = :array }.to raise_error(ArgumentError, /got array/)
end

context "with the sorted_list lookup" do
around do |example|
loaded = ValidEmail2.instance_variable_get(:@disposable_emails)
ValidEmail2.instance_variable_set(:@disposable_emails, nil)
ValidEmail2.disposable_domains_lookup = :sorted_list
example.run
ensure
ValidEmail2.disposable_domains_lookup = :set
ValidEmail2.instance_variable_set(:@disposable_emails, loaded)
end

it "checks a disposable domain without loading the Set of disposable_emails" do
expect(ValidEmail2::Address.new("foo@mailinator.com").disposable_domain?).to be(true)
expect(ValidEmail2.instance_variable_get(:@disposable_emails)).to be_nil
end

it "has all entries from the other list" do
disposable_domains = ValidEmail2.disposable_domains
disposable_emails = ValidEmail2.disposable_emails
expect(disposable_domains).to be_a(ValidEmail2::SortedDomainList)
expect(disposable_emails).to be_a(Set)
expect(disposable_emails.reject { |domain| disposable_domains.include?(domain) }).to be_empty
end
end
end

describe "with deny list validation" do
it "is valid if the domain is not deny-listed" do
user = TestUserDisallowDenyListed.new(email: "foo@gmail.com")
Expand Down
Loading