diff --git a/lib/net/imap.rb b/lib/net/imap.rb index 16f05445e..5b68463f0 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -2209,6 +2209,7 @@ def uid_expunge(uid_set) # provided as an array or a string. # See {"Argument translation"}[rdoc-ref:#search@Argument+translation] # and {"Search criteria"}[rdoc-ref:#search@Search+criteria], below. + # Please note the warning for when +criteria+ is a String. # # +return+ options control what kind of information is returned about # messages matching the search +criteria+. Specifying +return+ should force @@ -2619,7 +2620,8 @@ def search(...) # backward compatibility) but adds SearchResult#modseq when the +CONDSTORE+ # capability has been enabled. # - # See #search for documentation of parameters. + # See #search for documentation of parameters. Please note the + # warning for when +criteria+ is a String. # # ==== Capabilities # @@ -2705,7 +2707,8 @@ def fetch(...) # {SequenceSet[...]}[rdoc-ref:SequenceSet@Creating+sequence+sets]. # (For message sequence numbers, use #fetch instead.) # - # +attr+ behaves the same as with #fetch. + # +attr+ behaves the same as with #fetch. Please note the #fetch + # warning on the +attr+ argument. # >>> # *Note:* Servers _MUST_ implicitly include the +UID+ message data item as # part of any +FETCH+ response caused by a +UID+ command, regardless of @@ -2917,8 +2920,10 @@ def uid_move(set, mailbox) # Sends a {SORT command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3] # to search a mailbox for messages that match +search_keys+ and return an - # array of message sequence numbers, sorted by +sort_keys+. +search_keys+ - # are interpreted the same as for #search. + # array of message sequence numbers, sorted by +sort_keys+. + # + # +search_keys+ are interpreted the same as the +criteria+ argument for + # #search. Please note the #search warning for String +criteria+. # #-- # TODO: describe +sort_keys+ @@ -2943,8 +2948,10 @@ def sort(sort_keys, search_keys, charset) # Sends a {UID SORT command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3] # to search a mailbox for messages that match +search_keys+ and return an - # array of unique identifiers, sorted by +sort_keys+. +search_keys+ are - # interpreted the same as for #search. + # array of unique identifiers, sorted by +sort_keys+. + # + # +search_keys+ are interpreted the same as the +criteria+ argument for + # #search. Please note the #search warning for String +criteria+. # # Related: #sort, #search, #uid_search, #thread, #uid_thread # @@ -2958,8 +2965,10 @@ def uid_sort(sort_keys, search_keys, charset) # Sends a {THREAD command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3] # to search a mailbox and return message sequence numbers in threaded - # format, as a ThreadMember tree. +search_keys+ are interpreted the same as - # for #search. + # format, as a ThreadMember tree. + # + # +search_keys+ are interpreted the same as the +criteria+ argument for + # #search. Please note the #search warning for String +criteria+. # # The supported algorithms are: # @@ -2985,6 +2994,9 @@ def thread(algorithm, search_keys, charset) # Similar to #thread, but returns unique identifiers instead of # message sequence numbers. # + # +search_keys+ are interpreted the same as the +criteria+ argument for + # #search. Please note the #search warning for String +criteria+. + # # Related: #thread, #search, #uid_search, #sort, #uid_sort # # ==== Capabilities diff --git a/lib/net/imap/command_data.rb b/lib/net/imap/command_data.rb index bcc9de32f..09b7d9395 100644 --- a/lib/net/imap/command_data.rb +++ b/lib/net/imap/command_data.rb @@ -17,14 +17,15 @@ def validate_data(data) when nil when String when Integer - NumValidator.ensure_number(data) + # Covers modseq-valzer, which is the largest valid IMAP integer + if data.negative? + raise DataFormatError, "Integer argument must be unsigned: #{data}" + elsif 0xffff_ffff_ffff_ffff < data + raise DataFormatError, "Integer argument must fit in 64 bits: #{data}" + end when Array - if data[0] == 'CHANGEDSINCE' - NumValidator.ensure_mod_sequence_value(data[1]) - else - data.each do |i| - validate_data(i) - end + data.each do |i| + validate_data(i) end when Time, Date, DateTime when Symbol @@ -112,8 +113,9 @@ def send_literal(str, tag = nil, binary: false, non_sync: nil) end end + # NOTE: +num+ should already be an Integer def send_number_data(num) - put_string(num.to_s) + put_string(Integer(num).to_s) end def send_list_data(list, tag = nil) @@ -190,7 +192,12 @@ def send_data(imap, tag) = imap.__send__(:put_string, data) class RawData < CommandData # :nodoc: def initialize(data:) - data = split_parts(data) + case data + in String then data = self.class.split(data) + in Array if data.all? { _1 in RawText | Literal } + else + raise TypeError, "expected String or Array[#{RawText} | #{Literal}]" + end super validate end @@ -204,9 +211,11 @@ def validate end end - private - - def split_parts(data) + # Splits an input +string+ into an array of RawText and Literal/Literal8. + # + # NOTE: unlike RawData#validate, this does not prevent the final RawText + # from ending with a literal prefix. + def self.split(data) data = data.b # dups and ensures BINARY encoding parts = [] while data.match(/(~)?\{(0|[1-9]\d*)(\+)?\}\r\n/n) @@ -220,7 +229,7 @@ def split_parts(data) parts end - def extract_literal(data, binary:, bytesize:, non_sync:) + def self.extract_literal(data, binary:, bytesize:, non_sync:) if data.bytesize < bytesize raise DataFormatError, "Too few bytes in string for literal, " \ "expected: %s, remaining: %s" % [bytesize, data.bytesize] @@ -228,6 +237,7 @@ def extract_literal(data, binary:, bytesize:, non_sync:) literal = data.byteslice(0, bytesize) (binary ? Literal8 : Literal).new(data: literal, non_sync:) end + private_class_method :extract_literal end class Atom < CommandData # :nodoc: diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 03e54b202..988711834 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -2189,10 +2189,7 @@ def next_token if $1 return Token.new(T_SPACE, $+) elsif $2 - len = $+.to_i - val = @str[@pos, len] - @pos += len - return Token.new(T_LITERAL8, val) + literal_token($+, T_LITERAL8) elsif $3 && $7 # greedily match ATOM, prefixed with NUMBER, NIL, or PLUS. return Token.new(T_ATOM, $3) @@ -2220,10 +2217,7 @@ def next_token elsif $15 return Token.new(T_RBRA, $+) elsif $16 - len = $+.to_i - val = @str[@pos, len] - @pos += len - return Token.new(T_LITERAL, val) + literal_token($+) elsif $17 return Token.new(T_PERCENT, $+) elsif $18 @@ -2249,10 +2243,7 @@ def next_token elsif $4 return Token.new(T_QUOTED, Patterns.unescape_quoted($+)) elsif $5 - len = $+.to_i - val = @str[@pos, len] - @pos += len - return Token.new(T_LITERAL, val) + literal_token($+) elsif $6 return Token.new(T_LPAR, $+) elsif $7 @@ -2267,6 +2258,23 @@ def next_token else parse_error("invalid @lex_state - %s", @lex_state.inspect) end + rescue DataFormatError => error + parse_error error.message + end + + def literal_token(len, type = T_LITERAL) + len = coerce_number64 len.to_i + val = @str[@pos, len] + @pos += len + Token.new(type, val) + end + + # copied/adapted from NumValidator in v0.6 + def coerce_number64(num) + int = num.to_i + return int if 0 <= int && int <= 0x7fff_ffff_ffff_ffff + raise DataFormatError, + "number64 must be unsigned 63-bit integer: #{num}" end end diff --git a/lib/net/imap/response_reader.rb b/lib/net/imap/response_reader.rb index dd19d98da..590f6266c 100644 --- a/lib/net/imap/response_reader.rb +++ b/lib/net/imap/response_reader.rb @@ -4,6 +4,8 @@ module Net class IMAP # See https://www.rfc-editor.org/rfc/rfc9051#section-2.2.2 class ResponseReader # :nodoc: + include NumValidator + attr_reader :client def initialize(client, sock) @@ -35,7 +37,10 @@ def done? = line_done? && !literal_size def line_done? = buff.end_with?(CRLF) def get_literal_size(buff) - buff.end_with?("}\r\n") && buff.rindex(/\{(\d+)\}\r\n\z/n) && $1.to_i + buff.end_with?("}\r\n") && buff.rindex(/\{(\d+)\}\r\n\z/n) && + coerce_number64($1) + rescue DataFormatError + raise DataFormatError, format("invalid response literal size (%s)", $1) end def read_line @@ -74,6 +79,14 @@ def max_response_remaining! ) end + # copied/adapted from NumValidator in v0.6 + def coerce_number64(num) + int = num.to_i + return int if 0 <= int && int <= 0x7fff_ffff_ffff_ffff + raise DataFormatError, + "number64 must be unsigned 63-bit integer: #{num}" + end + end end end diff --git a/test/net/imap/fake_server/command_reader.rb b/test/net/imap/fake_server/command_reader.rb index 03d1da50d..8857cd9b8 100644 --- a/test/net/imap/fake_server/command_reader.rb +++ b/test/net/imap/fake_server/command_reader.rb @@ -3,13 +3,15 @@ require "net/imap" class Net::IMAP::FakeServer - CommandParseError = RuntimeError + CommandParseError = Class.new(RuntimeError) class CommandReader + attr_reader :config attr_reader :last_command attr_accessor :literal_acceptor - def initialize(socket) + def initialize(socket, config:) + @config = config @socket = socket @last_command = nil @literal_acceptor = proc {|buff, size| true } @@ -35,8 +37,11 @@ def get_command end throw :eof if buf.empty? @last_command = parse(buf) - rescue CommandParseError => err - raise IOError, err.message if socket.eof? && !buf.end_with?("\r\n") + rescue CommandParseError + if config.ignore_abrupt_eof? && socket.eof? && !buf.end_with?("\r\n") + throw :eof + end + raise end private @@ -46,7 +51,7 @@ def get_command # TODO: convert bad command exception to tagged BAD response, when possible def parse(buf) /\A([^ ]+) ((?:UID )?\w+)(?: (.+))?\r\n\z/min =~ buf or - raise CommandParseError, "bad request: %p" [buf] + raise CommandParseError, "bad request: %p" % [buf] case $2.upcase when "LOGIN", "SELECT", "EXAMINE", "ENABLE", "AUTHENTICATE" Command.new $1, $2, scan_astrings($3), buf diff --git a/test/net/imap/fake_server/configuration.rb b/test/net/imap/fake_server/configuration.rb index 91fe72a42..fde14aa78 100644 --- a/test/net/imap/fake_server/configuration.rb +++ b/test/net/imap/fake_server/configuration.rb @@ -45,6 +45,8 @@ class Configuration mailboxes: { "INBOX" => { name: "INBOX" }.freeze, }.freeze, + + ignore_abrupt_eof: false, } def initialize(with_extensions: [], without_extensions: [], **opts, &block) @@ -68,6 +70,7 @@ def initialize(with_extensions: [], without_extensions: [], **opts, &block) alias greeting_bye? greeting_bye alias greeting_capabilities? greeting_capabilities alias sasl_ir? sasl_ir + alias ignore_abrupt_eof? ignore_abrupt_eof def on(event, &handler) handler or raise ArgumentError diff --git a/test/net/imap/fake_server/connection.rb b/test/net/imap/fake_server/connection.rb index 7be9902b6..bd9225772 100644 --- a/test/net/imap/fake_server/connection.rb +++ b/test/net/imap/fake_server/connection.rb @@ -12,7 +12,7 @@ def initialize(server, tcp_socket:) @config = server.config @socket = Socket.new tcp_socket, config: config @state = ConnectionState.new socket: socket, config: config - @reader = CommandReader.new socket + @reader = CommandReader.new socket, config: config @writer = ResponseWriter.new socket, config: config, state: state @router = CommandRouter.new writer, config: config, state: state @mutex = Thread::Mutex.new diff --git a/test/net/imap/fake_server/socket.rb b/test/net/imap/fake_server/socket.rb index 65593d86f..84e94d589 100644 --- a/test/net/imap/fake_server/socket.rb +++ b/test/net/imap/fake_server/socket.rb @@ -18,10 +18,10 @@ def initialize(tcp_socket, config:) def tls?; !!@tls_socket end def closed?; @closed end - def eof?; socket.eof? end - def gets(...) socket.gets(...) end - def read(...) socket.read(...) end - def print(...) socket.print(...) end + def eof?; ignore_closed?(true) { socket.eof? } end + def gets(...) ignore_closed?(nil) { socket.gets(...) } end + def read(...) ignore_closed?(nil) { socket.read(...) } end + def print(...) ignore_closed?(nil) { socket.print(...) } end def use_tls @tls_socket ||= OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_ctx).tap do |s| @@ -48,5 +48,13 @@ def ssl_ctx end end + def ignore_closed?(fallback) + yield + rescue IOError => err + close if !closed? && (@tcp_socket.closed? || @tls_socket.closed?) + return fallback if err.message.match?(/stream closed|closed stream/i) + raise + end + end end diff --git a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml index e6bf9d482..a0145b602 100644 --- a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml +++ b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml @@ -7,6 +7,22 @@ data: raw_data: "* NOOP\r\n" + "literal numeric formatted with zero-prefix": + :response: "* 20367 FETCH (BODY[HEADER.FIELDS (Foo)] {012}\r\nFoo: bar\r\n\r\n)\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: FETCH + data: !ruby/struct:Net::IMAP::FetchData + seqno: 20367 + attr: + BODY[HEADER.FIELDS (Foo)]: "Foo: bar\r\n\r\n" + raw_data: "* 20367 FETCH (BODY[HEADER.FIELDS (Foo)] {012}\r\nFoo: bar\r\n\r\n)\r\n" + + "invalid literal numeric format (too large)": + :test_type: :assert_parse_failure + :message: "number64 must be unsigned 63-bit integer: 99999999999999999999" + :response: + "* 20367 FETCH (BODY[] {99999999999999999999}\r\nwon't parse this)\r\n" + test_invalid_noop_response_with_unparseable_data: :response: "* NOOP froopy snood\r\n" :expected: !ruby/struct:Net::IMAP::IgnoredResponse diff --git a/test/net/imap/test_command_data.rb b/test/net/imap/test_command_data.rb index 3ba579f36..042cdf37f 100644 --- a/test/net/imap/test_command_data.rb +++ b/test/net/imap/test_command_data.rb @@ -356,6 +356,23 @@ class RawDataTest < CommandDataTest raw = RawData.new(data: " {123} ") assert_equal [RawText[" {123} "]], raw.data end + + data( + "simple raw text" => 'hello "world"', + "text, literal, text" => "OK {5}\r\nhello {5}\r\nworld", + "empty literals" => "{0}\r\n{0+}\r\n~{0}\r\n~{0+}\r\n", + "binary and regular" => "foo ~{7}\r\n\0bar\r\nbaz {4}\r\nquux", + ) + test ".split" do |string| + assert_equal(RawData[string].data, RawData.split(string)) + end + + test ".split allows final literal prefix" do + assert_equal [RawText["text {123}"]], RawData.split("text {123}") + assert_equal [RawText["text+ {123+}"]], RawData.split("text+ {123+}") + assert_equal [RawText["~text ~{123}"]], RawData.split("~text ~{123}") + assert_equal [RawText["~text+ ~{123+}"]], RawData.split("~text+ ~{123+}") + end end end diff --git a/test/net/imap/test_imap.rb b/test/net/imap/test_imap.rb index d27e3d257..e2447ca8d 100644 --- a/test/net/imap/test_imap.rb +++ b/test/net/imap/test_imap.rb @@ -627,19 +627,23 @@ def test_send_invalid_number sock.print("RUBY0001 OK TEST completed\r\n") sock.gets # Integer: 2**32 - 1 sock.print("RUBY0002 OK TEST completed\r\n") - sock.gets # MessageSet: 1 + sock.gets # Integer: 2**32 sock.print("RUBY0003 OK TEST completed\r\n") - sock.gets # MessageSet: 2**32 - 1 + sock.gets # Integer: 2**64 - 1 sock.print("RUBY0004 OK TEST completed\r\n") - sock.gets # SequenceSet: -1 => "*" + sock.gets # MessageSet: 1 sock.print("RUBY0005 OK TEST completed\r\n") - sock.gets # SequenceSet: 1 + sock.gets # MessageSet: 2**32 - 1 sock.print("RUBY0006 OK TEST completed\r\n") - sock.gets # SequenceSet: 2**32 - 1 + sock.gets # SequenceSet: -1 => "*" sock.print("RUBY0007 OK TEST completed\r\n") + sock.gets # SequenceSet: 1 + sock.print("RUBY0008 OK TEST completed\r\n") + sock.gets # SequenceSet: 2**32 - 1 + sock.print("RUBY0009 OK TEST completed\r\n") sock.gets # LOGOUT sock.print("* BYE terminating connection\r\n") - sock.print("RUBY0008 OK LOGOUT completed\r\n") + sock.print("RUBY0010 OK LOGOUT completed\r\n") ensure sock.close server.close @@ -653,8 +657,10 @@ def test_send_invalid_number end imap.__send__(:send_command, "TEST", 0) imap.__send__(:send_command, "TEST", 2**32 - 1) + imap.__send__(:send_command, "TEST", 2**32) + imap.__send__(:send_command, "TEST", 2**64 - 1) assert_raise(Net::IMAP::DataFormatError) do - imap.__send__(:send_command, "TEST", 2**32) + imap.__send__(:send_command, "TEST", 2**64) end # MessageSet numbers may be non-zero uint32 stderr = EnvUtil.verbose_warning do diff --git a/test/net/imap/test_response_reader.rb b/test/net/imap/test_response_reader.rb index d18b40650..4a0bafa69 100644 --- a/test/net/imap/test_response_reader.rb +++ b/test/net/imap/test_response_reader.rb @@ -25,6 +25,8 @@ def literal(str) = "{#{str.bytesize}}\r\n#{str}" zero_literal = "tag ok #{literal ""} #{literal ""}\r\n" illegal_crs = "tag ok #{many_crs} #{many_crs}\r\n" illegal_lfs = "tag ok #{literal "\r"}\n#{literal "\r"}\n\r\n" + zero_padded = "+ {010}\r\n1234567890\r\n" # NOTE: it's decimal, not octal! + goofy_zero = "+ {000}\r\n\r\n" io = StringIO.new([ simple, long_line, @@ -33,6 +35,8 @@ def literal(str) = "{#{str.bytesize}}\r\n#{str}" zero_literal, illegal_crs, illegal_lfs, + zero_padded, + goofy_zero, simple, ].join) rcvr = Net::IMAP::ResponseReader.new(client, io) @@ -43,6 +47,8 @@ def literal(str) = "{#{str.bytesize}}\r\n#{str}" assert_equal zero_literal, rcvr.read_response_buffer.to_str assert_equal illegal_crs, rcvr.read_response_buffer.to_str assert_equal illegal_lfs, rcvr.read_response_buffer.to_str + assert_equal zero_padded, rcvr.read_response_buffer.to_str + assert_equal goofy_zero, rcvr.read_response_buffer.to_str assert_equal simple, rcvr.read_response_buffer.to_str assert_equal "", rcvr.read_response_buffer.to_str end @@ -82,4 +88,19 @@ def literal(str) = "{#{str.bytesize}}\r\n#{str}" end end + data( + bad_int64: "+ {99999999999999999999}\r\ndon't even try to read this...", + ) + test "#read_response_buffer with invalid literal size" do |invalid| + client = FakeClient.new + client.config.max_response_size = nil # any size is allowed! + io = StringIO.new(invalid, "rb") + rcvr = Net::IMAP::ResponseReader.new(client, io) + assert_raise Net::IMAP::DataFormatError do + result = rcvr.read_response_buffer + flunk "Got result: %p" % [result] + end + # assert io.closed? + end + end