[nio-cvs] r100 - in branches/home/psmith/restructure/src: buffer io protocol/yarpc statemachine
psmith at common-lisp.net
psmith at common-lisp.net
Tue Feb 27 03:00:16 UTC 2007
Author: psmith
Date: Mon Feb 26 22:00:16 2007
New Revision: 100
Modified:
branches/home/psmith/restructure/src/buffer/buffer.lisp
branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp
branches/home/psmith/restructure/src/io/async-fd.lisp
branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp
branches/home/psmith/restructure/src/statemachine/state-machine.lisp
Log:
Fixed large packet problems: calculated header size correctly; use buffer-pointer to take into account buffer-position on write and moved external format to UTF-8
Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp
==============================================================================
--- branches/home/psmith/restructure/src/buffer/buffer.lisp (original)
+++ branches/home/psmith/restructure/src/buffer/buffer.lisp Mon Feb 26 22:00:16 2007
@@ -197,7 +197,7 @@
; Read bytes from bytebuffer abd return a string using the supplied decoding
;TODO move octets-to-string into nio-compat
-(defmethod bytebuffer-read-string((bb byte-buffer) &optional (num-bytes-to-read (remaining bb)) (external-format :ascii))
+(defmethod bytebuffer-read-string((bb byte-buffer) &optional (num-bytes-to-read (remaining bb)) (external-format :utf-8))
(sb-ext:octets-to-string (bytebuffer-read-vector bb num-bytes-to-read) :external-format external-format))
; Read a byte from bytebuffer and return it incrementing the byte-buffers position
@@ -246,7 +246,7 @@
;; Writes data from string str to bytebuffer using specified encoding
;TODO move string-to-octets into nio-compat
-(defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii))
+(defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :utf-8))
:documentation "Returns number of bytes written to bytebuffer"
(bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format)))
Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp
==============================================================================
--- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original)
+++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Mon Feb 26 22:00:16 2007
@@ -31,6 +31,6 @@
bytebuffer-write-vector bytebuffer-write-string
bytebuffer-read-vector bytebuffer-read-string
bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32
- flip unflip clear buffer-position copy-buffer buffer-capacity compact mark reset
+ flip unflip clear buffer-position buffer-limit copy-buffer buffer-capacity compact mark reset
buffer-too-small-error recommended-size buffer-pointer
))
Modified: branches/home/psmith/restructure/src/io/async-fd.lisp
==============================================================================
--- branches/home/psmith/restructure/src/io/async-fd.lisp (original)
+++ branches/home/psmith/restructure/src/io/async-fd.lisp Mon Feb 26 22:00:16 2007
@@ -106,6 +106,7 @@
(t
;;Update buffer position
(inc-position foreign-read-buffer new-bytes)
+#+nio-debug (format t "read-more : Updated buffer ~A~%" foreign-read-buffer)
(when (> (remaining foreign-read-buffer) 0)
(setf (read-ready state-machine) nil)))))))
@@ -138,7 +139,7 @@
(do ((total-written 0))
((or (eql now-written -1) (eql (remaining foreign-write-buffer) 0)) total-written)
(progn
- (setf now-written (%write write-fd (buffer-buf foreign-write-buffer) (remaining foreign-write-buffer)))
+ (setf now-written (%write write-fd (buffer-pointer foreign-write-buffer) (remaining foreign-write-buffer)))
(when (not (eql now-written -1))
(inc-position foreign-write-buffer now-written)
(incf total-written now-written)))
@@ -166,7 +167,7 @@
(defmacro realloc-buffer(async-fd accessor size)
`(let ((buffer (,accessor ,async-fd)))
- (if (>= (buffer-capacity buffer) size)
+ (if (>= (buffer-capacity buffer) ,size)
t
(let ((new-buffer (byte-buffer ,size)))
(copy-buffer buffer new-buffer)
@@ -181,7 +182,6 @@
(ecase mode
(:read (realloc-buffer async-fd foreign-read-buffer size))
(:write (realloc-buffer async-fd foreign-write-buffer size)))))
-
(defun force-close-async-fd (async-fd)
@@ -208,3 +208,12 @@
(defun async-fd-write-fd (async-fd)
(slot-value async-fd 'write-fd))
+
+(defun test-realloc()
+ (let* ((sm (create-state-machine 'async-fd 1 1 6))
+ (pos-b4-resize (bytebuffer-write-string (foreign-read-buffer sm) "this string is OK")))
+ (recommend-buffer-size sm :read 4096)
+ (assert (eql 4096 (buffer-capacity (foreign-read-buffer sm))))
+ (assert (eql 4096 (nio-buffer:buffer-limit (foreign-read-buffer sm))))
+ (assert (eql pos-b4-resize (nio-buffer:buffer-position (foreign-read-buffer sm))))))
+
Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp
==============================================================================
--- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original)
+++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Mon Feb 26 22:00:16 2007
@@ -45,6 +45,8 @@
(defconstant +yarpc-packet-header-size+
(+ +PACKET-ID-SIZE+ +PACKET-LENGTH-SIZE+))
+(defconstant +yarpc-rpc-packet-header-size+ (+ +yarpc-packet-header-size+ +PACKET-REQUEST-ID-SIZE+))
+
(defmethod get-packet ((pf yarpc-packet-factory) buf)
(flip buf)
(if (>= (remaining buf) +yarpc-packet-header-size+) ;; First byte denotes packet ID ;;bytes 2,3,4,5 denote packet size ;; 6,7,8,9 request-id
@@ -92,7 +94,7 @@
(nio-buffer:bytebuffer-write-8 buf +CALL-METHOD-PACKET-ID+)
(nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later
(nio-buffer:bytebuffer-write-32 buf (request-id packet))
- (nio-buffer:bytebuffer-write-string buf (call-string packet))
+ (nio-buffer:bytebuffer-write-string buf (call-string packet) :utf-8)
(nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1)
#+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf)
)
@@ -102,8 +104,8 @@
(defmethod get-packet-size ((packet call-method-packet))
- (+ +yarpc-packet-header-size+
- (length (sb-ext:string-to-octets (write-to-string (call-string packet))))))
+ (+ +yarpc-rpc-packet-header-size+
+ (length (sb-ext:string-to-octets (call-string packet) :external-format :utf-8))))
(defclass method-response-packet (yarpc-packet)
((response :initarg :response
@@ -131,8 +133,9 @@
)
(buffer-too-small-error (err)
(nio-buffer:reset buf)
+#+nio-debug (format-log t "yarpc-packet-factory:write-bytes - buffer too small caught, reset to ~A~%" buf)
(error err))))
(defmethod get-packet-size ((packet method-response-packet))
- (+ +yarpc-packet-header-size+
+ (+ +yarpc-rpc-packet-header-size+
(length (sb-ext:string-to-octets (funcall +serialise-packet-fn+ (response packet)) :external-format :utf-8))))
Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp
==============================================================================
--- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original)
+++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Mon Feb 26 22:00:16 2007
@@ -79,11 +79,13 @@
(handler-case
(write-bytes outgoing-packet foreign-write-buffer)
(buffer-too-small-error (write-error1)
- (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet))
+ (let ((new-size (get-packet-size outgoing-packet)))
+ (format-log t "state-machine::process-write - write-error1 trying resize to ~A" new-size)
+ (if (recommend-buffer-size sm :write new-size)
(handler-case
(write-bytes outgoing-packet foreign-write-buffer)
- (buffer-too-small-error (write-error1) (format t "Failed to write packet after resize (something already in write buffer?, dropping packet ~A~% out buffer:~%~A~%" outgoing-packet foreign-write-buffer)))
- (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet))))))))
+ (buffer-too-small-error (write-error2) (format t "Failed to write packet after resize (something already in write buffer?, dropping packet ~A~% out buffer:~%~A~%" outgoing-packet foreign-write-buffer)))
+ (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet)))))))))
More information about the Nio-cvs
mailing list