
(defun custom-read-line (stream &key (max 512) (utf-8b nil))
  "Reads a line from STREAM.  Uses #\Return as the line terminator, but
strips any trailing #\Return or #\Newline characters.
Will read up to MAX characters (defauts to 512).
By default, invalid encountered UTF-8 sequences will be imported as
LATIN-1 characters, unless UTF-8B, in which case invalid octets will
be imported as-is in a special unicode range suitable for later litteral
output."
  (let ((line (make-array max
			  :element-type 'character
			  :adjustable t
			  :fill-pointer 0)))
    (macrolet ((add-char (c)
		 `(vector-push ,c line))
	       (add-utf-8b-octet (o)
		 `(vector-push (code-char (logior #xDC00 (logand ,o #xFF)))
			       line)))
      (flet ((finalize-line ()
	       (loop
		  for c = (vector-pop line)
		  while (member c '(#\Return #\Newline))
		  finally (vector-push c line))
	       line))
	(loop
	   do
	     (let (
		   ;; No way to determine invalid octet values with old ECL,
		   ;; Return an unknown character code
		   (c #+old-ecl(handler-case
				   (read-char stream)
				 (simple-error ()
				   #\UFFFD))
		      ;; SBCL provides invalid octets which we can import and
		      ;; then issue an ATTEMPT-RESYNC restart to resume
		      #+sbcl(handler-bind
				((sb-int:stream-decoding-error
				  #'(lambda (e)
				      ;; Treat invalid UTF-8 octets as
				      ;; ISO-8859 characters.
				      (mapcar #'(lambda (c)
						  (when (> c 127)
						    (add-char (code-char c))))
					      (sb-int:character-decoding-error-octets e))
				      (invoke-restart 'sb-int:attempt-resync))))
			      (read-char stream))
		      ;; Test with new ECL
		      #+ecl(handler-bind
			       ((ext:stream-decoding-error
				 #'(lambda (e)
				     (mapcar #'(lambda (o)
						 (if (and utf-8b (> o 127))
						     (add-utf-8b-octet o)
						     (add-char
						      (code-char o))))
					     (ext:character-decoding-error-octets e))
				     (invoke-restart 'continue))))
			     (read-char stream))))
	       (when (char= #\Newline c)
		 (return (values (finalize-line) t)))
	       (add-char c)))))))

(defun display-utf8b-latin1 (string)
  "Formats a string which may contain UTF8-B encoded invalid sequence
octets for screen display, treating those as ISO-8859/LATIN-1 characters.
Note that if using this function to output back to file, the original
stream would be altered.  The UTF-8 decoder must output these bytes
litterally in the output stream to preserve the original octets."
  (map 'string #'(lambda (c)
		   (let ((n (char-code c)))
		     (if (>= #xDCFF n #xDC80)
			 (code-char (logand n #xFF))
			 c)))
       string))

(defun test ()
  (with-open-file (stream "/tmp/InvalidUTF8.txt")
    (loop
       do
	 (let ((line (handler-case
			 (custom-read-line stream :utf-8b t)
		       (end-of-file ()
			 (loop-finish)))))
	   (format t "~A~%" (display-utf8b-latin1 line))))))
