Neviem či je toto riešenie jednoduchšie ? Skús:

procedure TForm1.Memo1Change(Sender: TObject);
var LineNum: Integer;
begin
  LineNum:=Memo1.Perform(EM_LINEFROMCHAR, Memo1.SelStart, 0);
  if (Length(Memo1.Lines.Strings[Linenum]) > 60) or (Memo1.Lines.Count > 5) then
    Memo1.Perform(EM_UNDO, 0, 0);
  Memo1.Perform(EM_EMPTYUNDOBUFFER, 0, 0);
end;