diff --git a/NEWS.md b/NEWS.md
index 93aa1fa4..45dcb16e 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -35,6 +35,9 @@ where the formatting is also better._
   the functions called rather than just their names. (#504 @zeileis)
 - Legend labels are now correct if `by` is logical. Thanks to @TCornulier for
   the report. (#512 @grantmcdermott)
+- Axis limits are now correctly calculated for factor (and character) variables,
+  by coercing to numeric first. We also avoid the redundancy of re-calculating
+  axis limits for secondary plot layers. (#513 @grantmcdermott) 
 
 ### Documentation
 
diff --git a/R/lim.R b/R/lim.R
index 46d9c0e4..7eb13aad 100644
--- a/R/lim.R
+++ b/R/lim.R
@@ -23,14 +23,14 @@ lim_args = function(settings) {
   }
 
   if (is.null(xlim)) {
-    xlim = range(c(
+    xlim = range(as.numeric(c(
       datapoints[["x"]], datapoints[["xmin"]],
-      datapoints[["xmax"]]), finite = TRUE)
+      datapoints[["xmax"]])), finite = TRUE)
   }
   if (is.null(ylim)) {
-    ylim = range(c(
+    ylim = range(as.numeric(c(
       datapoints[["y"]], datapoints[["ymin"]],
-      datapoints[["ymax"]]), finite = TRUE)
+      datapoints[["ymax"]])), finite = TRUE)
   }
 
   if (identical(type, "boxplot")) {
diff --git a/R/tinyplot.R b/R/tinyplot.R
index ab40ee48..72fd8b12 100644
--- a/R/tinyplot.R
+++ b/R/tinyplot.R
@@ -882,8 +882,9 @@ tinyplot.default = function(
   #
   
   # do this after computing yaxb because limits will depend on the previous calculations
-  lim_args(settings)
-
+  if (!add) {
+    lim_args(settings)
+  }
 
   #
   ## facets: count -----